diff --git "a/checkpoint-15000/trainer_state.json" "b/checkpoint-15000/trainer_state.json" new file mode 100644--- /dev/null +++ "b/checkpoint-15000/trainer_state.json" @@ -0,0 +1,109537 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.29789093220002383, + "global_step": 15000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 1.5873015873015873e-06, + "loss": 10.4755, + "theoretical_loss": 20.841284642178216, + "tokens_seen": 65536 + }, + { + "epoch": 0.0, + "learning_rate": 3.1746031746031746e-06, + "loss": 10.4444, + "theoretical_loss": 17.59466794495971, + "tokens_seen": 131072 + }, + { + "epoch": 0.0, + "learning_rate": 4.7619047619047615e-06, + "loss": 10.3622, + "theoretical_loss": 15.967943933467634, + "tokens_seen": 196608 + }, + { + "epoch": 0.0, + "learning_rate": 6.349206349206349e-06, + "loss": 10.1407, + "theoretical_loss": 14.920783596619636, + "tokens_seen": 262144 + }, + { + "epoch": 0.0, + "learning_rate": 7.936507936507936e-06, + "loss": 9.8682, + "theoretical_loss": 14.164683778394041, + "tokens_seen": 327680 + }, + { + "epoch": 0.0, + "learning_rate": 9.523809523809523e-06, + "loss": 9.6237, + "theoretical_loss": 13.581028313181289, + "tokens_seen": 393216 + }, + { + "epoch": 0.0, + "learning_rate": 1.1111111111111112e-05, + "loss": 9.3898, + "theoretical_loss": 13.110275842032255, + "tokens_seen": 458752 + }, + { + "epoch": 0.0, + "learning_rate": 1.2698412698412699e-05, + "loss": 9.2034, + "theoretical_loss": 12.71859646611439, + "tokens_seen": 524288 + }, + { + "epoch": 0.0, + "learning_rate": 1.4285714285714285e-05, + "loss": 9.093, + "theoretical_loss": 12.38505930450469, + "tokens_seen": 589824 + }, + { + "epoch": 0.0, + "learning_rate": 1.5873015873015872e-05, + "loss": 8.9514, + "theoretical_loss": 12.095879447666144, + "tokens_seen": 655360 + }, + { + "epoch": 0.0, + "learning_rate": 1.746031746031746e-05, + "loss": 8.8767, + "theoretical_loss": 11.841533071732742, + "tokens_seen": 720896 + }, + { + "epoch": 0.0, + "learning_rate": 1.9047619047619046e-05, + "loss": 8.7559, + "theoretical_loss": 11.615186049337796, + "tokens_seen": 786432 + }, + { + "epoch": 0.0, + "learning_rate": 2.0634920634920636e-05, + "loss": 8.6985, + "theoretical_loss": 11.411780980817854, + "tokens_seen": 851968 + }, + { + "epoch": 0.0, + "learning_rate": 2.2222222222222223e-05, + "loss": 8.6195, + "theoretical_loss": 11.227478542742938, + "tokens_seen": 917504 + }, + { + "epoch": 0.0, + "learning_rate": 2.380952380952381e-05, + "loss": 8.5266, + "theoretical_loss": 11.059300758608979, + "tokens_seen": 983040 + }, + { + "epoch": 0.0, + "learning_rate": 2.5396825396825397e-05, + "loss": 8.4812, + "theoretical_loss": 10.904894927088016, + "tokens_seen": 1048576 + }, + { + "epoch": 0.0, + "learning_rate": 2.6984126984126984e-05, + "loss": 8.574, + "theoretical_loss": 10.76237258069379, + "tokens_seen": 1114112 + }, + { + "epoch": 0.0, + "learning_rate": 2.857142857142857e-05, + "loss": 8.3926, + "theoretical_loss": 10.630196716861345, + "tokens_seen": 1179648 + }, + { + "epoch": 0.0, + "learning_rate": 3.0158730158730158e-05, + "loss": 8.3532, + "theoretical_loss": 10.507101012987707, + "tokens_seen": 1245184 + }, + { + "epoch": 0.0, + "learning_rate": 3.1746031746031745e-05, + "loss": 8.2864, + "theoretical_loss": 10.392030784394397, + "tokens_seen": 1310720 + }, + { + "epoch": 0.0, + "learning_rate": 3.3333333333333335e-05, + "loss": 8.1119, + "theoretical_loss": 10.284099061527812, + "tokens_seen": 1376256 + }, + { + "epoch": 0.0, + "learning_rate": 3.492063492063492e-05, + "loss": 8.1857, + "theoretical_loss": 10.182553393901085, + "tokens_seen": 1441792 + }, + { + "epoch": 0.0, + "learning_rate": 3.650793650793651e-05, + "loss": 8.1333, + "theoretical_loss": 10.086750402363664, + "tokens_seen": 1507328 + }, + { + "epoch": 0.0, + "learning_rate": 3.809523809523809e-05, + "loss": 8.1003, + "theoretical_loss": 9.996136019471344, + "tokens_seen": 1572864 + }, + { + "epoch": 0.0, + "learning_rate": 3.968253968253968e-05, + "loss": 7.941, + "theoretical_loss": 9.910229967024176, + "tokens_seen": 1638400 + }, + { + "epoch": 0.0, + "learning_rate": 4.126984126984127e-05, + "loss": 8.0529, + "theoretical_loss": 9.828613432171625, + "tokens_seen": 1703936 + }, + { + "epoch": 0.0, + "learning_rate": 4.2857142857142856e-05, + "loss": 7.9549, + "theoretical_loss": 9.75091918753851, + "tokens_seen": 1769472 + }, + { + "epoch": 0.0, + "learning_rate": 4.4444444444444447e-05, + "loss": 7.9587, + "theoretical_loss": 9.676823599712613, + "tokens_seen": 1835008 + }, + { + "epoch": 0.0, + "learning_rate": 4.603174603174603e-05, + "loss": 7.8542, + "theoretical_loss": 9.606040111776705, + "tokens_seen": 1900544 + }, + { + "epoch": 0.0, + "learning_rate": 4.761904761904762e-05, + "loss": 7.8426, + "theoretical_loss": 9.538313887395919, + "tokens_seen": 1966080 + }, + { + "epoch": 0.0, + "learning_rate": 4.9206349206349204e-05, + "loss": 7.7807, + "theoretical_loss": 9.473417378263001, + "tokens_seen": 2031616 + }, + { + "epoch": 0.0, + "learning_rate": 5.0793650793650794e-05, + "loss": 7.7797, + "theoretical_loss": 9.411146631541524, + "tokens_seen": 2097152 + }, + { + "epoch": 0.0, + "learning_rate": 5.2380952380952384e-05, + "loss": 7.6946, + "theoretical_loss": 9.351318194865048, + "tokens_seen": 2162688 + }, + { + "epoch": 0.0, + "learning_rate": 5.396825396825397e-05, + "loss": 7.6297, + "theoretical_loss": 9.293766507291341, + "tokens_seen": 2228224 + }, + { + "epoch": 0.0, + "learning_rate": 5.555555555555556e-05, + "loss": 7.735, + "theoretical_loss": 9.238341688075822, + "tokens_seen": 2293760 + }, + { + "epoch": 0.0, + "learning_rate": 5.714285714285714e-05, + "loss": 7.5127, + "theoretical_loss": 9.184907653139359, + "tokens_seen": 2359296 + }, + { + "epoch": 0.0, + "learning_rate": 5.873015873015873e-05, + "loss": 7.5195, + "theoretical_loss": 9.133340503044415, + "tokens_seen": 2424832 + }, + { + "epoch": 0.0, + "learning_rate": 6.0317460317460316e-05, + "loss": 7.3811, + "theoretical_loss": 9.0835271371648, + "tokens_seen": 2490368 + }, + { + "epoch": 0.0, + "learning_rate": 6.19047619047619e-05, + "loss": 7.3734, + "theoretical_loss": 9.035364057274894, + "tokens_seen": 2555904 + }, + { + "epoch": 0.0, + "learning_rate": 6.349206349206349e-05, + "loss": 7.4337, + "theoretical_loss": 8.988756330540422, + "tokens_seen": 2621440 + }, + { + "epoch": 0.0, + "learning_rate": 6.507936507936509e-05, + "loss": 7.2469, + "theoretical_loss": 8.943616687271678, + "tokens_seen": 2686976 + }, + { + "epoch": 0.0, + "learning_rate": 6.666666666666667e-05, + "loss": 7.2583, + "theoretical_loss": 8.89986473310929, + "tokens_seen": 2752512 + }, + { + "epoch": 0.0, + "learning_rate": 6.825396825396825e-05, + "loss": 7.0739, + "theoretical_loss": 8.857426258784564, + "tokens_seen": 2818048 + }, + { + "epoch": 0.0, + "learning_rate": 6.984126984126984e-05, + "loss": 7.0247, + "theoretical_loss": 8.816232633409479, + "tokens_seen": 2883584 + }, + { + "epoch": 0.0, + "learning_rate": 7.142857142857143e-05, + "loss": 7.2029, + "theoretical_loss": 8.776220269542522, + "tokens_seen": 2949120 + }, + { + "epoch": 0.0, + "learning_rate": 7.301587301587302e-05, + "loss": 7.0231, + "theoretical_loss": 8.737330150151898, + "tokens_seen": 3014656 + }, + { + "epoch": 0.0, + "learning_rate": 7.460317460317461e-05, + "loss": 6.9914, + "theoretical_loss": 8.699507409139912, + "tokens_seen": 3080192 + }, + { + "epoch": 0.0, + "learning_rate": 7.619047619047618e-05, + "loss": 6.982, + "theoretical_loss": 8.662700958366539, + "tokens_seen": 3145728 + }, + { + "epoch": 0.0, + "learning_rate": 7.777777777777778e-05, + "loss": 6.8828, + "theoretical_loss": 8.62686315516739, + "tokens_seen": 3211264 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.9953128099441528, + "objective/train/avg_token_score": 0.004687183536589146, + "objective/train/docs_used": 8371, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 6.929591178894043, + "objective/train/original_loss": 6.963510513305664, + "objective/train/theoretical_loss": 8.591949505242134, + "objective/train/tokens_used": 23736800, + "objective/train/unlikelihood_loss": 0.0002652419207151979, + "theoretical_loss": 8.591949505242134, + "tokens_seen": 3276800 + }, + { + "epoch": 0.0, + "learning_rate": 7.936507936507937e-05, + "loss": 6.8345, + "theoretical_loss": 8.591949505242134, + "tokens_seen": 3276800 + }, + { + "epoch": 0.0, + "learning_rate": 8.095238095238096e-05, + "loss": 6.7026, + "theoretical_loss": 8.557918396526182, + "tokens_seen": 3342336 + }, + { + "epoch": 0.0, + "learning_rate": 8.253968253968255e-05, + "loss": 6.6325, + "theoretical_loss": 8.524730860277067, + "tokens_seen": 3407872 + }, + { + "epoch": 0.0, + "learning_rate": 8.412698412698413e-05, + "loss": 6.6598, + "theoretical_loss": 8.492350356128252, + "tokens_seen": 3473408 + }, + { + "epoch": 0.0, + "learning_rate": 8.571428571428571e-05, + "loss": 6.5781, + "theoretical_loss": 8.460742578303845, + "tokens_seen": 3538944 + }, + { + "epoch": 0.0, + "learning_rate": 8.730158730158731e-05, + "loss": 6.584, + "theoretical_loss": 8.429875280561783, + "tokens_seen": 3604480 + }, + { + "epoch": 0.0, + "learning_rate": 8.888888888888889e-05, + "loss": 6.5512, + "theoretical_loss": 8.399718117751275, + "tokens_seen": 3670016 + }, + { + "epoch": 0.0, + "learning_rate": 9.047619047619048e-05, + "loss": 6.4639, + "theoretical_loss": 8.370242502142142, + "tokens_seen": 3735552 + }, + { + "epoch": 0.0, + "learning_rate": 9.206349206349206e-05, + "loss": 6.4369, + "theoretical_loss": 8.341421472916394, + "tokens_seen": 3801088 + }, + { + "epoch": 0.0, + "learning_rate": 9.365079365079366e-05, + "loss": 6.2528, + "theoretical_loss": 8.313229577412248, + "tokens_seen": 3866624 + }, + { + "epoch": 0.0, + "learning_rate": 9.523809523809524e-05, + "loss": 6.246, + "theoretical_loss": 8.28564276288293, + "tokens_seen": 3932160 + }, + { + "epoch": 0.0, + "learning_rate": 9.682539682539682e-05, + "loss": 6.294, + "theoretical_loss": 8.258638277681179, + "tokens_seen": 3997696 + }, + { + "epoch": 0.0, + "learning_rate": 9.841269841269841e-05, + "loss": 6.1423, + "theoretical_loss": 8.232194580909036, + "tokens_seen": 4063232 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001, + "loss": 6.1802, + "theoretical_loss": 8.20629125968411, + "tokens_seen": 4128768 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010158730158730159, + "loss": 6.2186, + "theoretical_loss": 8.180908953270682, + "tokens_seen": 4194304 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010317460317460319, + "loss": 6.0668, + "theoretical_loss": 8.156029283408564, + "tokens_seen": 4259840 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010476190476190477, + "loss": 6.0827, + "theoretical_loss": 8.131634790246775, + "tokens_seen": 4325376 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010634920634920635, + "loss": 6.1007, + "theoretical_loss": 8.107708873353706, + "tokens_seen": 4390912 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010793650793650794, + "loss": 5.9812, + "theoretical_loss": 8.084235737332481, + "tokens_seen": 4456448 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010952380952380953, + "loss": 5.9124, + "theoretical_loss": 8.061200341620074, + "tokens_seen": 4521984 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011111111111111112, + "loss": 5.9326, + "theoretical_loss": 8.038588354092902, + "tokens_seen": 4587520 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001126984126984127, + "loss": 5.7801, + "theoretical_loss": 8.016386108140445, + "tokens_seen": 4653056 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011428571428571428, + "loss": 5.9082, + "theoretical_loss": 7.994580562902867, + "tokens_seen": 4718592 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001158730158730159, + "loss": 5.983, + "theoretical_loss": 7.973159266399054, + "tokens_seen": 4784128 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011746031746031746, + "loss": 5.8184, + "theoretical_loss": 7.952110321298584, + "tokens_seen": 4849664 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011904761904761905, + "loss": 5.9356, + "theoretical_loss": 7.931422353115133, + "tokens_seen": 4915200 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012063492063492063, + "loss": 5.792, + "theoretical_loss": 7.911084480620269, + "tokens_seen": 4980736 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012222222222222224, + "loss": 5.711, + "theoretical_loss": 7.891086288295694, + "tokens_seen": 5046272 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001238095238095238, + "loss": 5.839, + "theoretical_loss": 7.871417800659003, + "tokens_seen": 5111808 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001253968253968254, + "loss": 5.8066, + "theoretical_loss": 7.8520694583133785, + "tokens_seen": 5177344 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012698412698412698, + "loss": 5.6526, + "theoretical_loss": 7.833032095585231, + "tokens_seen": 5242880 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012857142857142858, + "loss": 5.6699, + "theoretical_loss": 7.8142969196261385, + "tokens_seen": 5308416 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013015873015873017, + "loss": 5.6177, + "theoretical_loss": 7.79585549086643, + "tokens_seen": 5373952 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013174603174603174, + "loss": 5.7942, + "theoretical_loss": 7.777699704717685, + "tokens_seen": 5439488 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013333333333333334, + "loss": 5.7747, + "theoretical_loss": 7.759821774430384, + "tokens_seen": 5505024 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013492063492063494, + "loss": 5.6801, + "theoretical_loss": 7.742214215020986, + "tokens_seen": 5570560 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001365079365079365, + "loss": 5.6088, + "theoretical_loss": 7.72486982819, + "tokens_seen": 5636096 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001380952380952381, + "loss": 5.5572, + "theoretical_loss": 7.707781688159247, + "tokens_seen": 5701632 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013968253968253967, + "loss": 5.5776, + "theoretical_loss": 7.690943128362394, + "tokens_seen": 5767168 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001412698412698413, + "loss": 5.5834, + "theoretical_loss": 7.674347728928345, + "tokens_seen": 5832704 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014285714285714287, + "loss": 5.5071, + "theoretical_loss": 7.6579893049019265, + "tokens_seen": 5898240 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014444444444444444, + "loss": 5.6441, + "theoretical_loss": 7.641861895150762, + "tokens_seen": 5963776 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014603174603174603, + "loss": 5.5997, + "theoretical_loss": 7.625959751911346, + "tokens_seen": 6029312 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014761904761904763, + "loss": 5.5899, + "theoretical_loss": 7.610277330930954, + "tokens_seen": 6094848 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014920634920634923, + "loss": 5.5089, + "theoretical_loss": 7.5948092821654365, + "tokens_seen": 6160384 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001507936507936508, + "loss": 5.4879, + "theoretical_loss": 7.579550440996015, + "tokens_seen": 6225920 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015238095238095237, + "loss": 5.5742, + "theoretical_loss": 7.564495819930985, + "tokens_seen": 6291456 + }, + { + "epoch": 0.0, + "learning_rate": 0.000153968253968254, + "loss": 5.5275, + "theoretical_loss": 7.549640600760828, + "tokens_seen": 6356992 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015555555555555556, + "loss": 5.4252, + "theoretical_loss": 7.534980127137563, + "tokens_seen": 6422528 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015714285714285716, + "loss": 5.432, + "theoretical_loss": 7.520509897551315, + "tokens_seen": 6488064 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.9908171892166138, + "objective/train/avg_token_score": 0.00918284710496664, + "objective/train/docs_used": 9704, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 5.436452865600586, + "objective/train/original_loss": 5.488391876220703, + "objective/train/theoretical_loss": 7.5062255586790965, + "objective/train/tokens_used": 27013600, + "objective/train/unlikelihood_loss": 0.001355330110527575, + "theoretical_loss": 7.5062255586790965, + "tokens_seen": 6553600 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015873015873015873, + "loss": 5.494, + "theoretical_loss": 7.5062255586790965, + "tokens_seen": 6553600 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016031746031746033, + "loss": 5.5278, + "theoretical_loss": 7.492122899082528, + "tokens_seen": 6619136 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016190476190476192, + "loss": 5.2861, + "theoretical_loss": 7.478197843232982, + "tokens_seen": 6684672 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001634920634920635, + "loss": 5.4625, + "theoretical_loss": 7.464446445844085, + "tokens_seen": 6750208 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001650793650793651, + "loss": 5.5041, + "theoretical_loss": 7.450864886492971, + "tokens_seen": 6815744 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001666666666666667, + "loss": 5.2766, + "theoretical_loss": 7.437449464512941, + "tokens_seen": 6881280 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016825396825396826, + "loss": 5.3216, + "theoretical_loss": 7.424196594141411, + "tokens_seen": 6946816 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016984126984126986, + "loss": 5.4384, + "theoretical_loss": 7.4111027999080985, + "tokens_seen": 7012352 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017142857142857143, + "loss": 5.4936, + "theoretical_loss": 7.398164712249439, + "tokens_seen": 7077888 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017301587301587302, + "loss": 5.3274, + "theoretical_loss": 7.3853790633361545, + "tokens_seen": 7143424 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017460317460317462, + "loss": 5.3547, + "theoretical_loss": 7.3727426831017535, + "tokens_seen": 7208960 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001761904761904762, + "loss": 5.3694, + "theoretical_loss": 7.360252495460585, + "tokens_seen": 7274496 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017777777777777779, + "loss": 5.3464, + "theoretical_loss": 7.347905514704743, + "tokens_seen": 7340032 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017936507936507938, + "loss": 5.3432, + "theoretical_loss": 7.33569884206989, + "tokens_seen": 7405568 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018095238095238095, + "loss": 5.4617, + "theoretical_loss": 7.32362966246063, + "tokens_seen": 7471104 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018253968253968255, + "loss": 5.1651, + "theoretical_loss": 7.311695241326718, + "tokens_seen": 7536640 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018412698412698412, + "loss": 5.2532, + "theoretical_loss": 7.299892921681898, + "tokens_seen": 7602176 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018571428571428572, + "loss": 5.364, + "theoretical_loss": 7.288220121257689, + "tokens_seen": 7667712 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018730158730158731, + "loss": 5.315, + "theoretical_loss": 7.276674329784936, + "tokens_seen": 7733248 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018888888888888888, + "loss": 5.2799, + "theoretical_loss": 7.265253106396328, + "tokens_seen": 7798784 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019047619047619048, + "loss": 5.4116, + "theoretical_loss": 7.253954077143563, + "tokens_seen": 7864320 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019206349206349208, + "loss": 5.2859, + "theoretical_loss": 7.242774932623186, + "tokens_seen": 7929856 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019365079365079365, + "loss": 5.4065, + "theoretical_loss": 7.2317134257054665, + "tokens_seen": 7995392 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019523809523809525, + "loss": 5.1153, + "theoretical_loss": 7.220767369361081, + "tokens_seen": 8060928 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019682539682539682, + "loss": 5.2473, + "theoretical_loss": 7.2099346345805815, + "tokens_seen": 8126464 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019841269841269844, + "loss": 5.2971, + "theoretical_loss": 7.1992131483820305, + "tokens_seen": 8192000 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002, + "loss": 5.1758, + "theoretical_loss": 7.188600891902338, + "tokens_seen": 8257536 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020158730158730158, + "loss": 5.2227, + "theoretical_loss": 7.178095898568204, + "tokens_seen": 8323072 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020317460317460318, + "loss": 5.1411, + "theoretical_loss": 7.167696252342704, + "tokens_seen": 8388608 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020476190476190475, + "loss": 5.1665, + "theoretical_loss": 7.15740008604386, + "tokens_seen": 8454144 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020634920634920637, + "loss": 5.1855, + "theoretical_loss": 7.147205579731682, + "tokens_seen": 8519680 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020793650793650797, + "loss": 5.1706, + "theoretical_loss": 7.1371109591604105, + "tokens_seen": 8585216 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020952380952380954, + "loss": 5.1392, + "theoretical_loss": 7.127114494292836, + "tokens_seen": 8650752 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021111111111111113, + "loss": 5.0521, + "theoretical_loss": 7.117214497873759, + "tokens_seen": 8716288 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002126984126984127, + "loss": 5.0186, + "theoretical_loss": 7.107409324059815, + "tokens_seen": 8781824 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002142857142857143, + "loss": 5.2658, + "theoretical_loss": 7.097697367103032, + "tokens_seen": 8847360 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021587301587301587, + "loss": 5.0658, + "theoretical_loss": 7.0880770600856335, + "tokens_seen": 8912896 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021746031746031744, + "loss": 4.9223, + "theoretical_loss": 7.0785468737037345, + "tokens_seen": 8978432 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021904761904761907, + "loss": 5.2035, + "theoretical_loss": 7.069105315097691, + "tokens_seen": 9043968 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022063492063492066, + "loss": 5.0555, + "theoretical_loss": 7.059750926726999, + "tokens_seen": 9109504 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022222222222222223, + "loss": 5.1345, + "theoretical_loss": 7.050482285287744, + "tokens_seen": 9175040 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022380952380952383, + "loss": 5.109, + "theoretical_loss": 7.041298000670675, + "tokens_seen": 9240576 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002253968253968254, + "loss": 5.1491, + "theoretical_loss": 7.032196714958137, + "tokens_seen": 9306112 + }, + { + "epoch": 0.0, + "learning_rate": 0.000226984126984127, + "loss": 5.0923, + "theoretical_loss": 7.023177101458123, + "tokens_seen": 9371648 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022857142857142857, + "loss": 5.1178, + "theoretical_loss": 7.014237863773846, + "tokens_seen": 9437184 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023015873015873016, + "loss": 5.0458, + "theoretical_loss": 7.005377734907255, + "tokens_seen": 9502720 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002317460317460318, + "loss": 5.0774, + "theoretical_loss": 6.996595476395081, + "tokens_seen": 9568256 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023333333333333336, + "loss": 5.1337, + "theoretical_loss": 6.98788987747596, + "tokens_seen": 9633792 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023492063492063493, + "loss": 5.0676, + "theoretical_loss": 6.979259754287375, + "tokens_seen": 9699328 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023650793650793653, + "loss": 5.0398, + "theoretical_loss": 6.970703949091095, + "tokens_seen": 9764864 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.9910753965377808, + "objective/train/avg_token_score": 0.00892461184412241, + "objective/train/docs_used": 11015, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 5.1138224601745605, + "objective/train/original_loss": 5.160437107086182, + "objective/train/theoretical_loss": 6.962221329525966, + "objective/train/tokens_used": 30290400, + "objective/train/unlikelihood_loss": 0.0020566661842167377, + "theoretical_loss": 6.962221329525966, + "tokens_seen": 9830400 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002380952380952381, + "loss": 5.1006, + "theoretical_loss": 6.962221329525966, + "tokens_seen": 9830400 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002396825396825397, + "loss": 5.0831, + "theoretical_loss": 6.953810787886883, + "tokens_seen": 9895936 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024126984126984126, + "loss": 5.0469, + "theoretical_loss": 6.945471240428864, + "tokens_seen": 9961472 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024285714285714286, + "loss": 5.0469, + "theoretical_loss": 6.937201626695213, + "tokens_seen": 10027008 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002444444444444445, + "loss": 5.1141, + "theoretical_loss": 6.929000908868754, + "tokens_seen": 10092544 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024603174603174605, + "loss": 5.0101, + "theoretical_loss": 6.920868071145226, + "tokens_seen": 10158080 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002476190476190476, + "loss": 5.0588, + "theoretical_loss": 6.912802119127937, + "tokens_seen": 10223616 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024920634920634925, + "loss": 5.2155, + "theoretical_loss": 6.904802079242816, + "tokens_seen": 10289152 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002507936507936508, + "loss": 4.9437, + "theoretical_loss": 6.896866998173063, + "tokens_seen": 10354688 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002523809523809524, + "loss": 5.0547, + "theoretical_loss": 6.888995942312617, + "tokens_seen": 10420224 + }, + { + "epoch": 0.0, + "learning_rate": 0.00025396825396825396, + "loss": 5.0461, + "theoretical_loss": 6.881187997237696, + "tokens_seen": 10485760 + }, + { + "epoch": 0.0, + "learning_rate": 0.00025555555555555553, + "loss": 4.9953, + "theoretical_loss": 6.873442267195708, + "tokens_seen": 10551296 + }, + { + "epoch": 0.0, + "learning_rate": 0.00025714285714285715, + "loss": 5.1654, + "theoretical_loss": 6.8657578746108605, + "tokens_seen": 10616832 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002587301587301588, + "loss": 4.8415, + "theoretical_loss": 6.858133959605815, + "tokens_seen": 10682368 + }, + { + "epoch": 0.0, + "learning_rate": 0.00026031746031746035, + "loss": 4.9456, + "theoretical_loss": 6.8505696795387845, + "tokens_seen": 10747904 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002619047619047619, + "loss": 4.9676, + "theoretical_loss": 6.843064208555461, + "tokens_seen": 10813440 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002634920634920635, + "loss": 5.0099, + "theoretical_loss": 6.835616737155247, + "tokens_seen": 10878976 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002650793650793651, + "loss": 4.8238, + "theoretical_loss": 6.828226471771211, + "tokens_seen": 10944512 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002666666666666667, + "loss": 4.9021, + "theoretical_loss": 6.8208926343632905, + "tokens_seen": 11010048 + }, + { + "epoch": 0.0, + "learning_rate": 0.00026825396825396825, + "loss": 4.6999, + "theoretical_loss": 6.813614462024214, + "tokens_seen": 11075584 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002698412698412699, + "loss": 4.9596, + "theoretical_loss": 6.806391206597704, + "tokens_seen": 11141120 + }, + { + "epoch": 0.0, + "learning_rate": 0.00027142857142857144, + "loss": 4.8783, + "theoretical_loss": 6.799222134308489, + "tokens_seen": 11206656 + }, + { + "epoch": 0.0, + "learning_rate": 0.000273015873015873, + "loss": 4.9912, + "theoretical_loss": 6.792106525403691, + "tokens_seen": 11272192 + }, + { + "epoch": 0.0, + "learning_rate": 0.00027460317460317464, + "loss": 4.8223, + "theoretical_loss": 6.785043673805189, + "tokens_seen": 11337728 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002761904761904762, + "loss": 4.9036, + "theoretical_loss": 6.778032886772552, + "tokens_seen": 11403264 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002777777777777778, + "loss": 4.9886, + "theoretical_loss": 6.771073484576156, + "tokens_seen": 11468800 + }, + { + "epoch": 0.0, + "learning_rate": 0.00027936507936507935, + "loss": 4.8748, + "theoretical_loss": 6.764164800180141, + "tokens_seen": 11534336 + }, + { + "epoch": 0.0, + "learning_rate": 0.00028095238095238097, + "loss": 4.8016, + "theoretical_loss": 6.757306178934827, + "tokens_seen": 11599872 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002825396825396826, + "loss": 4.9062, + "theoretical_loss": 6.750496978278292, + "tokens_seen": 11665408 + }, + { + "epoch": 0.0, + "learning_rate": 0.00028412698412698417, + "loss": 4.8568, + "theoretical_loss": 6.743736567446755, + "tokens_seen": 11730944 + }, + { + "epoch": 0.0, + "learning_rate": 0.00028571428571428574, + "loss": 4.7161, + "theoretical_loss": 6.737024327193478, + "tokens_seen": 11796480 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002873015873015873, + "loss": 4.7303, + "theoretical_loss": 6.73035964951589, + "tokens_seen": 11862016 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002888888888888889, + "loss": 4.9115, + "theoretical_loss": 6.723741937390625, + "tokens_seen": 11927552 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002904761904761905, + "loss": 4.9591, + "theoretical_loss": 6.717170604516246, + "tokens_seen": 11993088 + }, + { + "epoch": 0.0, + "learning_rate": 0.00029206349206349207, + "loss": 4.863, + "theoretical_loss": 6.71064507506334, + "tokens_seen": 12058624 + }, + { + "epoch": 0.0, + "learning_rate": 0.00029365079365079364, + "loss": 4.8365, + "theoretical_loss": 6.704164783431781, + "tokens_seen": 12124160 + }, + { + "epoch": 0.0, + "learning_rate": 0.00029523809523809526, + "loss": 4.8273, + "theoretical_loss": 6.6977291740148805, + "tokens_seen": 12189696 + }, + { + "epoch": 0.0, + "learning_rate": 0.00029682539682539683, + "loss": 4.7314, + "theoretical_loss": 6.691337700970224, + "tokens_seen": 12255232 + }, + { + "epoch": 0.0, + "learning_rate": 0.00029841269841269846, + "loss": 4.9959, + "theoretical_loss": 6.684989827996945, + "tokens_seen": 12320768 + }, + { + "epoch": 0.0, + "learning_rate": 0.00030000000000000003, + "loss": 4.9806, + "theoretical_loss": 6.6786850281192365, + "tokens_seen": 12386304 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003015873015873016, + "loss": 4.8476, + "theoretical_loss": 6.672422783475897, + "tokens_seen": 12451840 + }, + { + "epoch": 0.0, + "learning_rate": 0.00030317460317460317, + "loss": 5.0168, + "theoretical_loss": 6.666202585115692, + "tokens_seen": 12517376 + }, + { + "epoch": 0.0, + "learning_rate": 0.00030476190476190474, + "loss": 4.8744, + "theoretical_loss": 6.66002393279837, + "tokens_seen": 12582912 + }, + { + "epoch": 0.0, + "learning_rate": 0.00030634920634920636, + "loss": 4.8636, + "theoretical_loss": 6.653886334801115, + "tokens_seen": 12648448 + }, + { + "epoch": 0.0, + "learning_rate": 0.000307936507936508, + "loss": 4.7679, + "theoretical_loss": 6.647789307730289, + "tokens_seen": 12713984 + }, + { + "epoch": 0.0, + "learning_rate": 0.00030952380952380956, + "loss": 4.7884, + "theoretical_loss": 6.641732376338277, + "tokens_seen": 12779520 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003111111111111111, + "loss": 4.8005, + "theoretical_loss": 6.6357150733452785, + "tokens_seen": 12845056 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003126984126984127, + "loss": 4.7984, + "theoretical_loss": 6.629736939265876, + "tokens_seen": 12910592 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003142857142857143, + "loss": 4.7541, + "theoretical_loss": 6.623797522240246, + "tokens_seen": 12976128 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003158730158730159, + "loss": 4.7873, + "theoretical_loss": 6.617896377869851, + "tokens_seen": 13041664 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.9912475347518921, + "objective/train/avg_token_score": 0.008752506226301193, + "objective/train/docs_used": 12170, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 4.886944770812988, + "objective/train/original_loss": 4.931352138519287, + "objective/train/theoretical_loss": 6.61203306905748, + "objective/train/tokens_used": 33567200, + "objective/train/unlikelihood_loss": 0.001898016780614853, + "theoretical_loss": 6.61203306905748, + "tokens_seen": 13107200 + }, + { + "epoch": 0.0, + "learning_rate": 0.00031746031746031746, + "loss": 4.8465, + "theoretical_loss": 6.61203306905748, + "tokens_seen": 13107200 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003190476190476191, + "loss": 4.6854, + "theoretical_loss": 6.6062071658514965, + "tokens_seen": 13172736 + }, + { + "epoch": 0.0, + "learning_rate": 0.00032063492063492065, + "loss": 4.8793, + "theoretical_loss": 6.600418245294164, + "tokens_seen": 13238272 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003222222222222222, + "loss": 4.7428, + "theoretical_loss": 6.594665891273929, + "tokens_seen": 13303808 + }, + { + "epoch": 0.0, + "learning_rate": 0.00032380952380952385, + "loss": 4.9047, + "theoretical_loss": 6.588949694381529, + "tokens_seen": 13369344 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003253968253968254, + "loss": 4.7527, + "theoretical_loss": 6.583269251769811, + "tokens_seen": 13434880 + }, + { + "epoch": 0.0, + "learning_rate": 0.000326984126984127, + "loss": 4.8349, + "theoretical_loss": 6.577624167017157, + "tokens_seen": 13500416 + }, + { + "epoch": 0.0, + "learning_rate": 0.00032857142857142856, + "loss": 4.6754, + "theoretical_loss": 6.5720140499943955, + "tokens_seen": 13565952 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003301587301587302, + "loss": 4.7073, + "theoretical_loss": 6.566438516735092, + "tokens_seen": 13631488 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003317460317460318, + "loss": 4.6892, + "theoretical_loss": 6.560897189309134, + "tokens_seen": 13697024 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003333333333333334, + "loss": 4.6562, + "theoretical_loss": 6.5553896956994855, + "tokens_seen": 13762560 + }, + { + "epoch": 0.0, + "learning_rate": 0.00033492063492063495, + "loss": 4.7095, + "theoretical_loss": 6.5499156696820435, + "tokens_seen": 13828096 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003365079365079365, + "loss": 4.7216, + "theoretical_loss": 6.544474750708482, + "tokens_seen": 13893632 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003380952380952381, + "loss": 4.7049, + "theoretical_loss": 6.539066583792016, + "tokens_seen": 13959168 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003396825396825397, + "loss": 4.8103, + "theoretical_loss": 6.533690819395976, + "tokens_seen": 14024704 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003412698412698413, + "loss": 4.721, + "theoretical_loss": 6.528347113325134, + "tokens_seen": 14090240 + }, + { + "epoch": 0.0, + "learning_rate": 0.00034285714285714285, + "loss": 4.7148, + "theoretical_loss": 6.523035126619684, + "tokens_seen": 14155776 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003444444444444445, + "loss": 4.7429, + "theoretical_loss": 6.517754525451811, + "tokens_seen": 14221312 + }, + { + "epoch": 0.0, + "learning_rate": 0.00034603174603174604, + "loss": 4.7287, + "theoretical_loss": 6.512504981024765, + "tokens_seen": 14286848 + }, + { + "epoch": 0.0, + "learning_rate": 0.00034761904761904767, + "loss": 4.6444, + "theoretical_loss": 6.507286169474376, + "tokens_seen": 14352384 + }, + { + "epoch": 0.0, + "learning_rate": 0.00034920634920634924, + "loss": 4.7226, + "theoretical_loss": 6.502097771772935, + "tokens_seen": 14417920 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003507936507936508, + "loss": 4.6621, + "theoretical_loss": 6.496939473635374, + "tokens_seen": 14483456 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003523809523809524, + "loss": 4.5794, + "theoretical_loss": 6.491810965427682, + "tokens_seen": 14548992 + }, + { + "epoch": 0.0, + "learning_rate": 0.00035396825396825395, + "loss": 4.6649, + "theoretical_loss": 6.486711942077496, + "tokens_seen": 14614528 + }, + { + "epoch": 0.0, + "learning_rate": 0.00035555555555555557, + "loss": 4.6843, + "theoretical_loss": 6.481642102986796, + "tokens_seen": 14680064 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003571428571428572, + "loss": 4.7769, + "theoretical_loss": 6.476601151946659, + "tokens_seen": 14745600 + }, + { + "epoch": 0.0, + "learning_rate": 0.00035873015873015877, + "loss": 4.6632, + "theoretical_loss": 6.471588797054005, + "tokens_seen": 14811136 + }, + { + "epoch": 0.0, + "learning_rate": 0.00036031746031746034, + "loss": 4.6595, + "theoretical_loss": 6.46660475063028, + "tokens_seen": 14876672 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003619047619047619, + "loss": 4.7172, + "theoretical_loss": 6.46164872914203, + "tokens_seen": 14942208 + }, + { + "epoch": 0.0, + "learning_rate": 0.00036349206349206353, + "loss": 4.7936, + "theoretical_loss": 6.456720453123305, + "tokens_seen": 15007744 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003650793650793651, + "loss": 4.6977, + "theoretical_loss": 6.4518196470998515, + "tokens_seen": 15073280 + }, + { + "epoch": 0.0, + "learning_rate": 0.00036666666666666667, + "loss": 4.5771, + "theoretical_loss": 6.446946039515035, + "tokens_seen": 15138816 + }, + { + "epoch": 0.0, + "learning_rate": 0.00036825396825396824, + "loss": 4.5335, + "theoretical_loss": 6.442099362657457, + "tokens_seen": 15204352 + }, + { + "epoch": 0.0, + "learning_rate": 0.00036984126984126986, + "loss": 4.4152, + "theoretical_loss": 6.437279352590206, + "tokens_seen": 15269888 + }, + { + "epoch": 0.0, + "learning_rate": 0.00037142857142857143, + "loss": 4.571, + "theoretical_loss": 6.432485749081721, + "tokens_seen": 15335424 + }, + { + "epoch": 0.0, + "learning_rate": 0.00037301587301587306, + "loss": 4.5632, + "theoretical_loss": 6.427718295538199, + "tokens_seen": 15400960 + }, + { + "epoch": 0.0, + "learning_rate": 0.00037460317460317463, + "loss": 4.5716, + "theoretical_loss": 6.422976738937518, + "tokens_seen": 15466496 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003761904761904762, + "loss": 4.6853, + "theoretical_loss": 6.418260829764641, + "tokens_seen": 15532032 + }, + { + "epoch": 0.0, + "learning_rate": 0.00037777777777777777, + "loss": 4.7564, + "theoretical_loss": 6.413570321948453, + "tokens_seen": 15597568 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003793650793650794, + "loss": 4.5313, + "theoretical_loss": 6.40890497279999, + "tokens_seen": 15663104 + }, + { + "epoch": 0.0, + "learning_rate": 0.00038095238095238096, + "loss": 4.6649, + "theoretical_loss": 6.404264542952033, + "tokens_seen": 15728640 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003825396825396826, + "loss": 4.5488, + "theoretical_loss": 6.399648796300032, + "tokens_seen": 15794176 + }, + { + "epoch": 0.0, + "learning_rate": 0.00038412698412698416, + "loss": 4.5326, + "theoretical_loss": 6.395057499944313, + "tokens_seen": 15859712 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003857142857142857, + "loss": 4.6869, + "theoretical_loss": 6.390490424133546, + "tokens_seen": 15925248 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003873015873015873, + "loss": 4.6643, + "theoretical_loss": 6.3859473422094375, + "tokens_seen": 15990784 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003888888888888889, + "loss": 4.7256, + "theoretical_loss": 6.381428030552619, + "tokens_seen": 16056320 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003904761904761905, + "loss": 4.7032, + "theoretical_loss": 6.376932268529695, + "tokens_seen": 16121856 + }, + { + "epoch": 0.0, + "learning_rate": 0.00039206349206349206, + "loss": 4.6803, + "theoretical_loss": 6.372459838441432, + "tokens_seen": 16187392 + }, + { + "epoch": 0.0, + "learning_rate": 0.00039365079365079363, + "loss": 4.6183, + "theoretical_loss": 6.368010525472036, + "tokens_seen": 16252928 + }, + { + "epoch": 0.0, + "learning_rate": 0.00039523809523809526, + "loss": 4.6416, + "theoretical_loss": 6.363584117639526, + "tokens_seen": 16318464 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.9927738308906555, + "objective/train/avg_token_score": 0.007226217538118362, + "objective/train/docs_used": 13382, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 4.352573394775391, + "objective/train/original_loss": 4.3809919357299805, + "objective/train/theoretical_loss": 6.35918040574715, + "objective/train/tokens_used": 36844000, + "objective/train/unlikelihood_loss": 0.0038730271626263857, + "theoretical_loss": 6.35918040574715, + "tokens_seen": 16384000 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003968253968253969, + "loss": 4.5309, + "theoretical_loss": 6.35918040574715, + "tokens_seen": 16384000 + }, + { + "epoch": 0.0, + "learning_rate": 0.00039841269841269845, + "loss": 4.5571, + "theoretical_loss": 6.354799183335816, + "tokens_seen": 16449536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004, + "loss": 4.4283, + "theoretical_loss": 6.350440246637536, + "tokens_seen": 16515072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004015873015873016, + "loss": 4.4531, + "theoretical_loss": 6.3461033945298295, + "tokens_seen": 16580608 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040317460317460316, + "loss": 4.5382, + "theoretical_loss": 6.341788428491087, + "tokens_seen": 16646144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004047619047619048, + "loss": 4.4828, + "theoretical_loss": 6.337495152556851, + "tokens_seen": 16711680 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040634920634920635, + "loss": 4.5352, + "theoretical_loss": 6.333223373277006, + "tokens_seen": 16777216 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004079365079365079, + "loss": 4.7241, + "theoretical_loss": 6.32897289967384, + "tokens_seen": 16842752 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004095238095238095, + "loss": 4.545, + "theoretical_loss": 6.324743543200984, + "tokens_seen": 16908288 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004111111111111111, + "loss": 4.542, + "theoretical_loss": 6.320535117703164, + "tokens_seen": 16973824 + }, + { + "epoch": 0.01, + "learning_rate": 0.00041269841269841274, + "loss": 4.5529, + "theoretical_loss": 6.316347439376797, + "tokens_seen": 17039360 + }, + { + "epoch": 0.01, + "learning_rate": 0.00041428571428571437, + "loss": 4.5244, + "theoretical_loss": 6.312180326731362, + "tokens_seen": 17104896 + }, + { + "epoch": 0.01, + "learning_rate": 0.00041587301587301594, + "loss": 4.5556, + "theoretical_loss": 6.308033600551573, + "tokens_seen": 17170432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004174603174603175, + "loss": 4.3093, + "theoretical_loss": 6.303907083860294, + "tokens_seen": 17235968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004190476190476191, + "loss": 4.6846, + "theoretical_loss": 6.2998006018822075, + "tokens_seen": 17301504 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042063492063492065, + "loss": 4.4682, + "theoretical_loss": 6.295713982008203, + "tokens_seen": 17367040 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042222222222222227, + "loss": 4.5388, + "theoretical_loss": 6.2916470537604745, + "tokens_seen": 17432576 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042380952380952384, + "loss": 4.4485, + "theoretical_loss": 6.287599648758312, + "tokens_seen": 17498112 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004253968253968254, + "loss": 4.3163, + "theoretical_loss": 6.283571600684564, + "tokens_seen": 17563648 + }, + { + "epoch": 0.01, + "learning_rate": 0.000426984126984127, + "loss": 4.2622, + "theoretical_loss": 6.279562745252756, + "tokens_seen": 17629184 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004285714285714286, + "loss": 4.4746, + "theoretical_loss": 6.2755729201748665, + "tokens_seen": 17694720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004301587301587302, + "loss": 4.5726, + "theoretical_loss": 6.271601965129717, + "tokens_seen": 17760256 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043174603174603174, + "loss": 4.5421, + "theoretical_loss": 6.267649721731981, + "tokens_seen": 17825792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004333333333333333, + "loss": 4.4347, + "theoretical_loss": 6.263716033501797, + "tokens_seen": 17891328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004349206349206349, + "loss": 4.3358, + "theoretical_loss": 6.259800745834964, + "tokens_seen": 17956864 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004365079365079365, + "loss": 4.4183, + "theoretical_loss": 6.255903705973709, + "tokens_seen": 18022400 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043809523809523813, + "loss": 4.4832, + "theoretical_loss": 6.252024762978021, + "tokens_seen": 18087936 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043968253968253976, + "loss": 4.3822, + "theoretical_loss": 6.248163767697523, + "tokens_seen": 18153472 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004412698412698413, + "loss": 4.5726, + "theoretical_loss": 6.244320572743888, + "tokens_seen": 18219008 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004428571428571429, + "loss": 4.4388, + "theoretical_loss": 6.240495032463768, + "tokens_seen": 18284544 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044444444444444447, + "loss": 4.4043, + "theoretical_loss": 6.236687002912243, + "tokens_seen": 18350080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004460317460317461, + "loss": 4.3967, + "theoretical_loss": 6.232896341826766, + "tokens_seen": 18415616 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044761904761904766, + "loss": 4.2459, + "theoretical_loss": 6.229122908601596, + "tokens_seen": 18481152 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044920634920634923, + "loss": 4.5508, + "theoretical_loss": 6.225366564262705, + "tokens_seen": 18546688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004507936507936508, + "loss": 4.4064, + "theoretical_loss": 6.2216271714431635, + "tokens_seen": 18612224 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045238095238095237, + "loss": 4.4125, + "theoretical_loss": 6.217904594358968, + "tokens_seen": 18677760 + }, + { + "epoch": 0.01, + "learning_rate": 0.000453968253968254, + "loss": 4.436, + "theoretical_loss": 6.214198698785328, + "tokens_seen": 18743296 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045555555555555556, + "loss": 4.4064, + "theoretical_loss": 6.210509352033379, + "tokens_seen": 18808832 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045714285714285713, + "loss": 4.5662, + "theoretical_loss": 6.206836422927337, + "tokens_seen": 18874368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004587301587301587, + "loss": 4.5651, + "theoretical_loss": 6.203179781782049, + "tokens_seen": 18939904 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046031746031746033, + "loss": 4.4912, + "theoretical_loss": 6.199539300380979, + "tokens_seen": 19005440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004619047619047619, + "loss": 4.459, + "theoretical_loss": 6.195914851954575, + "tokens_seen": 19070976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004634920634920636, + "loss": 4.3886, + "theoretical_loss": 6.19230631115903, + "tokens_seen": 19136512 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046507936507936515, + "loss": 4.3482, + "theoretical_loss": 6.188713554055438, + "tokens_seen": 19202048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004666666666666667, + "loss": 4.216, + "theoretical_loss": 6.185136458089309, + "tokens_seen": 19267584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004682539682539683, + "loss": 4.4629, + "theoretical_loss": 6.181574902070463, + "tokens_seen": 19333120 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046984126984126986, + "loss": 4.4538, + "theoretical_loss": 6.178028766153277, + "tokens_seen": 19398656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004714285714285715, + "loss": 4.4516, + "theoretical_loss": 6.174497931817287, + "tokens_seen": 19464192 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047301587301587305, + "loss": 4.4131, + "theoretical_loss": 6.17098228184813, + "tokens_seen": 19529728 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004746031746031746, + "loss": 4.3083, + "theoretical_loss": 6.167481700318827, + "tokens_seen": 19595264 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.9917816519737244, + "objective/train/avg_token_score": 0.008218348026275635, + "objective/train/docs_used": 14656, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 4.319173812866211, + "objective/train/original_loss": 4.356578350067139, + "objective/train/theoretical_loss": 6.163996072571392, + "objective/train/tokens_used": 40120800, + "objective/train/unlikelihood_loss": 0.00279328809119761, + "theoretical_loss": 6.163996072571392, + "tokens_seen": 19660800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004761904761904762, + "loss": 4.391, + "theoretical_loss": 6.163996072571392, + "tokens_seen": 19660800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004777777777777778, + "loss": 4.4053, + "theoretical_loss": 6.160525285198771, + "tokens_seen": 19726336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004793650793650794, + "loss": 4.319, + "theoretical_loss": 6.157069226027087, + "tokens_seen": 19791872 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048095238095238095, + "loss": 4.2471, + "theoretical_loss": 6.153627784098204, + "tokens_seen": 19857408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004825396825396825, + "loss": 4.3262, + "theoretical_loss": 6.150200849652586, + "tokens_seen": 19922944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004841269841269841, + "loss": 4.3516, + "theoretical_loss": 6.146788314112464, + "tokens_seen": 19988480 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004857142857142857, + "loss": 4.2878, + "theoretical_loss": 6.143390070065279, + "tokens_seen": 20054016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004873015873015873, + "loss": 4.4226, + "theoretical_loss": 6.140006011247426, + "tokens_seen": 20119552 + }, + { + "epoch": 0.01, + "learning_rate": 0.000488888888888889, + "loss": 4.4901, + "theoretical_loss": 6.136636032528262, + "tokens_seen": 20185088 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004904761904761905, + "loss": 4.416, + "theoretical_loss": 6.133280029894401, + "tokens_seen": 20250624 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004920634920634921, + "loss": 4.3676, + "theoretical_loss": 6.129937900434269, + "tokens_seen": 20316160 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004936507936507937, + "loss": 4.291, + "theoretical_loss": 6.126609542322921, + "tokens_seen": 20381696 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004952380952380952, + "loss": 4.2855, + "theoretical_loss": 6.123294854807119, + "tokens_seen": 20447232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004968253968253969, + "loss": 4.4341, + "theoretical_loss": 6.11999373819066, + "tokens_seen": 20512768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984126984126985, + "loss": 4.1383, + "theoretical_loss": 6.116706093819951, + "tokens_seen": 20578304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005, + "loss": 4.3596, + "theoretical_loss": 6.113431824069829, + "tokens_seen": 20643840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005015873015873016, + "loss": 4.3174, + "theoretical_loss": 6.110170832329612, + "tokens_seen": 20709376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005031746031746032, + "loss": 4.3176, + "theoretical_loss": 6.10692302298939, + "tokens_seen": 20774912 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005047619047619048, + "loss": 4.2894, + "theoretical_loss": 6.103688301426538, + "tokens_seen": 20840448 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005063492063492064, + "loss": 4.4263, + "theoretical_loss": 6.100466573992453, + "tokens_seen": 20905984 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005079365079365079, + "loss": 4.4126, + "theoretical_loss": 6.097257747999514, + "tokens_seen": 20971520 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005095238095238095, + "loss": 4.3339, + "theoretical_loss": 6.094061731708257, + "tokens_seen": 21037056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005111111111111111, + "loss": 4.2748, + "theoretical_loss": 6.090878434314753, + "tokens_seen": 21102592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005126984126984128, + "loss": 4.2673, + "theoretical_loss": 6.0877077659382, + "tokens_seen": 21168128 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005142857142857143, + "loss": 4.365, + "theoretical_loss": 6.084549637608715, + "tokens_seen": 21233664 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005158730158730159, + "loss": 4.3411, + "theoretical_loss": 6.081403961255324, + "tokens_seen": 21299200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005174603174603176, + "loss": 4.3378, + "theoretical_loss": 6.078270649694149, + "tokens_seen": 21364736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005190476190476191, + "loss": 4.2868, + "theoretical_loss": 6.07514961661678, + "tokens_seen": 21430272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005206349206349207, + "loss": 4.1847, + "theoretical_loss": 6.072040776578845, + "tokens_seen": 21495808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005222222222222222, + "loss": 4.3559, + "theoretical_loss": 6.068944044988752, + "tokens_seen": 21561344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005238095238095238, + "loss": 4.265, + "theoretical_loss": 6.065859338096619, + "tokens_seen": 21626880 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005253968253968255, + "loss": 4.1809, + "theoretical_loss": 6.0627865729833745, + "tokens_seen": 21692416 + }, + { + "epoch": 0.01, + "learning_rate": 0.000526984126984127, + "loss": 4.3868, + "theoretical_loss": 6.059725667550044, + "tokens_seen": 21757952 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005285714285714286, + "loss": 4.1844, + "theoretical_loss": 6.0566765405071905, + "tokens_seen": 21823488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005301587301587302, + "loss": 4.2241, + "theoretical_loss": 6.0536391113645305, + "tokens_seen": 21889024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005317460317460317, + "loss": 4.3067, + "theoretical_loss": 6.050613300420713, + "tokens_seen": 21954560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005333333333333334, + "loss": 4.1769, + "theoretical_loss": 6.047599028753265, + "tokens_seen": 22020096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005349206349206349, + "loss": 4.3459, + "theoretical_loss": 6.04459621820868, + "tokens_seen": 22085632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005365079365079365, + "loss": 4.2616, + "theoretical_loss": 6.041604791392675, + "tokens_seen": 22151168 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005380952380952381, + "loss": 4.2068, + "theoretical_loss": 6.038624671660598, + "tokens_seen": 22216704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005396825396825397, + "loss": 4.0923, + "theoretical_loss": 6.035655783107973, + "tokens_seen": 22282240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005412698412698414, + "loss": 4.1461, + "theoretical_loss": 6.0326980505612084, + "tokens_seen": 22347776 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005428571428571429, + "loss": 4.1102, + "theoretical_loss": 6.029751399568434, + "tokens_seen": 22413312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005444444444444445, + "loss": 4.1785, + "theoretical_loss": 6.026815756390491, + "tokens_seen": 22478848 + }, + { + "epoch": 0.01, + "learning_rate": 0.000546031746031746, + "loss": 4.2742, + "theoretical_loss": 6.023891047992053, + "tokens_seen": 22544384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005476190476190477, + "loss": 4.2707, + "theoretical_loss": 6.020977202032884, + "tokens_seen": 22609920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005492063492063493, + "loss": 4.1034, + "theoretical_loss": 6.018074146859234, + "tokens_seen": 22675456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005507936507936508, + "loss": 4.2671, + "theoretical_loss": 6.015181811495358, + "tokens_seen": 22740992 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005523809523809524, + "loss": 4.1057, + "theoretical_loss": 6.012300125635177, + "tokens_seen": 22806528 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005539682539682539, + "loss": 4.1616, + "theoretical_loss": 6.009429019634049, + "tokens_seen": 22872064 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.993457019329071, + "objective/train/avg_token_score": 0.006543011404573917, + "objective/train/docs_used": 15907, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 4.3004984855651855, + "objective/train/original_loss": 4.329875469207764, + "objective/train/theoretical_loss": 6.006568424500674, + "objective/train/tokens_used": 43397600, + "objective/train/unlikelihood_loss": 0.0021972618997097015, + "theoretical_loss": 6.006568424500674, + "tokens_seen": 22937600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005555555555555556, + "loss": 4.2437, + "theoretical_loss": 6.006568424500674, + "tokens_seen": 22937600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005571428571428572, + "loss": 3.934, + "theoretical_loss": 6.003718271889123, + "tokens_seen": 23003136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005587301587301587, + "loss": 4.1031, + "theoretical_loss": 6.000878494090979, + "tokens_seen": 23068672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005603174603174603, + "loss": 4.2621, + "theoretical_loss": 5.998049024027599, + "tokens_seen": 23134208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005619047619047619, + "loss": 4.2651, + "theoretical_loss": 5.995229795242498, + "tokens_seen": 23199744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005634920634920636, + "loss": 4.1415, + "theoretical_loss": 5.99242074189383, + "tokens_seen": 23265280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005650793650793652, + "loss": 4.0397, + "theoretical_loss": 5.989621798747002, + "tokens_seen": 23330816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005666666666666667, + "loss": 4.0572, + "theoretical_loss": 5.98683290116738, + "tokens_seen": 23396352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005682539682539683, + "loss": 4.0316, + "theoretical_loss": 5.9840539851131105, + "tokens_seen": 23461888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005698412698412698, + "loss": 4.1022, + "theoretical_loss": 5.981284987128044, + "tokens_seen": 23527424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005714285714285715, + "loss": 4.0396, + "theoretical_loss": 5.97852584433477, + "tokens_seen": 23592960 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005730158730158731, + "loss": 4.0791, + "theoretical_loss": 5.975776494427746, + "tokens_seen": 23658496 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005746031746031746, + "loss": 4.1164, + "theoretical_loss": 5.973036875666525, + "tokens_seen": 23724032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005761904761904762, + "loss": 4.1492, + "theoretical_loss": 5.970306926869093, + "tokens_seen": 23789568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005777777777777778, + "loss": 4.262, + "theoretical_loss": 5.967586587405295, + "tokens_seen": 23855104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005793650793650794, + "loss": 4.0716, + "theoretical_loss": 5.96487579719035, + "tokens_seen": 23920640 + }, + { + "epoch": 0.01, + "learning_rate": 0.000580952380952381, + "loss": 4.0449, + "theoretical_loss": 5.962174496678481, + "tokens_seen": 23986176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005825396825396825, + "loss": 3.9521, + "theoretical_loss": 5.959482626856606, + "tokens_seen": 24051712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005841269841269841, + "loss": 4.185, + "theoretical_loss": 5.956800129238152, + "tokens_seen": 24117248 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005857142857142857, + "loss": 3.9808, + "theoretical_loss": 5.954126945856933, + "tokens_seen": 24182784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005873015873015873, + "loss": 4.0566, + "theoretical_loss": 5.951463019261125, + "tokens_seen": 24248320 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005888888888888889, + "loss": 3.9817, + "theoretical_loss": 5.948808292507333, + "tokens_seen": 24313856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005904761904761905, + "loss": 3.9998, + "theoretical_loss": 5.946162709154724, + "tokens_seen": 24379392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005920634920634922, + "loss": 4.102, + "theoretical_loss": 5.943526213259268, + "tokens_seen": 24444928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005936507936507937, + "loss": 4.0435, + "theoretical_loss": 5.940898749368037, + "tokens_seen": 24510464 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005952380952380953, + "loss": 4.1925, + "theoretical_loss": 5.9382802625136, + "tokens_seen": 24576000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005968253968253969, + "loss": 4.0455, + "theoretical_loss": 5.935670698208488, + "tokens_seen": 24641536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005984126984126984, + "loss": 4.0122, + "theoretical_loss": 5.933070002439743, + "tokens_seen": 24707072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006000000000000001, + "loss": 4.0265, + "theoretical_loss": 5.930478121663538, + "tokens_seen": 24772608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006015873015873016, + "loss": 4.0538, + "theoretical_loss": 5.927895002799874, + "tokens_seen": 24838144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006031746031746032, + "loss": 4.1304, + "theoretical_loss": 5.925320593227351, + "tokens_seen": 24903680 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006047619047619048, + "loss": 3.9777, + "theoretical_loss": 5.922754840778015, + "tokens_seen": 24969216 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006063492063492063, + "loss": 3.961, + "theoretical_loss": 5.9201976937322645, + "tokens_seen": 25034752 + }, + { + "epoch": 0.01, + "learning_rate": 0.000607936507936508, + "loss": 4.1261, + "theoretical_loss": 5.917649100813846, + "tokens_seen": 25100288 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006095238095238095, + "loss": 3.9787, + "theoretical_loss": 5.915109011184899, + "tokens_seen": 25165824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006111111111111111, + "loss": 4.0351, + "theoretical_loss": 5.9125773744410814, + "tokens_seen": 25231360 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006126984126984127, + "loss": 4.0585, + "theoretical_loss": 5.91005414060676, + "tokens_seen": 25296896 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006142857142857143, + "loss": 4.1275, + "theoretical_loss": 5.90753926013026, + "tokens_seen": 25362432 + }, + { + "epoch": 0.01, + "learning_rate": 0.000615873015873016, + "loss": 4.0656, + "theoretical_loss": 5.90503268387919, + "tokens_seen": 25427968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006174603174603175, + "loss": 3.9604, + "theoretical_loss": 5.9025343631358185, + "tokens_seen": 25493504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006190476190476191, + "loss": 4.1794, + "theoretical_loss": 5.900044249592523, + "tokens_seen": 25559040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006206349206349206, + "loss": 4.1216, + "theoretical_loss": 5.897562295347297, + "tokens_seen": 25624576 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006222222222222223, + "loss": 3.9792, + "theoretical_loss": 5.895088452899312, + "tokens_seen": 25690112 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006238095238095239, + "loss": 4.1082, + "theoretical_loss": 5.892622675144553, + "tokens_seen": 25755648 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006253968253968254, + "loss": 4.0045, + "theoretical_loss": 5.890164915371495, + "tokens_seen": 25821184 + }, + { + "epoch": 0.01, + "learning_rate": 0.000626984126984127, + "loss": 4.0449, + "theoretical_loss": 5.887715127256854, + "tokens_seen": 25886720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006285714285714286, + "loss": 4.073, + "theoretical_loss": 5.885273264861381, + "tokens_seen": 25952256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006301587301587302, + "loss": 4.1225, + "theoretical_loss": 5.882839282625721, + "tokens_seen": 26017792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006317460317460318, + "loss": 3.9433, + "theoretical_loss": 5.880413135366323, + "tokens_seen": 26083328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006333333333333333, + "loss": 3.853, + "theoretical_loss": 5.877994778271411, + "tokens_seen": 26148864 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.9926624894142151, + "objective/train/avg_token_score": 0.007337525486946106, + "objective/train/docs_used": 17033, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 4.0623908042907715, + "objective/train/original_loss": 4.093017578125, + "objective/train/theoretical_loss": 5.8755841668969895, + "objective/train/tokens_used": 46674400, + "objective/train/unlikelihood_loss": 0.0027733040042221546, + "theoretical_loss": 5.8755841668969895, + "tokens_seen": 26214400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006349206349206349, + "loss": 4.1268, + "theoretical_loss": 5.8755841668969895, + "tokens_seen": 26214400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006365079365079365, + "loss": 3.8325, + "theoretical_loss": 5.873181257162926, + "tokens_seen": 26279936 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006380952380952382, + "loss": 3.9132, + "theoretical_loss": 5.870786005349071, + "tokens_seen": 26345472 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006396825396825398, + "loss": 3.9635, + "theoretical_loss": 5.868398368091427, + "tokens_seen": 26411008 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006412698412698413, + "loss": 4.0241, + "theoretical_loss": 5.866018302378377, + "tokens_seen": 26476544 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006428571428571429, + "loss": 3.8804, + "theoretical_loss": 5.863645765546956, + "tokens_seen": 26542080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006444444444444444, + "loss": 3.9605, + "theoretical_loss": 5.861280715279168, + "tokens_seen": 26607616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006460317460317461, + "loss": 3.8847, + "theoretical_loss": 5.858923109598366, + "tokens_seen": 26673152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006476190476190477, + "loss": 3.8382, + "theoretical_loss": 5.8565729068656545, + "tokens_seen": 26738688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006492063492063492, + "loss": 3.9661, + "theoretical_loss": 5.85423006577636, + "tokens_seen": 26804224 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006507936507936508, + "loss": 4.0409, + "theoretical_loss": 5.851894545356539, + "tokens_seen": 26869760 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006523809523809524, + "loss": 3.8772, + "theoretical_loss": 5.849566304959527, + "tokens_seen": 26935296 + }, + { + "epoch": 0.01, + "learning_rate": 0.000653968253968254, + "loss": 3.6876, + "theoretical_loss": 5.84724530426254, + "tokens_seen": 27000832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006555555555555556, + "loss": 3.8421, + "theoretical_loss": 5.844931503263309, + "tokens_seen": 27066368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006571428571428571, + "loss": 3.9492, + "theoretical_loss": 5.842624862276773, + "tokens_seen": 27131904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006587301587301587, + "loss": 3.818, + "theoretical_loss": 5.840325341931795, + "tokens_seen": 27197440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006603174603174604, + "loss": 3.6742, + "theoretical_loss": 5.838032903167934, + "tokens_seen": 27262976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006619047619047619, + "loss": 4.0104, + "theoretical_loss": 5.835747507232257, + "tokens_seen": 27328512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006634920634920636, + "loss": 3.9244, + "theoretical_loss": 5.833469115676177, + "tokens_seen": 27394048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006650793650793651, + "loss": 3.9438, + "theoretical_loss": 5.8311976903523535, + "tokens_seen": 27459584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006666666666666668, + "loss": 3.9489, + "theoretical_loss": 5.828933193411611, + "tokens_seen": 27525120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006682539682539683, + "loss": 3.8411, + "theoretical_loss": 5.826675587299913, + "tokens_seen": 27590656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006698412698412699, + "loss": 3.9391, + "theoretical_loss": 5.824424834755357, + "tokens_seen": 27656192 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006714285714285715, + "loss": 3.8234, + "theoretical_loss": 5.8221808988052235, + "tokens_seen": 27721728 + }, + { + "epoch": 0.01, + "learning_rate": 0.000673015873015873, + "loss": 3.8632, + "theoretical_loss": 5.819943742763051, + "tokens_seen": 27787264 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006746031746031747, + "loss": 3.8603, + "theoretical_loss": 5.817713330225754, + "tokens_seen": 27852800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006761904761904762, + "loss": 3.8404, + "theoretical_loss": 5.815489625070766, + "tokens_seen": 27918336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006777777777777778, + "loss": 3.7855, + "theoretical_loss": 5.813272591453236, + "tokens_seen": 27983872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006793650793650794, + "loss": 3.7392, + "theoretical_loss": 5.811062193803242, + "tokens_seen": 28049408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006809523809523809, + "loss": 3.8443, + "theoretical_loss": 5.808858396823043, + "tokens_seen": 28114944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006825396825396826, + "loss": 3.7486, + "theoretical_loss": 5.806661165484377, + "tokens_seen": 28180480 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006841269841269841, + "loss": 3.9541, + "theoretical_loss": 5.804470465025777, + "tokens_seen": 28246016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006857142857142857, + "loss": 3.8956, + "theoretical_loss": 5.802286260949924, + "tokens_seen": 28311552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006873015873015873, + "loss": 3.8683, + "theoretical_loss": 5.8001085190210375, + "tokens_seen": 28377088 + }, + { + "epoch": 0.01, + "learning_rate": 0.000688888888888889, + "loss": 3.945, + "theoretical_loss": 5.797937205262292, + "tokens_seen": 28442624 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006904761904761906, + "loss": 3.7177, + "theoretical_loss": 5.795772285953266, + "tokens_seen": 28508160 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006920634920634921, + "loss": 3.7928, + "theoretical_loss": 5.793613727627429, + "tokens_seen": 28573696 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006936507936507937, + "loss": 3.605, + "theoretical_loss": 5.791461497069644, + "tokens_seen": 28639232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006952380952380953, + "loss": 3.8603, + "theoretical_loss": 5.789315561313717, + "tokens_seen": 28704768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006968253968253969, + "loss": 3.8081, + "theoretical_loss": 5.787175887639959, + "tokens_seen": 28770304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006984126984126985, + "loss": 3.868, + "theoretical_loss": 5.785042443572798, + "tokens_seen": 28835840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007, + "loss": 3.7387, + "theoretical_loss": 5.782915196878396, + "tokens_seen": 28901376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007015873015873016, + "loss": 3.7876, + "theoretical_loss": 5.780794115562314, + "tokens_seen": 28966912 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007031746031746032, + "loss": 3.9029, + "theoretical_loss": 5.778679167867196, + "tokens_seen": 29032448 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007047619047619048, + "loss": 3.7661, + "theoretical_loss": 5.7765703222704765, + "tokens_seen": 29097984 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007063492063492064, + "loss": 3.846, + "theoretical_loss": 5.7744675474821285, + "tokens_seen": 29163520 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007079365079365079, + "loss": 3.8861, + "theoretical_loss": 5.772370812442426, + "tokens_seen": 29229056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007095238095238095, + "loss": 3.7575, + "theoretical_loss": 5.770280086319733, + "tokens_seen": 29294592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007111111111111111, + "loss": 3.8464, + "theoretical_loss": 5.768195338508333, + "tokens_seen": 29360128 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007126984126984128, + "loss": 3.7137, + "theoretical_loss": 5.766116538626263, + "tokens_seen": 29425664 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.9907845854759216, + "objective/train/avg_token_score": 0.00921542476862669, + "objective/train/docs_used": 18174, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 3.9509756565093994, + "objective/train/original_loss": 3.989859104156494, + "objective/train/theoretical_loss": 5.7640436565131905, + "objective/train/tokens_used": 49951200, + "objective/train/unlikelihood_loss": 0.004624219611287117, + "theoretical_loss": 5.7640436565131905, + "tokens_seen": 29491200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007142857142857144, + "loss": 3.8573, + "theoretical_loss": 5.7640436565131905, + "tokens_seen": 29491200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007158730158730159, + "loss": 3.8101, + "theoretical_loss": 5.761976662228307, + "tokens_seen": 29556736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007174603174603175, + "loss": 3.7128, + "theoretical_loss": 5.759915526048243, + "tokens_seen": 29622272 + }, + { + "epoch": 0.01, + "learning_rate": 0.000719047619047619, + "loss": 3.7951, + "theoretical_loss": 5.757860218465023, + "tokens_seen": 29687808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007206349206349207, + "loss": 3.8424, + "theoretical_loss": 5.755810710184019, + "tokens_seen": 29753344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007222222222222223, + "loss": 3.7002, + "theoretical_loss": 5.753766972121954, + "tokens_seen": 29818880 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007238095238095238, + "loss": 3.8656, + "theoretical_loss": 5.751728975404911, + "tokens_seen": 29884416 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007253968253968254, + "loss": 3.8879, + "theoretical_loss": 5.7496966913663705, + "tokens_seen": 29949952 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007269841269841271, + "loss": 3.7188, + "theoretical_loss": 5.747670091545274, + "tokens_seen": 30015488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007285714285714286, + "loss": 3.7982, + "theoretical_loss": 5.745649147684103, + "tokens_seen": 30081024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007301587301587302, + "loss": 3.8925, + "theoretical_loss": 5.743633831726987, + "tokens_seen": 30146560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007317460317460317, + "loss": 3.7778, + "theoretical_loss": 5.741624115817827, + "tokens_seen": 30212096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007333333333333333, + "loss": 3.7747, + "theoretical_loss": 5.739619972298446, + "tokens_seen": 30277632 + }, + { + "epoch": 0.01, + "learning_rate": 0.000734920634920635, + "loss": 3.8514, + "theoretical_loss": 5.737621373706753, + "tokens_seen": 30343168 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007365079365079365, + "loss": 3.7153, + "theoretical_loss": 5.735628292774945, + "tokens_seen": 30408704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007380952380952382, + "loss": 3.6853, + "theoretical_loss": 5.7336407024276985, + "tokens_seen": 30474240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007396825396825397, + "loss": 3.6937, + "theoretical_loss": 5.731658575780418, + "tokens_seen": 30539776 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007412698412698414, + "loss": 3.6721, + "theoretical_loss": 5.729681886137476, + "tokens_seen": 30605312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007428571428571429, + "loss": 3.598, + "theoretical_loss": 5.727710606990488, + "tokens_seen": 30670848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007444444444444445, + "loss": 3.7946, + "theoretical_loss": 5.725744712016601, + "tokens_seen": 30736384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007460317460317461, + "loss": 3.7388, + "theoretical_loss": 5.723784175076802, + "tokens_seen": 30801920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007476190476190476, + "loss": 3.697, + "theoretical_loss": 5.72182897021425, + "tokens_seen": 30867456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007492063492063493, + "loss": 3.5768, + "theoretical_loss": 5.719879071652615, + "tokens_seen": 30932992 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007507936507936508, + "loss": 3.702, + "theoretical_loss": 5.717934453794454, + "tokens_seen": 30998528 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007523809523809524, + "loss": 3.746, + "theoretical_loss": 5.715995091219584, + "tokens_seen": 31064064 + }, + { + "epoch": 0.01, + "learning_rate": 0.000753968253968254, + "loss": 3.6567, + "theoretical_loss": 5.7140609586834925, + "tokens_seen": 31129600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007555555555555555, + "loss": 3.6683, + "theoretical_loss": 5.712132031115749, + "tokens_seen": 31195136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007571428571428572, + "loss": 3.6242, + "theoretical_loss": 5.710208283618447, + "tokens_seen": 31260672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007587301587301588, + "loss": 3.8178, + "theoretical_loss": 5.708289691464657, + "tokens_seen": 31326208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007603174603174603, + "loss": 3.8619, + "theoretical_loss": 5.706376230096897, + "tokens_seen": 31391744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007619047619047619, + "loss": 3.6388, + "theoretical_loss": 5.704467875125619, + "tokens_seen": 31457280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007634920634920635, + "loss": 3.7142, + "theoretical_loss": 5.702564602327719, + "tokens_seen": 31522816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007650793650793652, + "loss": 3.5803, + "theoretical_loss": 5.700666387645049, + "tokens_seen": 31588352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007666666666666667, + "loss": 3.8599, + "theoretical_loss": 5.698773207182963, + "tokens_seen": 31653888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007682539682539683, + "loss": 3.7165, + "theoretical_loss": 5.696885037208865, + "tokens_seen": 31719424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007698412698412699, + "loss": 3.6805, + "theoretical_loss": 5.695001854150782, + "tokens_seen": 31784960 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007714285714285715, + "loss": 3.6022, + "theoretical_loss": 5.693123634595938, + "tokens_seen": 31850496 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007730158730158731, + "loss": 3.7355, + "theoretical_loss": 5.69125035528937, + "tokens_seen": 31916032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007746031746031746, + "loss": 3.6734, + "theoretical_loss": 5.689381993132532, + "tokens_seen": 31981568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007761904761904762, + "loss": 3.7047, + "theoretical_loss": 5.687518525181924, + "tokens_seen": 32047104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007777777777777778, + "loss": 3.6644, + "theoretical_loss": 5.685659928647743, + "tokens_seen": 32112640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007793650793650794, + "loss": 3.7867, + "theoretical_loss": 5.68380618089254, + "tokens_seen": 32178176 + }, + { + "epoch": 0.01, + "learning_rate": 0.000780952380952381, + "loss": 3.6108, + "theoretical_loss": 5.6819572594298915, + "tokens_seen": 32243712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007825396825396825, + "loss": 3.6043, + "theoretical_loss": 5.680113141923088, + "tokens_seen": 32309248 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007841269841269841, + "loss": 3.7696, + "theoretical_loss": 5.67827380618384, + "tokens_seen": 32374784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007857142857142857, + "loss": 3.6882, + "theoretical_loss": 5.6764392301709865, + "tokens_seen": 32440320 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007873015873015873, + "loss": 3.5926, + "theoretical_loss": 5.674609391989232, + "tokens_seen": 32505856 + }, + { + "epoch": 0.01, + "learning_rate": 0.000788888888888889, + "loss": 3.8022, + "theoretical_loss": 5.672784269887885, + "tokens_seen": 32571392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007904761904761905, + "loss": 3.8038, + "theoretical_loss": 5.670963842259615, + "tokens_seen": 32636928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007920634920634921, + "loss": 3.8028, + "theoretical_loss": 5.669148087639222, + "tokens_seen": 32702464 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.994178056716919, + "objective/train/avg_token_score": 0.00582193722948432, + "objective/train/docs_used": 19239, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 3.758636236190796, + "objective/train/original_loss": 3.7812983989715576, + "objective/train/theoretical_loss": 5.66733698470242, + "objective/train/tokens_used": 53228000, + "objective/train/unlikelihood_loss": 0.003600894473493099, + "theoretical_loss": 5.66733698470242, + "tokens_seen": 32768000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007936507936507938, + "loss": 3.7669, + "theoretical_loss": 5.66733698470242, + "tokens_seen": 32768000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007952380952380953, + "loss": 3.6846, + "theoretical_loss": 5.665530512264631, + "tokens_seen": 32833536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007968253968253969, + "loss": 3.6956, + "theoretical_loss": 5.663728649279793, + "tokens_seen": 32899072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007984126984126984, + "loss": 3.5335, + "theoretical_loss": 5.6619313748391855, + "tokens_seen": 32964608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0008, + "loss": 3.7521, + "theoretical_loss": 5.6601386681702515, + "tokens_seen": 33030144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007999839518555668, + "loss": 3.667, + "theoretical_loss": 5.658350508635454, + "tokens_seen": 33095680 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007999679037111334, + "loss": 3.5634, + "theoretical_loss": 5.656566875731127, + "tokens_seen": 33161216 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007999518555667002, + "loss": 3.579, + "theoretical_loss": 5.654787749086345, + "tokens_seen": 33226752 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007999358074222668, + "loss": 3.6296, + "theoretical_loss": 5.6530131084618045, + "tokens_seen": 33292288 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007999197592778336, + "loss": 3.6263, + "theoretical_loss": 5.651242933748718, + "tokens_seen": 33357824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007999037111334002, + "loss": 3.6093, + "theoretical_loss": 5.649477204967717, + "tokens_seen": 33423360 + }, + { + "epoch": 0.01, + "learning_rate": 0.000799887662988967, + "loss": 3.7369, + "theoretical_loss": 5.647715902267764, + "tokens_seen": 33488896 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007998716148445336, + "loss": 3.6277, + "theoretical_loss": 5.645959005925086, + "tokens_seen": 33554432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007998555667001004, + "loss": 3.6113, + "theoretical_loss": 5.644206496342103, + "tokens_seen": 33619968 + }, + { + "epoch": 0.01, + "learning_rate": 0.000799839518555667, + "loss": 3.648, + "theoretical_loss": 5.642458354046386, + "tokens_seen": 33685504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007998234704112338, + "loss": 3.5841, + "theoretical_loss": 5.6407145596896076, + "tokens_seen": 33751040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007998074222668004, + "loss": 3.6664, + "theoretical_loss": 5.638975094046518, + "tokens_seen": 33816576 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007997913741223672, + "loss": 3.4693, + "theoretical_loss": 5.637239938013924, + "tokens_seen": 33882112 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007997753259779338, + "loss": 3.5231, + "theoretical_loss": 5.6355090726096755, + "tokens_seen": 33947648 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007997592778335006, + "loss": 3.5183, + "theoretical_loss": 5.633782478971672, + "tokens_seen": 34013184 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007997432296890672, + "loss": 3.5625, + "theoretical_loss": 5.632060138356873, + "tokens_seen": 34078720 + }, + { + "epoch": 0.01, + "learning_rate": 0.000799727181544634, + "loss": 3.489, + "theoretical_loss": 5.630342032140315, + "tokens_seen": 34144256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007997111334002006, + "loss": 3.6622, + "theoretical_loss": 5.628628141814151, + "tokens_seen": 34209792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007996950852557674, + "loss": 3.5305, + "theoretical_loss": 5.62691844898668, + "tokens_seen": 34275328 + }, + { + "epoch": 0.01, + "learning_rate": 0.000799679037111334, + "loss": 3.4731, + "theoretical_loss": 5.625212935381407, + "tokens_seen": 34340864 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007996629889669008, + "loss": 3.4872, + "theoretical_loss": 5.6235115828361035, + "tokens_seen": 34406400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007996469408224675, + "loss": 3.375, + "theoretical_loss": 5.621814373301866, + "tokens_seen": 34471936 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007996308926780342, + "loss": 3.59, + "theoretical_loss": 5.620121288842209, + "tokens_seen": 34537472 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007996148445336009, + "loss": 3.5507, + "theoretical_loss": 5.618432311632143, + "tokens_seen": 34603008 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007995987963891676, + "loss": 3.5084, + "theoretical_loss": 5.616747423957278, + "tokens_seen": 34668544 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007995827482447343, + "loss": 3.5096, + "theoretical_loss": 5.615066608212926, + "tokens_seen": 34734080 + }, + { + "epoch": 0.01, + "learning_rate": 0.000799566700100301, + "loss": 3.5863, + "theoretical_loss": 5.613389846903216, + "tokens_seen": 34799616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007995506519558677, + "loss": 3.5401, + "theoretical_loss": 5.611717122640225, + "tokens_seen": 34865152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007995346038114344, + "loss": 3.4429, + "theoretical_loss": 5.610048418143098, + "tokens_seen": 34930688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007995185556670011, + "loss": 3.466, + "theoretical_loss": 5.6083837162372, + "tokens_seen": 34996224 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007995025075225678, + "loss": 3.5189, + "theoretical_loss": 5.606722999853261, + "tokens_seen": 35061760 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007994864593781345, + "loss": 3.4197, + "theoretical_loss": 5.605066252026537, + "tokens_seen": 35127296 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007994704112337012, + "loss": 3.5093, + "theoretical_loss": 5.603413455895973, + "tokens_seen": 35192832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007994543630892679, + "loss": 3.471, + "theoretical_loss": 5.601764594703377, + "tokens_seen": 35258368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007994383149448346, + "loss": 3.26, + "theoretical_loss": 5.600119651792607, + "tokens_seen": 35323904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007994222668004013, + "loss": 3.3692, + "theoretical_loss": 5.598478610608758, + "tokens_seen": 35389440 + }, + { + "epoch": 0.01, + "learning_rate": 0.000799406218655968, + "loss": 3.3839, + "theoretical_loss": 5.596841454697365, + "tokens_seen": 35454976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007993901705115347, + "loss": 3.5465, + "theoretical_loss": 5.595208167703601, + "tokens_seen": 35520512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007993741223671014, + "loss": 3.4238, + "theoretical_loss": 5.5935787333714995, + "tokens_seen": 35586048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007993580742226681, + "loss": 3.7297, + "theoretical_loss": 5.59195313554317, + "tokens_seen": 35651584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007993420260782347, + "loss": 3.5732, + "theoretical_loss": 5.590331358158036, + "tokens_seen": 35717120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007993259779338015, + "loss": 3.6411, + "theoretical_loss": 5.58871338525206, + "tokens_seen": 35782656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007993099297893681, + "loss": 3.446, + "theoretical_loss": 5.587099200956995, + "tokens_seen": 35848192 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007992938816449349, + "loss": 3.6217, + "theoretical_loss": 5.585488789499637, + "tokens_seen": 35913728 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007992778335005015, + "loss": 3.6044, + "theoretical_loss": 5.583882135201078, + "tokens_seen": 35979264 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.9903879761695862, + "objective/train/avg_token_score": 0.009612065739929676, + "objective/train/docs_used": 20470, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 3.416790723800659, + "objective/train/original_loss": 3.446444034576416, + "objective/train/theoretical_loss": 5.582279222475973, + "objective/train/tokens_used": 56504800, + "objective/train/unlikelihood_loss": 0.011937998235225677, + "theoretical_loss": 5.582279222475973, + "tokens_seen": 36044800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007992617853560683, + "loss": 3.3938, + "theoretical_loss": 5.582279222475973, + "tokens_seen": 36044800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007992457372116349, + "loss": 3.6886, + "theoretical_loss": 5.580680035831821, + "tokens_seen": 36110336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007992296890672017, + "loss": 3.5093, + "theoretical_loss": 5.579084559868232, + "tokens_seen": 36175872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007992136409227683, + "loss": 3.5962, + "theoretical_loss": 5.577492779276222, + "tokens_seen": 36241408 + }, + { + "epoch": 0.01, + "learning_rate": 0.000799197592778335, + "loss": 3.4189, + "theoretical_loss": 5.575904678837501, + "tokens_seen": 36306944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007991815446339017, + "loss": 3.5895, + "theoretical_loss": 5.574320243423783, + "tokens_seen": 36372480 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007991654964894684, + "loss": 3.3496, + "theoretical_loss": 5.572739457996079, + "tokens_seen": 36438016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007991494483450351, + "loss": 3.3646, + "theoretical_loss": 5.5711623076040215, + "tokens_seen": 36503552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007991334002006018, + "loss": 3.612, + "theoretical_loss": 5.569588777385177, + "tokens_seen": 36569088 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007991173520561685, + "loss": 3.4909, + "theoretical_loss": 5.568018852564379, + "tokens_seen": 36634624 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007991013039117352, + "loss": 3.4629, + "theoretical_loss": 5.566452518453051, + "tokens_seen": 36700160 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007990852557673019, + "loss": 3.318, + "theoretical_loss": 5.564889760448561, + "tokens_seen": 36765696 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007990692076228686, + "loss": 3.4753, + "theoretical_loss": 5.5633305640335475, + "tokens_seen": 36831232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007990531594784353, + "loss": 3.4847, + "theoretical_loss": 5.561774914775286, + "tokens_seen": 36896768 + }, + { + "epoch": 0.01, + "learning_rate": 0.000799037111334002, + "loss": 3.4894, + "theoretical_loss": 5.5602227983250385, + "tokens_seen": 36962304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007990210631895687, + "loss": 3.4255, + "theoretical_loss": 5.558674200417419, + "tokens_seen": 37027840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007990050150451354, + "loss": 3.5749, + "theoretical_loss": 5.557129106869764, + "tokens_seen": 37093376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007989889669007021, + "loss": 3.4571, + "theoretical_loss": 5.555587503581506, + "tokens_seen": 37158912 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007989729187562688, + "loss": 3.5224, + "theoretical_loss": 5.554049376533554, + "tokens_seen": 37224448 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007989568706118355, + "loss": 3.4853, + "theoretical_loss": 5.552514711787683, + "tokens_seen": 37289984 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007989408224674022, + "loss": 3.3871, + "theoretical_loss": 5.550983495485925, + "tokens_seen": 37355520 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007989247743229689, + "loss": 3.5957, + "theoretical_loss": 5.549455713849963, + "tokens_seen": 37421056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007989087261785356, + "loss": 3.5807, + "theoretical_loss": 5.547931353180543, + "tokens_seen": 37486592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007988926780341023, + "loss": 3.4249, + "theoretical_loss": 5.546410399856875, + "tokens_seen": 37552128 + }, + { + "epoch": 0.01, + "learning_rate": 0.000798876629889669, + "loss": 3.4061, + "theoretical_loss": 5.544892840336053, + "tokens_seen": 37617664 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007988605817452357, + "loss": 3.4379, + "theoretical_loss": 5.543378661152472, + "tokens_seen": 37683200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007988445336008024, + "loss": 3.4015, + "theoretical_loss": 5.5418678489172555, + "tokens_seen": 37748736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007988284854563691, + "loss": 3.4975, + "theoretical_loss": 5.540360390317682, + "tokens_seen": 37814272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007988124373119358, + "loss": 3.3161, + "theoretical_loss": 5.538856272116626, + "tokens_seen": 37879808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007987963891675025, + "loss": 3.2856, + "theoretical_loss": 5.5373554811519945, + "tokens_seen": 37945344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007987803410230693, + "loss": 3.3544, + "theoretical_loss": 5.535858004336175, + "tokens_seen": 38010880 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007987642928786359, + "loss": 3.4335, + "theoretical_loss": 5.534363828655488, + "tokens_seen": 38076416 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007987482447342027, + "loss": 3.4032, + "theoretical_loss": 5.532872941169639, + "tokens_seen": 38141952 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007987321965897693, + "loss": 3.3933, + "theoretical_loss": 5.531385329011181, + "tokens_seen": 38207488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007987161484453361, + "loss": 3.4412, + "theoretical_loss": 5.529900979384985, + "tokens_seen": 38273024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007987001003009027, + "loss": 3.2914, + "theoretical_loss": 5.5284198795677, + "tokens_seen": 38338560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007986840521564695, + "loss": 3.2401, + "theoretical_loss": 5.526942016907242, + "tokens_seen": 38404096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007986680040120361, + "loss": 3.4269, + "theoretical_loss": 5.5254673788222615, + "tokens_seen": 38469632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007986519558676029, + "loss": 3.3831, + "theoretical_loss": 5.523995952801634, + "tokens_seen": 38535168 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007986359077231695, + "loss": 3.2247, + "theoretical_loss": 5.522527726403953, + "tokens_seen": 38600704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007986198595787363, + "loss": 3.5093, + "theoretical_loss": 5.521062687257016, + "tokens_seen": 38666240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007986038114343029, + "loss": 3.4017, + "theoretical_loss": 5.519600823057332, + "tokens_seen": 38731776 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007985877632898697, + "loss": 3.3349, + "theoretical_loss": 5.518142121569613, + "tokens_seen": 38797312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007985717151454363, + "loss": 3.2946, + "theoretical_loss": 5.516686570626293, + "tokens_seen": 38862848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007985556670010031, + "loss": 3.4696, + "theoretical_loss": 5.515234158127032, + "tokens_seen": 38928384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007985396188565697, + "loss": 3.4758, + "theoretical_loss": 5.5137848720382365, + "tokens_seen": 38993920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007985235707121365, + "loss": 3.3384, + "theoretical_loss": 5.5123387003925775, + "tokens_seen": 39059456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007985075225677031, + "loss": 3.3566, + "theoretical_loss": 5.510895631288515, + "tokens_seen": 39124992 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007984914744232699, + "loss": 3.3581, + "theoretical_loss": 5.50945565288983, + "tokens_seen": 39190528 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007984754262788365, + "loss": 3.3198, + "theoretical_loss": 5.508018753425152, + "tokens_seen": 39256064 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.9876997470855713, + "objective/train/avg_token_score": 0.012300250120460987, + "objective/train/docs_used": 21760, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 3.058933973312378, + "objective/train/original_loss": 3.0919055938720703, + "objective/train/theoretical_loss": 5.506584921187503, + "objective/train/tokens_used": 59781600, + "objective/train/unlikelihood_loss": 0.014180874451994896, + "theoretical_loss": 5.506584921187503, + "tokens_seen": 39321600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007984593781344033, + "loss": 3.2015, + "theoretical_loss": 5.506584921187503, + "tokens_seen": 39321600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007984433299899699, + "loss": 3.359, + "theoretical_loss": 5.505154144533835, + "tokens_seen": 39387136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007984272818455367, + "loss": 3.4326, + "theoretical_loss": 5.503726411884568, + "tokens_seen": 39452672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007984112337011033, + "loss": 3.4091, + "theoretical_loss": 5.502301711723156, + "tokens_seen": 39518208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007983951855566701, + "loss": 3.2631, + "theoretical_loss": 5.5008800325956235, + "tokens_seen": 39583744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007983791374122367, + "loss": 3.4544, + "theoretical_loss": 5.499461363110136, + "tokens_seen": 39649280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007983630892678035, + "loss": 3.2205, + "theoretical_loss": 5.498045691936548, + "tokens_seen": 39714816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007983470411233701, + "loss": 3.4056, + "theoretical_loss": 5.496633007805977, + "tokens_seen": 39780352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007983309929789369, + "loss": 3.3053, + "theoretical_loss": 5.495223299510372, + "tokens_seen": 39845888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007983149448345036, + "loss": 3.3165, + "theoretical_loss": 5.493816555902079, + "tokens_seen": 39911424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007982988966900703, + "loss": 3.4407, + "theoretical_loss": 5.492412765893421, + "tokens_seen": 39976960 + }, + { + "epoch": 0.01, + "learning_rate": 0.000798282848545637, + "loss": 3.3954, + "theoretical_loss": 5.49101191845628, + "tokens_seen": 40042496 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007982668004012037, + "loss": 3.2427, + "theoretical_loss": 5.489614002621677, + "tokens_seen": 40108032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007982507522567704, + "loss": 3.3051, + "theoretical_loss": 5.4882190074793575, + "tokens_seen": 40173568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007982347041123371, + "loss": 3.3654, + "theoretical_loss": 5.486826922177387, + "tokens_seen": 40239104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007982186559679038, + "loss": 3.2869, + "theoretical_loss": 5.4854377359217406, + "tokens_seen": 40304640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007982026078234705, + "loss": 3.3349, + "theoretical_loss": 5.484051437975901, + "tokens_seen": 40370176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007981865596790372, + "loss": 3.2604, + "theoretical_loss": 5.482668017660462, + "tokens_seen": 40435712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007981705115346039, + "loss": 3.4681, + "theoretical_loss": 5.481287464352727, + "tokens_seen": 40501248 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007981544633901706, + "loss": 3.3535, + "theoretical_loss": 5.479909767486326, + "tokens_seen": 40566784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007981384152457373, + "loss": 3.3289, + "theoretical_loss": 5.478534916550817, + "tokens_seen": 40632320 + }, + { + "epoch": 0.01, + "learning_rate": 0.000798122367101304, + "loss": 3.5542, + "theoretical_loss": 5.477162901091306, + "tokens_seen": 40697856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007981063189568707, + "loss": 3.283, + "theoretical_loss": 5.475793710708061, + "tokens_seen": 40763392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007980902708124374, + "loss": 3.421, + "theoretical_loss": 5.474427335056138, + "tokens_seen": 40828928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007980742226680041, + "loss": 3.3056, + "theoretical_loss": 5.473063763844998, + "tokens_seen": 40894464 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007980581745235708, + "loss": 3.4329, + "theoretical_loss": 5.471702986838139, + "tokens_seen": 40960000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007980421263791375, + "loss": 3.4192, + "theoretical_loss": 5.470344993852729, + "tokens_seen": 41025536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007980260782347042, + "loss": 3.2943, + "theoretical_loss": 5.46898977475923, + "tokens_seen": 41091072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007980100300902709, + "loss": 3.2752, + "theoretical_loss": 5.467637319481042, + "tokens_seen": 41156608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007979939819458376, + "loss": 3.2841, + "theoretical_loss": 5.466287617994144, + "tokens_seen": 41222144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007979779338014043, + "loss": 3.4712, + "theoretical_loss": 5.464940660326731, + "tokens_seen": 41287680 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797961885656971, + "loss": 3.2829, + "theoretical_loss": 5.463596436558861, + "tokens_seen": 41353216 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007979458375125376, + "loss": 3.2259, + "theoretical_loss": 5.462254936822105, + "tokens_seen": 41418752 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007979297893681044, + "loss": 3.1703, + "theoretical_loss": 5.4609161512992, + "tokens_seen": 41484288 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797913741223671, + "loss": 3.2851, + "theoretical_loss": 5.4595800702237005, + "tokens_seen": 41549824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007978976930792378, + "loss": 3.2441, + "theoretical_loss": 5.458246683879636, + "tokens_seen": 41615360 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007978816449348044, + "loss": 3.2848, + "theoretical_loss": 5.456915982601173, + "tokens_seen": 41680896 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007978655967903712, + "loss": 3.2685, + "theoretical_loss": 5.455587956772277, + "tokens_seen": 41746432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007978495486459378, + "loss": 3.3314, + "theoretical_loss": 5.454262596826377, + "tokens_seen": 41811968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007978335005015046, + "loss": 3.3856, + "theoretical_loss": 5.452939893246034, + "tokens_seen": 41877504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007978174523570712, + "loss": 3.1893, + "theoretical_loss": 5.451619836562617, + "tokens_seen": 41943040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007978014042126379, + "loss": 3.1232, + "theoretical_loss": 5.450302417355973, + "tokens_seen": 42008576 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007977853560682046, + "loss": 3.2074, + "theoretical_loss": 5.448987626254097, + "tokens_seen": 42074112 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007977693079237713, + "loss": 3.3872, + "theoretical_loss": 5.447675453932829, + "tokens_seen": 42139648 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797753259779338, + "loss": 3.2395, + "theoretical_loss": 5.446365891115516, + "tokens_seen": 42205184 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007977372116349047, + "loss": 3.2714, + "theoretical_loss": 5.445058928572712, + "tokens_seen": 42270720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007977211634904714, + "loss": 3.0432, + "theoretical_loss": 5.443754557121853, + "tokens_seen": 42336256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007977051153460381, + "loss": 3.2969, + "theoretical_loss": 5.442452767626957, + "tokens_seen": 42401792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007976890672016048, + "loss": 3.1923, + "theoretical_loss": 5.441153550998311, + "tokens_seen": 42467328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007976730190571715, + "loss": 3.4146, + "theoretical_loss": 5.439856898192163, + "tokens_seen": 42532864 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.9928086996078491, + "objective/train/avg_token_score": 0.007191329728811979, + "objective/train/docs_used": 22938, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 3.3044016361236572, + "objective/train/original_loss": 3.3297057151794434, + "objective/train/theoretical_loss": 5.438562800210427, + "objective/train/tokens_used": 63058400, + "objective/train/unlikelihood_loss": 0.005170746706426144, + "theoretical_loss": 5.438562800210427, + "tokens_seen": 42598400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007976569709127382, + "loss": 3.3533, + "theoretical_loss": 5.438562800210427, + "tokens_seen": 42598400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007976409227683049, + "loss": 3.2081, + "theoretical_loss": 5.437271248100375, + "tokens_seen": 42663936 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007976248746238716, + "loss": 3.143, + "theoretical_loss": 5.435982232954342, + "tokens_seen": 42729472 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007976088264794383, + "loss": 3.2206, + "theoretical_loss": 5.434695745909433, + "tokens_seen": 42795008 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797592778335005, + "loss": 3.232, + "theoretical_loss": 5.433411778147226, + "tokens_seen": 42860544 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007975767301905718, + "loss": 3.1156, + "theoretical_loss": 5.432130320893483, + "tokens_seen": 42926080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007975606820461384, + "loss": 3.2418, + "theoretical_loss": 5.430851365417862, + "tokens_seen": 42991616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007975446339017052, + "loss": 3.1327, + "theoretical_loss": 5.429574903033627, + "tokens_seen": 43057152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007975285857572718, + "loss": 3.2758, + "theoretical_loss": 5.428300925097373, + "tokens_seen": 43122688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007975125376128386, + "loss": 3.1657, + "theoretical_loss": 5.427029423008732, + "tokens_seen": 43188224 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007974964894684052, + "loss": 3.131, + "theoretical_loss": 5.425760388210108, + "tokens_seen": 43253760 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797480441323972, + "loss": 3.2374, + "theoretical_loss": 5.424493812186386, + "tokens_seen": 43319296 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007974643931795386, + "loss": 3.1276, + "theoretical_loss": 5.423229686464667, + "tokens_seen": 43384832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007974483450351054, + "loss": 3.1774, + "theoretical_loss": 5.421968002613992, + "tokens_seen": 43450368 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797432296890672, + "loss": 3.4098, + "theoretical_loss": 5.420708752245071, + "tokens_seen": 43515904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007974162487462388, + "loss": 3.2694, + "theoretical_loss": 5.419451927010014, + "tokens_seen": 43581440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007974002006018054, + "loss": 3.2897, + "theoretical_loss": 5.418197518602068, + "tokens_seen": 43646976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007973841524573722, + "loss": 3.2437, + "theoretical_loss": 5.41694551875535, + "tokens_seen": 43712512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007973681043129388, + "loss": 3.27, + "theoretical_loss": 5.415695919244586, + "tokens_seen": 43778048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007973520561685056, + "loss": 3.1996, + "theoretical_loss": 5.41444871188485, + "tokens_seen": 43843584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007973360080240722, + "loss": 3.0322, + "theoretical_loss": 5.413203888531308, + "tokens_seen": 43909120 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797319959879639, + "loss": 3.1124, + "theoretical_loss": 5.411961441078963, + "tokens_seen": 43974656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007973039117352056, + "loss": 3.1975, + "theoretical_loss": 5.410721361462395, + "tokens_seen": 44040192 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007972878635907724, + "loss": 3.3937, + "theoretical_loss": 5.409483641655516, + "tokens_seen": 44105728 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797271815446339, + "loss": 3.1535, + "theoretical_loss": 5.408248273671317, + "tokens_seen": 44171264 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007972557673019058, + "loss": 3.0217, + "theoretical_loss": 5.407015249561624, + "tokens_seen": 44236800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007972397191574724, + "loss": 3.2422, + "theoretical_loss": 5.405784561416844, + "tokens_seen": 44302336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007972236710130392, + "loss": 3.0995, + "theoretical_loss": 5.404556201365727, + "tokens_seen": 44367872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007972076228686058, + "loss": 3.1661, + "theoretical_loss": 5.403330161575123, + "tokens_seen": 44433408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007971915747241726, + "loss": 3.1439, + "theoretical_loss": 5.402106434249742, + "tokens_seen": 44498944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007971755265797392, + "loss": 3.1394, + "theoretical_loss": 5.400885011631912, + "tokens_seen": 44564480 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797159478435306, + "loss": 3.2345, + "theoretical_loss": 5.399665886001349, + "tokens_seen": 44630016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007971434302908726, + "loss": 3.1363, + "theoretical_loss": 5.398449049674916, + "tokens_seen": 44695552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007971273821464394, + "loss": 3.188, + "theoretical_loss": 5.397234495006392, + "tokens_seen": 44761088 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797111334002006, + "loss": 3.202, + "theoretical_loss": 5.396022214386246, + "tokens_seen": 44826624 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007970952858575728, + "loss": 2.9884, + "theoretical_loss": 5.394812200241393, + "tokens_seen": 44892160 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007970792377131394, + "loss": 3.0576, + "theoretical_loss": 5.393604445034988, + "tokens_seen": 44957696 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007970631895687062, + "loss": 3.1721, + "theoretical_loss": 5.392398941266183, + "tokens_seen": 45023232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007970471414242728, + "loss": 3.004, + "theoretical_loss": 5.391195681469908, + "tokens_seen": 45088768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007970310932798396, + "loss": 3.19, + "theoretical_loss": 5.389994658216651, + "tokens_seen": 45154304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007970150451354062, + "loss": 3.1209, + "theoretical_loss": 5.388795864112235, + "tokens_seen": 45219840 + }, + { + "epoch": 0.01, + "learning_rate": 0.000796998996990973, + "loss": 3.086, + "theoretical_loss": 5.387599291797599, + "tokens_seen": 45285376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007969829488465396, + "loss": 3.1158, + "theoretical_loss": 5.386404933948584, + "tokens_seen": 45350912 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007969669007021064, + "loss": 3.0565, + "theoretical_loss": 5.38521278327571, + "tokens_seen": 45416448 + }, + { + "epoch": 0.01, + "learning_rate": 0.000796950852557673, + "loss": 2.9749, + "theoretical_loss": 5.384022832523971, + "tokens_seen": 45481984 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007969348044132398, + "loss": 3.0934, + "theoretical_loss": 5.382835074472616, + "tokens_seen": 45547520 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007969187562688065, + "loss": 3.2471, + "theoretical_loss": 5.381649501934939, + "tokens_seen": 45613056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007969027081243732, + "loss": 3.192, + "theoretical_loss": 5.3804661077580755, + "tokens_seen": 45678592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007968866599799399, + "loss": 3.1112, + "theoretical_loss": 5.379284884822787, + "tokens_seen": 45744128 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007968706118355066, + "loss": 3.2505, + "theoretical_loss": 5.37810582604326, + "tokens_seen": 45809664 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.9905271530151367, + "objective/train/avg_token_score": 0.009472854435443878, + "objective/train/docs_used": 24173, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 2.992762565612793, + "objective/train/original_loss": 3.0218505859375, + "objective/train/theoretical_loss": 5.376928924366901, + "objective/train/tokens_used": 66335200, + "objective/train/unlikelihood_loss": 0.007555294316262007, + "theoretical_loss": 5.376928924366901, + "tokens_seen": 45875200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007968545636910733, + "loss": 3.0285, + "theoretical_loss": 5.376928924366901, + "tokens_seen": 45875200 + }, + { + "epoch": 0.01, + "learning_rate": 0.00079683851554664, + "loss": 3.076, + "theoretical_loss": 5.37575417277413, + "tokens_seen": 45940736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007968224674022067, + "loss": 3.2188, + "theoretical_loss": 5.374581564278184, + "tokens_seen": 46006272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007968064192577734, + "loss": 3.0464, + "theoretical_loss": 5.373411091924918, + "tokens_seen": 46071808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007967903711133401, + "loss": 3.0265, + "theoretical_loss": 5.372242748792598, + "tokens_seen": 46137344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007967743229689068, + "loss": 3.1034, + "theoretical_loss": 5.371076527991711, + "tokens_seen": 46202880 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007967582748244735, + "loss": 3.1293, + "theoretical_loss": 5.36991242266477, + "tokens_seen": 46268416 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007967422266800402, + "loss": 2.8727, + "theoretical_loss": 5.36875042598612, + "tokens_seen": 46333952 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007967261785356069, + "loss": 3.0456, + "theoretical_loss": 5.3675905311617385, + "tokens_seen": 46399488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007967101303911736, + "loss": 3.096, + "theoretical_loss": 5.366432731429054, + "tokens_seen": 46465024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007966940822467403, + "loss": 3.3051, + "theoretical_loss": 5.365277020056752, + "tokens_seen": 46530560 + }, + { + "epoch": 0.01, + "learning_rate": 0.000796678034102307, + "loss": 2.9616, + "theoretical_loss": 5.364123390344582, + "tokens_seen": 46596096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007966619859578737, + "loss": 3.2195, + "theoretical_loss": 5.362971835623181, + "tokens_seen": 46661632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007966459378134404, + "loss": 3.0819, + "theoretical_loss": 5.361822349253881, + "tokens_seen": 46727168 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007966298896690071, + "loss": 3.1308, + "theoretical_loss": 5.360674924628524, + "tokens_seen": 46792704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007966138415245738, + "loss": 3.1266, + "theoretical_loss": 5.359529555169287, + "tokens_seen": 46858240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007965977933801405, + "loss": 3.085, + "theoretical_loss": 5.358386234328487, + "tokens_seen": 46923776 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007965817452357072, + "loss": 3.0001, + "theoretical_loss": 5.357244955588417, + "tokens_seen": 46989312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007965656970912739, + "loss": 3.1811, + "theoretical_loss": 5.356105712461154, + "tokens_seen": 47054848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007965496489468405, + "loss": 3.1944, + "theoretical_loss": 5.354968498488391, + "tokens_seen": 47120384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007965336008024073, + "loss": 2.9514, + "theoretical_loss": 5.353833307241256, + "tokens_seen": 47185920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007965175526579739, + "loss": 3.0924, + "theoretical_loss": 5.352700132320135, + "tokens_seen": 47251456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007965015045135407, + "loss": 3.097, + "theoretical_loss": 5.351568967354504, + "tokens_seen": 47316992 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007964854563691073, + "loss": 3.2562, + "theoretical_loss": 5.350439806002757, + "tokens_seen": 47382528 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007964694082246741, + "loss": 2.9039, + "theoretical_loss": 5.349312641952025, + "tokens_seen": 47448064 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007964533600802407, + "loss": 2.9345, + "theoretical_loss": 5.348187468918022, + "tokens_seen": 47513600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007964373119358075, + "loss": 3.0987, + "theoretical_loss": 5.347064280644861, + "tokens_seen": 47579136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007964212637913741, + "loss": 3.0397, + "theoretical_loss": 5.345943070904897, + "tokens_seen": 47644672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007964052156469408, + "loss": 3.1332, + "theoretical_loss": 5.344823833498557, + "tokens_seen": 47710208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007963891675025075, + "loss": 2.9783, + "theoretical_loss": 5.343706562254177, + "tokens_seen": 47775744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007963731193580743, + "loss": 3.0531, + "theoretical_loss": 5.342591251027833, + "tokens_seen": 47841280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007963570712136409, + "loss": 3.0778, + "theoretical_loss": 5.3414778937031855, + "tokens_seen": 47906816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007963410230692077, + "loss": 2.9572, + "theoretical_loss": 5.340366484191316, + "tokens_seen": 47972352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007963249749247743, + "loss": 2.9102, + "theoretical_loss": 5.339257016430563, + "tokens_seen": 48037888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007963089267803411, + "loss": 2.8625, + "theoretical_loss": 5.338149484386367, + "tokens_seen": 48103424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007962928786359077, + "loss": 2.8076, + "theoretical_loss": 5.337043882051111, + "tokens_seen": 48168960 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007962768304914745, + "loss": 3.1662, + "theoretical_loss": 5.3359402034439665, + "tokens_seen": 48234496 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007962607823470411, + "loss": 3.0451, + "theoretical_loss": 5.334838442610732, + "tokens_seen": 48300032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007962447342026079, + "loss": 2.9153, + "theoretical_loss": 5.333738593623686, + "tokens_seen": 48365568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007962286860581745, + "loss": 2.8819, + "theoretical_loss": 5.332640650581424, + "tokens_seen": 48431104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007962126379137413, + "loss": 2.9641, + "theoretical_loss": 5.331544607608718, + "tokens_seen": 48496640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007961965897693079, + "loss": 2.9987, + "theoretical_loss": 5.330450458856356, + "tokens_seen": 48562176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007961805416248747, + "loss": 3.0426, + "theoretical_loss": 5.329358198500994, + "tokens_seen": 48627712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007961644934804413, + "loss": 3.0788, + "theoretical_loss": 5.328267820745011, + "tokens_seen": 48693248 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007961484453360081, + "loss": 2.9519, + "theoretical_loss": 5.327179319816354, + "tokens_seen": 48758784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007961323971915747, + "loss": 2.9443, + "theoretical_loss": 5.326092689968397, + "tokens_seen": 48824320 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007961163490471415, + "loss": 3.0188, + "theoretical_loss": 5.3250079254797935, + "tokens_seen": 48889856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007961003009027081, + "loss": 3.0229, + "theoretical_loss": 5.323925020654329, + "tokens_seen": 48955392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007960842527582749, + "loss": 3.0352, + "theoretical_loss": 5.322843969820781, + "tokens_seen": 49020928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007960682046138415, + "loss": 3.0911, + "theoretical_loss": 5.321764767332772, + "tokens_seen": 49086464 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.9820156693458557, + "objective/train/avg_token_score": 0.017984312027692795, + "objective/train/docs_used": 25394, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 2.9772536754608154, + "objective/train/original_loss": 3.037050247192383, + "objective/train/theoretical_loss": 5.320687407568633, + "objective/train/tokens_used": 69612000, + "objective/train/unlikelihood_loss": 0.017060665413737297, + "theoretical_loss": 5.320687407568633, + "tokens_seen": 49152000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007960521564694083, + "loss": 2.953, + "theoretical_loss": 5.320687407568633, + "tokens_seen": 49152000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007960361083249749, + "loss": 2.9305, + "theoretical_loss": 5.3196118849312555, + "tokens_seen": 49217536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007960200601805417, + "loss": 2.8388, + "theoretical_loss": 5.31853819384796, + "tokens_seen": 49283072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007960040120361083, + "loss": 2.9691, + "theoretical_loss": 5.31746632877035, + "tokens_seen": 49348608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007959879638916751, + "loss": 2.957, + "theoretical_loss": 5.316396284174177, + "tokens_seen": 49414144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007959719157472417, + "loss": 2.6974, + "theoretical_loss": 5.3153280545592025, + "tokens_seen": 49479680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007959558676028085, + "loss": 2.8876, + "theoretical_loss": 5.314261634449066, + "tokens_seen": 49545216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007959398194583751, + "loss": 2.9662, + "theoretical_loss": 5.313197018391143, + "tokens_seen": 49610752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007959237713139419, + "loss": 2.8788, + "theoretical_loss": 5.312134200956416, + "tokens_seen": 49676288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007959077231695085, + "loss": 2.8449, + "theoretical_loss": 5.3110731767393435, + "tokens_seen": 49741824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007958916750250753, + "loss": 2.871, + "theoretical_loss": 5.31001394035772, + "tokens_seen": 49807360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007958756268806419, + "loss": 2.7826, + "theoretical_loss": 5.308956486452549, + "tokens_seen": 49872896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007958595787362087, + "loss": 2.9761, + "theoretical_loss": 5.307900809687913, + "tokens_seen": 49938432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007958435305917753, + "loss": 2.8855, + "theoretical_loss": 5.306846904750847, + "tokens_seen": 50003968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007958274824473421, + "loss": 2.8678, + "theoretical_loss": 5.305794766351202, + "tokens_seen": 50069504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007958114343029087, + "loss": 2.8979, + "theoretical_loss": 5.304744389221523, + "tokens_seen": 50135040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007957953861584755, + "loss": 2.9236, + "theoretical_loss": 5.303695768116921, + "tokens_seen": 50200576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007957793380140421, + "loss": 2.7573, + "theoretical_loss": 5.302648897814946, + "tokens_seen": 50266112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007957632898696089, + "loss": 2.8582, + "theoretical_loss": 5.301603773115465, + "tokens_seen": 50331648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007957472417251755, + "loss": 2.9166, + "theoretical_loss": 5.300560388840532, + "tokens_seen": 50397184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007957311935807423, + "loss": 2.8211, + "theoretical_loss": 5.299518739834273, + "tokens_seen": 50462720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007957151454363089, + "loss": 2.7976, + "theoretical_loss": 5.298478820962751, + "tokens_seen": 50528256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007956990972918757, + "loss": 2.8449, + "theoretical_loss": 5.297440627113859, + "tokens_seen": 50593792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007956830491474423, + "loss": 2.8795, + "theoretical_loss": 5.296404153197189, + "tokens_seen": 50659328 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007956670010030091, + "loss": 2.8632, + "theoretical_loss": 5.295369394143913, + "tokens_seen": 50724864 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007956509528585757, + "loss": 3.0238, + "theoretical_loss": 5.294336344906668, + "tokens_seen": 50790400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007956349047141425, + "loss": 2.9689, + "theoretical_loss": 5.293305000459435, + "tokens_seen": 50855936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007956188565697091, + "loss": 2.8181, + "theoretical_loss": 5.292275355797421, + "tokens_seen": 50921472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007956028084252759, + "loss": 2.8916, + "theoretical_loss": 5.291247405936941, + "tokens_seen": 50987008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007955867602808426, + "loss": 2.7535, + "theoretical_loss": 5.290221145915307, + "tokens_seen": 51052544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007955707121364093, + "loss": 2.7688, + "theoretical_loss": 5.289196570790708, + "tokens_seen": 51118080 + }, + { + "epoch": 0.02, + "learning_rate": 0.000795554663991976, + "loss": 2.9593, + "theoretical_loss": 5.288173675642094, + "tokens_seen": 51183616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007955386158475427, + "loss": 2.7048, + "theoretical_loss": 5.287152455569069, + "tokens_seen": 51249152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007955225677031094, + "loss": 2.6929, + "theoretical_loss": 5.286132905691774, + "tokens_seen": 51314688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007955065195586761, + "loss": 2.7997, + "theoretical_loss": 5.285115021150775, + "tokens_seen": 51380224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007954904714142428, + "loss": 2.8348, + "theoretical_loss": 5.284098797106948, + "tokens_seen": 51445760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007954744232698095, + "loss": 2.918, + "theoretical_loss": 5.283084228741377, + "tokens_seen": 51511296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007954583751253762, + "loss": 2.7921, + "theoretical_loss": 5.282071311255235, + "tokens_seen": 51576832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007954423269809429, + "loss": 2.6101, + "theoretical_loss": 5.2810600398696845, + "tokens_seen": 51642368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007954262788365096, + "loss": 2.7814, + "theoretical_loss": 5.2800504098257575, + "tokens_seen": 51707904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007954102306920763, + "loss": 2.7093, + "theoretical_loss": 5.279042416384257, + "tokens_seen": 51773440 + }, + { + "epoch": 0.02, + "learning_rate": 0.000795394182547643, + "loss": 2.8512, + "theoretical_loss": 5.278036054825645, + "tokens_seen": 51838976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007953781344032097, + "loss": 2.7851, + "theoretical_loss": 5.277031320449941, + "tokens_seen": 51904512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007953620862587764, + "loss": 2.7801, + "theoretical_loss": 5.27602820857661, + "tokens_seen": 51970048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007953460381143431, + "loss": 2.8081, + "theoretical_loss": 5.275026714544463, + "tokens_seen": 52035584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007953299899699098, + "loss": 2.696, + "theoretical_loss": 5.27402683371155, + "tokens_seen": 52101120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007953139418254765, + "loss": 2.6681, + "theoretical_loss": 5.273028561455056, + "tokens_seen": 52166656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007952978936810432, + "loss": 2.8067, + "theoretical_loss": 5.272031893171203, + "tokens_seen": 52232192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007952818455366099, + "loss": 2.7718, + "theoretical_loss": 5.27103682427514, + "tokens_seen": 52297728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007952657973921766, + "loss": 2.8133, + "theoretical_loss": 5.270043350200851, + "tokens_seen": 52363264 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.994269073009491, + "objective/train/avg_token_score": 0.005730891600251198, + "objective/train/docs_used": 26541, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 2.636657953262329, + "objective/train/original_loss": 2.6505560874938965, + "objective/train/theoretical_loss": 5.269051466401043, + "objective/train/tokens_used": 72888800, + "objective/train/unlikelihood_loss": 0.00934041291475296, + "theoretical_loss": 5.269051466401043, + "tokens_seen": 52428800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007952497492477433, + "loss": 2.6182, + "theoretical_loss": 5.269051466401043, + "tokens_seen": 52428800 + }, + { + "epoch": 0.02, + "learning_rate": 0.00079523370110331, + "loss": 2.7373, + "theoretical_loss": 5.268061168347058, + "tokens_seen": 52494336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007952176529588767, + "loss": 2.7905, + "theoretical_loss": 5.267072451528764, + "tokens_seen": 52559872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007952016048144434, + "loss": 2.7865, + "theoretical_loss": 5.266085311454461, + "tokens_seen": 52625408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007951855566700101, + "loss": 2.7629, + "theoretical_loss": 5.265099743650779, + "tokens_seen": 52690944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007951695085255768, + "loss": 2.8629, + "theoretical_loss": 5.264115743662588, + "tokens_seen": 52756480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007951534603811434, + "loss": 2.7875, + "theoretical_loss": 5.26313330705289, + "tokens_seen": 52822016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007951374122367102, + "loss": 2.6782, + "theoretical_loss": 5.262152429402734, + "tokens_seen": 52887552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007951213640922768, + "loss": 2.673, + "theoretical_loss": 5.261173106311116, + "tokens_seen": 52953088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007951053159478436, + "loss": 2.8434, + "theoretical_loss": 5.2601953333948765, + "tokens_seen": 53018624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007950892678034102, + "loss": 2.7072, + "theoretical_loss": 5.25921910628862, + "tokens_seen": 53084160 + }, + { + "epoch": 0.02, + "learning_rate": 0.000795073219658977, + "loss": 2.638, + "theoretical_loss": 5.258244420644611, + "tokens_seen": 53149696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007950571715145436, + "loss": 2.7052, + "theoretical_loss": 5.257271272132685, + "tokens_seen": 53215232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007950411233701104, + "loss": 2.639, + "theoretical_loss": 5.256299656440154, + "tokens_seen": 53280768 + }, + { + "epoch": 0.02, + "learning_rate": 0.000795025075225677, + "loss": 2.6754, + "theoretical_loss": 5.255329569271717, + "tokens_seen": 53346304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007950090270812438, + "loss": 2.577, + "theoretical_loss": 5.254361006349367, + "tokens_seen": 53411840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007949929789368104, + "loss": 2.7154, + "theoretical_loss": 5.2533939634122975, + "tokens_seen": 53477376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007949769307923772, + "loss": 2.6511, + "theoretical_loss": 5.252428436216821, + "tokens_seen": 53542912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007949608826479438, + "loss": 2.6755, + "theoretical_loss": 5.251464420536269, + "tokens_seen": 53608448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007949448345035106, + "loss": 2.6843, + "theoretical_loss": 5.250501912160911, + "tokens_seen": 53673984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007949287863590772, + "loss": 2.7651, + "theoretical_loss": 5.24954090689786, + "tokens_seen": 53739520 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794912738214644, + "loss": 2.867, + "theoretical_loss": 5.248581400570989, + "tokens_seen": 53805056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007948966900702106, + "loss": 2.7362, + "theoretical_loss": 5.247623389020841, + "tokens_seen": 53870592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007948806419257774, + "loss": 2.6583, + "theoretical_loss": 5.246666868104547, + "tokens_seen": 53936128 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794864593781344, + "loss": 2.7012, + "theoretical_loss": 5.245711833695731, + "tokens_seen": 54001664 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007948485456369108, + "loss": 2.7442, + "theoretical_loss": 5.244758281684433, + "tokens_seen": 54067200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007948324974924774, + "loss": 2.6995, + "theoretical_loss": 5.24380620797702, + "tokens_seen": 54132736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007948164493480442, + "loss": 2.4358, + "theoretical_loss": 5.242855608496102, + "tokens_seen": 54198272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007948004012036108, + "loss": 2.7642, + "theoretical_loss": 5.2419064791804475, + "tokens_seen": 54263808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007947843530591776, + "loss": 2.6872, + "theoretical_loss": 5.240958815984901, + "tokens_seen": 54329344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007947683049147442, + "loss": 2.5897, + "theoretical_loss": 5.2400126148803, + "tokens_seen": 54394880 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794752256770311, + "loss": 2.6354, + "theoretical_loss": 5.23906787185339, + "tokens_seen": 54460416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007947362086258776, + "loss": 2.6464, + "theoretical_loss": 5.238124582906748, + "tokens_seen": 54525952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007947201604814444, + "loss": 2.7152, + "theoretical_loss": 5.237182744058696, + "tokens_seen": 54591488 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794704112337011, + "loss": 2.6205, + "theoretical_loss": 5.236242351343224, + "tokens_seen": 54657024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007946880641925778, + "loss": 2.5782, + "theoretical_loss": 5.235303400809904, + "tokens_seen": 54722560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007946720160481444, + "loss": 2.6758, + "theoretical_loss": 5.2343658885238185, + "tokens_seen": 54788096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007946559679037112, + "loss": 2.6709, + "theoretical_loss": 5.2334298105654735, + "tokens_seen": 54853632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007946399197592778, + "loss": 2.7045, + "theoretical_loss": 5.232495163030723, + "tokens_seen": 54919168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007946238716148446, + "loss": 2.7114, + "theoretical_loss": 5.231561942030694, + "tokens_seen": 54984704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007946078234704112, + "loss": 2.6581, + "theoretical_loss": 5.230630143691698, + "tokens_seen": 55050240 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794591775325978, + "loss": 2.8276, + "theoretical_loss": 5.22969976415517, + "tokens_seen": 55115776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007945757271815446, + "loss": 2.3885, + "theoretical_loss": 5.228770799577576, + "tokens_seen": 55181312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007945596790371114, + "loss": 2.6495, + "theoretical_loss": 5.227843246130343, + "tokens_seen": 55246848 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794543630892678, + "loss": 2.5486, + "theoretical_loss": 5.2269170999997865, + "tokens_seen": 55312384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007945275827482448, + "loss": 2.6656, + "theoretical_loss": 5.225992357387031, + "tokens_seen": 55377920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007945115346038114, + "loss": 2.6198, + "theoretical_loss": 5.225069014507933, + "tokens_seen": 55443456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007944954864593782, + "loss": 2.4447, + "theoretical_loss": 5.224147067593012, + "tokens_seen": 55508992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007944794383149448, + "loss": 2.6668, + "theoretical_loss": 5.223226512887375, + "tokens_seen": 55574528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007944633901705116, + "loss": 2.64, + "theoretical_loss": 5.222307346650636, + "tokens_seen": 55640064 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.9954494833946228, + "objective/train/avg_token_score": 0.004550525452941656, + "objective/train/docs_used": 27758, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 2.511780261993408, + "objective/train/original_loss": 2.522498607635498, + "objective/train/theoretical_loss": 5.221389565156855, + "objective/train/tokens_used": 76165600, + "objective/train/unlikelihood_loss": 0.005740185733884573, + "theoretical_loss": 5.221389565156855, + "tokens_seen": 55705600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007944473420260782, + "loss": 2.5534, + "theoretical_loss": 5.221389565156855, + "tokens_seen": 55705600 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794431293881645, + "loss": 2.8422, + "theoretical_loss": 5.2204731646944555, + "tokens_seen": 55771136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007944152457372116, + "loss": 2.5383, + "theoretical_loss": 5.219558141566156, + "tokens_seen": 55836672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007943991975927784, + "loss": 2.5989, + "theoretical_loss": 5.218644492088899, + "tokens_seen": 55902208 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794383149448345, + "loss": 2.5539, + "theoretical_loss": 5.217732212593778, + "tokens_seen": 55967744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007943671013039118, + "loss": 2.6612, + "theoretical_loss": 5.216821299425968, + "tokens_seen": 56033280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007943510531594784, + "loss": 2.5958, + "theoretical_loss": 5.215911748944652, + "tokens_seen": 56098816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007943350050150452, + "loss": 2.6564, + "theoretical_loss": 5.215003557522959, + "tokens_seen": 56164352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007943189568706118, + "loss": 2.6209, + "theoretical_loss": 5.21409672154788, + "tokens_seen": 56229888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007943029087261786, + "loss": 2.4463, + "theoretical_loss": 5.213191237420215, + "tokens_seen": 56295424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007942868605817452, + "loss": 2.6189, + "theoretical_loss": 5.212287101554496, + "tokens_seen": 56360960 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794270812437312, + "loss": 2.4595, + "theoretical_loss": 5.211384310378916, + "tokens_seen": 56426496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007942547642928786, + "loss": 2.6628, + "theoretical_loss": 5.210482860335269, + "tokens_seen": 56492032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007942387161484454, + "loss": 2.5435, + "theoretical_loss": 5.2095827478788745, + "tokens_seen": 56557568 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794222668004012, + "loss": 2.6216, + "theoretical_loss": 5.208683969478518, + "tokens_seen": 56623104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007942066198595788, + "loss": 2.5605, + "theoretical_loss": 5.2077865216163755, + "tokens_seen": 56688640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007941905717151455, + "loss": 2.4839, + "theoretical_loss": 5.206890400787959, + "tokens_seen": 56754176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007941745235707122, + "loss": 2.5195, + "theoretical_loss": 5.205995603502039, + "tokens_seen": 56819712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007941584754262789, + "loss": 2.5755, + "theoretical_loss": 5.205102126280586, + "tokens_seen": 56885248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007941424272818456, + "loss": 2.5385, + "theoretical_loss": 5.204209965658704, + "tokens_seen": 56950784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007941263791374123, + "loss": 2.5599, + "theoretical_loss": 5.203319118184565, + "tokens_seen": 57016320 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794110330992979, + "loss": 2.5215, + "theoretical_loss": 5.202429580419343, + "tokens_seen": 57081856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007940942828485457, + "loss": 2.5628, + "theoretical_loss": 5.201541348937157, + "tokens_seen": 57147392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007940782347041124, + "loss": 2.4622, + "theoretical_loss": 5.200654420324998, + "tokens_seen": 57212928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007940621865596791, + "loss": 2.4099, + "theoretical_loss": 5.199768791182677, + "tokens_seen": 57278464 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007940461384152458, + "loss": 2.5814, + "theoretical_loss": 5.198884458122748, + "tokens_seen": 57344000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007940300902708125, + "loss": 2.6054, + "theoretical_loss": 5.198001417770461, + "tokens_seen": 57409536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007940140421263792, + "loss": 2.5461, + "theoretical_loss": 5.1971196667636885, + "tokens_seen": 57475072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007939979939819459, + "loss": 2.5265, + "theoretical_loss": 5.196239201752871, + "tokens_seen": 57540608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007939819458375126, + "loss": 2.5353, + "theoretical_loss": 5.19536001940095, + "tokens_seen": 57606144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007939658976930794, + "loss": 2.4569, + "theoretical_loss": 5.1944821163833135, + "tokens_seen": 57671680 + }, + { + "epoch": 0.02, + "learning_rate": 0.000793949849548646, + "loss": 2.5071, + "theoretical_loss": 5.19360548938773, + "tokens_seen": 57737216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007939338014042128, + "loss": 2.4756, + "theoretical_loss": 5.192730135114292, + "tokens_seen": 57802752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007939177532597794, + "loss": 2.5997, + "theoretical_loss": 5.191856050275353, + "tokens_seen": 57868288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007939017051153462, + "loss": 2.655, + "theoretical_loss": 5.190983231595473, + "tokens_seen": 57933824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007938856569709128, + "loss": 2.4081, + "theoretical_loss": 5.1901116758113535, + "tokens_seen": 57999360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007938696088264795, + "loss": 2.413, + "theoretical_loss": 5.189241379671782, + "tokens_seen": 58064896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007938535606820462, + "loss": 2.6192, + "theoretical_loss": 5.188372339937576, + "tokens_seen": 58130432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007938375125376129, + "loss": 2.6009, + "theoretical_loss": 5.187504553381521, + "tokens_seen": 58195968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007938214643931796, + "loss": 2.3811, + "theoretical_loss": 5.1866380167883115, + "tokens_seen": 58261504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007938054162487463, + "loss": 2.5614, + "theoretical_loss": 5.1857727269545, + "tokens_seen": 58327040 + }, + { + "epoch": 0.02, + "learning_rate": 0.000793789368104313, + "loss": 2.5567, + "theoretical_loss": 5.184908680688437, + "tokens_seen": 58392576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007937733199598797, + "loss": 2.4501, + "theoretical_loss": 5.18404587481021, + "tokens_seen": 58458112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007937572718154463, + "loss": 2.3957, + "theoretical_loss": 5.183184306151595, + "tokens_seen": 58523648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007937412236710131, + "loss": 2.4153, + "theoretical_loss": 5.182323971555995, + "tokens_seen": 58589184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007937251755265797, + "loss": 2.592, + "theoretical_loss": 5.181464867878384, + "tokens_seen": 58654720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007937091273821465, + "loss": 2.7054, + "theoretical_loss": 5.1806069919852575, + "tokens_seen": 58720256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007936930792377131, + "loss": 2.4631, + "theoretical_loss": 5.17975034075457, + "tokens_seen": 58785792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007936770310932799, + "loss": 2.5008, + "theoretical_loss": 5.1788949110756874, + "tokens_seen": 58851328 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007936609829488465, + "loss": 2.3922, + "theoretical_loss": 5.178040699849327, + "tokens_seen": 58916864 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.9918819069862366, + "objective/train/avg_token_score": 0.008118084631860256, + "objective/train/docs_used": 29071, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 2.2796335220336914, + "objective/train/original_loss": 2.2954113483428955, + "objective/train/theoretical_loss": 5.1771877039875065, + "objective/train/tokens_used": 79442400, + "objective/train/unlikelihood_loss": 0.01613558456301689, + "theoretical_loss": 5.1771877039875065, + "tokens_seen": 58982400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007936449348044133, + "loss": 2.3608, + "theoretical_loss": 5.1771877039875065, + "tokens_seen": 58982400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007936288866599799, + "loss": 2.4356, + "theoretical_loss": 5.176335920413489, + "tokens_seen": 59047936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007936128385155467, + "loss": 2.5364, + "theoretical_loss": 5.175485346061732, + "tokens_seen": 59113472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007935967903711133, + "loss": 2.4112, + "theoretical_loss": 5.174635977877831, + "tokens_seen": 59179008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007935807422266801, + "loss": 2.5245, + "theoretical_loss": 5.173787812818468, + "tokens_seen": 59244544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007935646940822467, + "loss": 2.4729, + "theoretical_loss": 5.172940847851359, + "tokens_seen": 59310080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007935486459378135, + "loss": 2.4827, + "theoretical_loss": 5.172095079955204, + "tokens_seen": 59375616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007935325977933801, + "loss": 2.3052, + "theoretical_loss": 5.171250506119634, + "tokens_seen": 59441152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007935165496489469, + "loss": 2.6056, + "theoretical_loss": 5.170407123345155, + "tokens_seen": 59506688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007935005015045135, + "loss": 2.4306, + "theoretical_loss": 5.169564928643103, + "tokens_seen": 59572224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007934844533600803, + "loss": 2.4579, + "theoretical_loss": 5.168723919035591, + "tokens_seen": 59637760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007934684052156469, + "loss": 2.3348, + "theoretical_loss": 5.167884091555456, + "tokens_seen": 59703296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007934523570712137, + "loss": 2.4749, + "theoretical_loss": 5.167045443246213, + "tokens_seen": 59768832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007934363089267803, + "loss": 2.403, + "theoretical_loss": 5.166207971162001, + "tokens_seen": 59834368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007934202607823471, + "loss": 2.4256, + "theoretical_loss": 5.1653716723675345, + "tokens_seen": 59899904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007934042126379137, + "loss": 2.542, + "theoretical_loss": 5.164536543938056, + "tokens_seen": 59965440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007933881644934805, + "loss": 2.3687, + "theoretical_loss": 5.163702582959283, + "tokens_seen": 60030976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007933721163490471, + "loss": 2.6252, + "theoretical_loss": 5.162869786527363, + "tokens_seen": 60096512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007933560682046139, + "loss": 2.5445, + "theoretical_loss": 5.1620381517488205, + "tokens_seen": 60162048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007933400200601805, + "loss": 2.5014, + "theoretical_loss": 5.161207675740515, + "tokens_seen": 60227584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007933239719157473, + "loss": 2.4505, + "theoretical_loss": 5.160378355629584, + "tokens_seen": 60293120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007933079237713139, + "loss": 2.4687, + "theoretical_loss": 5.159550188553405, + "tokens_seen": 60358656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007932918756268807, + "loss": 2.4909, + "theoretical_loss": 5.15872317165954, + "tokens_seen": 60424192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007932758274824473, + "loss": 2.4575, + "theoretical_loss": 5.1578973021056935, + "tokens_seen": 60489728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007932597793380141, + "loss": 2.4113, + "theoretical_loss": 5.157072577059662, + "tokens_seen": 60555264 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007932437311935807, + "loss": 2.5626, + "theoretical_loss": 5.15624899369929, + "tokens_seen": 60620800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007932276830491475, + "loss": 2.4765, + "theoretical_loss": 5.155426549212422, + "tokens_seen": 60686336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007932116349047141, + "loss": 2.4922, + "theoretical_loss": 5.154605240796855, + "tokens_seen": 60751872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007931955867602809, + "loss": 2.4133, + "theoretical_loss": 5.153785065660296, + "tokens_seen": 60817408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007931795386158475, + "loss": 2.5474, + "theoretical_loss": 5.152966021020315, + "tokens_seen": 60882944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007931634904714143, + "loss": 2.4446, + "theoretical_loss": 5.152148104104297, + "tokens_seen": 60948480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007931474423269809, + "loss": 2.3764, + "theoretical_loss": 5.1513313121494, + "tokens_seen": 61014016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007931313941825477, + "loss": 2.4892, + "theoretical_loss": 5.15051564240251, + "tokens_seen": 61079552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007931153460381143, + "loss": 2.4919, + "theoretical_loss": 5.149701092120194, + "tokens_seen": 61145088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007930992978936811, + "loss": 2.3772, + "theoretical_loss": 5.148887658568658, + "tokens_seen": 61210624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007930832497492477, + "loss": 2.49, + "theoretical_loss": 5.148075339023699, + "tokens_seen": 61276160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007930672016048145, + "loss": 2.4395, + "theoretical_loss": 5.147264130770671, + "tokens_seen": 61341696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007930511534603811, + "loss": 2.4172, + "theoretical_loss": 5.146454031104426, + "tokens_seen": 61407232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007930351053159479, + "loss": 2.5592, + "theoretical_loss": 5.1456450373292855, + "tokens_seen": 61472768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007930190571715145, + "loss": 2.4515, + "theoretical_loss": 5.144837146758987, + "tokens_seen": 61538304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007930030090270813, + "loss": 2.4838, + "theoretical_loss": 5.144030356716646, + "tokens_seen": 61603840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007929869608826479, + "loss": 2.4743, + "theoretical_loss": 5.1432246645347135, + "tokens_seen": 61669376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007929709127382147, + "loss": 2.3535, + "theoretical_loss": 5.14242006755493, + "tokens_seen": 61734912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007929548645937813, + "loss": 2.4355, + "theoretical_loss": 5.141616563128288, + "tokens_seen": 61800448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007929388164493481, + "loss": 2.4821, + "theoretical_loss": 5.140814148614985, + "tokens_seen": 61865984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007929227683049147, + "loss": 2.4513, + "theoretical_loss": 5.1400128213843885, + "tokens_seen": 61931520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007929067201604815, + "loss": 2.5603, + "theoretical_loss": 5.139212578814986, + "tokens_seen": 61997056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007928906720160481, + "loss": 2.4544, + "theoretical_loss": 5.13841341829435, + "tokens_seen": 62062592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007928746238716149, + "loss": 2.3532, + "theoretical_loss": 5.137615337219098, + "tokens_seen": 62128128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007928585757271815, + "loss": 2.4885, + "theoretical_loss": 5.136818332994844, + "tokens_seen": 62193664 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.9942206144332886, + "objective/train/avg_token_score": 0.005779375322163105, + "objective/train/docs_used": 30276, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 2.6771793365478516, + "objective/train/original_loss": 2.6935808658599854, + "objective/train/theoretical_loss": 5.136022403036167, + "objective/train/tokens_used": 82719200, + "objective/train/unlikelihood_loss": 0.004698086529970169, + "theoretical_loss": 5.136022403036167, + "tokens_seen": 62259200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007928425275827483, + "loss": 2.5336, + "theoretical_loss": 5.136022403036167, + "tokens_seen": 62259200 + }, + { + "epoch": 0.02, + "learning_rate": 0.000792826479438315, + "loss": 2.3592, + "theoretical_loss": 5.135227544766565, + "tokens_seen": 62324736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007928104312938817, + "loss": 2.512, + "theoretical_loss": 5.134433755618414, + "tokens_seen": 62390272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007927943831494484, + "loss": 2.28, + "theoretical_loss": 5.1336410330329345, + "tokens_seen": 62455808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007927783350050151, + "loss": 2.4075, + "theoretical_loss": 5.132849374460145, + "tokens_seen": 62521344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007927622868605819, + "loss": 2.4352, + "theoretical_loss": 5.132058777358827, + "tokens_seen": 62586880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007927462387161485, + "loss": 2.3604, + "theoretical_loss": 5.1312692391964845, + "tokens_seen": 62652416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007927301905717153, + "loss": 2.368, + "theoretical_loss": 5.1304807574493, + "tokens_seen": 62717952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007927141424272819, + "loss": 2.4086, + "theoretical_loss": 5.129693329602108, + "tokens_seen": 62783488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007926980942828487, + "loss": 2.3634, + "theoretical_loss": 5.128906953148342, + "tokens_seen": 62849024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007926820461384153, + "loss": 2.4511, + "theoretical_loss": 5.128121625590007, + "tokens_seen": 62914560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007926659979939821, + "loss": 2.4682, + "theoretical_loss": 5.127337344437633, + "tokens_seen": 62980096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007926499498495487, + "loss": 2.5013, + "theoretical_loss": 5.126554107210245, + "tokens_seen": 63045632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007926339017051155, + "loss": 2.4125, + "theoretical_loss": 5.125771911435321, + "tokens_seen": 63111168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007926178535606821, + "loss": 2.3802, + "theoretical_loss": 5.124990754648754, + "tokens_seen": 63176704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007926018054162489, + "loss": 2.3876, + "theoretical_loss": 5.124210634394814, + "tokens_seen": 63242240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007925857572718155, + "loss": 2.308, + "theoretical_loss": 5.1234315482261135, + "tokens_seen": 63307776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007925697091273823, + "loss": 2.3271, + "theoretical_loss": 5.122653493703571, + "tokens_seen": 63373312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007925536609829489, + "loss": 2.4051, + "theoretical_loss": 5.121876468396373, + "tokens_seen": 63438848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007925376128385157, + "loss": 2.3895, + "theoretical_loss": 5.121100469881933, + "tokens_seen": 63504384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007925215646940823, + "loss": 2.2927, + "theoretical_loss": 5.120325495745863, + "tokens_seen": 63569920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007925055165496491, + "loss": 2.314, + "theoretical_loss": 5.119551543581933, + "tokens_seen": 63635456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007924894684052157, + "loss": 2.4049, + "theoretical_loss": 5.118778610992038, + "tokens_seen": 63700992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007924734202607824, + "loss": 2.4, + "theoretical_loss": 5.118006695586154, + "tokens_seen": 63766528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007924573721163491, + "loss": 2.3509, + "theoretical_loss": 5.1172357949823155, + "tokens_seen": 63832064 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007924413239719158, + "loss": 2.44, + "theoretical_loss": 5.116465906806567, + "tokens_seen": 63897600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007924252758274825, + "loss": 2.4316, + "theoretical_loss": 5.11569702869294, + "tokens_seen": 63963136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007924092276830492, + "loss": 2.2556, + "theoretical_loss": 5.114929158283407, + "tokens_seen": 64028672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007923931795386159, + "loss": 2.3595, + "theoretical_loss": 5.114162293227853, + "tokens_seen": 64094208 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007923771313941826, + "loss": 2.4, + "theoretical_loss": 5.113396431184043, + "tokens_seen": 64159744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007923610832497493, + "loss": 2.3169, + "theoretical_loss": 5.112631569817578, + "tokens_seen": 64225280 + }, + { + "epoch": 0.02, + "learning_rate": 0.000792345035105316, + "loss": 2.3382, + "theoretical_loss": 5.111867706801872, + "tokens_seen": 64290816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007923289869608826, + "loss": 2.2923, + "theoretical_loss": 5.111104839818113, + "tokens_seen": 64356352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007923129388164494, + "loss": 2.4212, + "theoretical_loss": 5.1103429665552245, + "tokens_seen": 64421888 + }, + { + "epoch": 0.02, + "learning_rate": 0.000792296890672016, + "loss": 2.4353, + "theoretical_loss": 5.109582084709841, + "tokens_seen": 64487424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007922808425275828, + "loss": 2.4444, + "theoretical_loss": 5.108822191986269, + "tokens_seen": 64552960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007922647943831494, + "loss": 2.2667, + "theoretical_loss": 5.108063286096453, + "tokens_seen": 64618496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007922487462387162, + "loss": 2.3466, + "theoretical_loss": 5.107305364759943, + "tokens_seen": 64684032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007922326980942828, + "loss": 2.3051, + "theoretical_loss": 5.106548425703869, + "tokens_seen": 64749568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007922166499498496, + "loss": 2.4669, + "theoretical_loss": 5.1057924666628915, + "tokens_seen": 64815104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007922006018054162, + "loss": 2.386, + "theoretical_loss": 5.105037485379189, + "tokens_seen": 64880640 + }, + { + "epoch": 0.02, + "learning_rate": 0.000792184553660983, + "loss": 2.4803, + "theoretical_loss": 5.104283479602407, + "tokens_seen": 64946176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007921685055165496, + "loss": 2.3986, + "theoretical_loss": 5.103530447089642, + "tokens_seen": 65011712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007921524573721164, + "loss": 2.3749, + "theoretical_loss": 5.102778385605397, + "tokens_seen": 65077248 + }, + { + "epoch": 0.02, + "learning_rate": 0.000792136409227683, + "loss": 2.403, + "theoretical_loss": 5.102027292921557, + "tokens_seen": 65142784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007921203610832498, + "loss": 2.4149, + "theoretical_loss": 5.101277166817351, + "tokens_seen": 65208320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007921043129388164, + "loss": 2.2564, + "theoretical_loss": 5.100528005079328, + "tokens_seen": 65273856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007920882647943832, + "loss": 2.3685, + "theoretical_loss": 5.099779805501318, + "tokens_seen": 65339392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007920722166499498, + "loss": 2.3838, + "theoretical_loss": 5.099032565884409, + "tokens_seen": 65404928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007920561685055166, + "loss": 2.4274, + "theoretical_loss": 5.098286284036908, + "tokens_seen": 65470464 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.9917979836463928, + "objective/train/avg_token_score": 0.00820205919444561, + "objective/train/docs_used": 31526, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 2.537433385848999, + "objective/train/original_loss": 2.5585427284240723, + "objective/train/theoretical_loss": 5.097540957774312, + "objective/train/tokens_used": 85996000, + "objective/train/unlikelihood_loss": 0.00952764879912138, + "theoretical_loss": 5.097540957774312, + "tokens_seen": 65536000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007920401203610832, + "loss": 2.449, + "theoretical_loss": 5.097540957774312, + "tokens_seen": 65536000 + }, + { + "epoch": 0.02, + "learning_rate": 0.00079202407221665, + "loss": 2.4276, + "theoretical_loss": 5.096796584919282, + "tokens_seen": 65601536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007920080240722166, + "loss": 2.4449, + "theoretical_loss": 5.096053163301608, + "tokens_seen": 65667072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007919919759277834, + "loss": 2.4961, + "theoretical_loss": 5.095310690758177, + "tokens_seen": 65732608 + }, + { + "epoch": 0.02, + "learning_rate": 0.00079197592778335, + "loss": 2.3658, + "theoretical_loss": 5.094569165132949, + "tokens_seen": 65798144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007919598796389168, + "loss": 2.2968, + "theoretical_loss": 5.09382858427692, + "tokens_seen": 65863680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007919438314944834, + "loss": 2.4134, + "theoretical_loss": 5.0930889460481, + "tokens_seen": 65929216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007919277833500502, + "loss": 2.356, + "theoretical_loss": 5.092350248311471, + "tokens_seen": 65994752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007919117352056168, + "loss": 2.3342, + "theoretical_loss": 5.091612488938971, + "tokens_seen": 66060288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007918956870611836, + "loss": 2.3297, + "theoretical_loss": 5.090875665809454, + "tokens_seen": 66125824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007918796389167502, + "loss": 2.2846, + "theoretical_loss": 5.09013977680867, + "tokens_seen": 66191360 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791863590772317, + "loss": 2.3499, + "theoretical_loss": 5.089404819829225, + "tokens_seen": 66256896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007918475426278836, + "loss": 2.3696, + "theoretical_loss": 5.088670792770563, + "tokens_seen": 66322432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007918314944834504, + "loss": 2.3309, + "theoretical_loss": 5.087937693538927, + "tokens_seen": 66387968 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791815446339017, + "loss": 2.3156, + "theoretical_loss": 5.0872055200473385, + "tokens_seen": 66453504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007917993981945838, + "loss": 2.304, + "theoretical_loss": 5.0864742702155645, + "tokens_seen": 66519040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007917833500501504, + "loss": 2.4346, + "theoretical_loss": 5.0857439419700885, + "tokens_seen": 66584576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007917673019057172, + "loss": 2.3094, + "theoretical_loss": 5.085014533244088, + "tokens_seen": 66650112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007917512537612838, + "loss": 2.3245, + "theoretical_loss": 5.084286041977396, + "tokens_seen": 66715648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007917352056168506, + "loss": 2.3362, + "theoretical_loss": 5.083558466116487, + "tokens_seen": 66781184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007917191574724172, + "loss": 2.2012, + "theoretical_loss": 5.082831803614433, + "tokens_seen": 66846720 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791703109327984, + "loss": 2.3585, + "theoretical_loss": 5.082106052430891, + "tokens_seen": 66912256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007916870611835506, + "loss": 2.295, + "theoretical_loss": 5.081381210532063, + "tokens_seen": 66977792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007916710130391174, + "loss": 2.4048, + "theoretical_loss": 5.08065727589068, + "tokens_seen": 67043328 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791654964894684, + "loss": 2.3257, + "theoretical_loss": 5.079934246485963, + "tokens_seen": 67108864 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007916389167502508, + "loss": 2.5082, + "theoretical_loss": 5.079212120303606, + "tokens_seen": 67174400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007916228686058174, + "loss": 2.2697, + "theoretical_loss": 5.078490895335744, + "tokens_seen": 67239936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007916068204613842, + "loss": 2.3535, + "theoretical_loss": 5.077770569580924, + "tokens_seen": 67305472 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791590772316951, + "loss": 2.2337, + "theoretical_loss": 5.077051141044086, + "tokens_seen": 67371008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007915747241725176, + "loss": 2.2817, + "theoretical_loss": 5.076332607736527, + "tokens_seen": 67436544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007915586760280844, + "loss": 2.3838, + "theoretical_loss": 5.0756149676758815, + "tokens_seen": 67502080 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791542627883651, + "loss": 2.4924, + "theoretical_loss": 5.0748982188860925, + "tokens_seen": 67567616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007915265797392178, + "loss": 2.374, + "theoretical_loss": 5.074182359397387, + "tokens_seen": 67633152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007915105315947844, + "loss": 2.208, + "theoretical_loss": 5.073467387246247, + "tokens_seen": 67698688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007914944834503512, + "loss": 2.3133, + "theoretical_loss": 5.072753300475385, + "tokens_seen": 67764224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007914784353059178, + "loss": 2.3283, + "theoretical_loss": 5.072040097133718, + "tokens_seen": 67829760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007914623871614846, + "loss": 2.2374, + "theoretical_loss": 5.0713277752763455, + "tokens_seen": 67895296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007914463390170512, + "loss": 2.2763, + "theoretical_loss": 5.070616332964518, + "tokens_seen": 67960832 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791430290872618, + "loss": 2.318, + "theoretical_loss": 5.069905768265615, + "tokens_seen": 68026368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007914142427281846, + "loss": 2.2568, + "theoretical_loss": 5.069196079253119, + "tokens_seen": 68091904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007913981945837514, + "loss": 2.1412, + "theoretical_loss": 5.068487264006592, + "tokens_seen": 68157440 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791382146439318, + "loss": 2.2114, + "theoretical_loss": 5.0677793206116455, + "tokens_seen": 68222976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007913660982948848, + "loss": 2.3649, + "theoretical_loss": 5.067072247159923, + "tokens_seen": 68288512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007913500501504514, + "loss": 2.269, + "theoretical_loss": 5.06636604174907, + "tokens_seen": 68354048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007913340020060182, + "loss": 2.3375, + "theoretical_loss": 5.065660702482711, + "tokens_seen": 68419584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007913179538615848, + "loss": 2.2703, + "theoretical_loss": 5.064956227470425, + "tokens_seen": 68485120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007913019057171516, + "loss": 2.3352, + "theoretical_loss": 5.06425261482772, + "tokens_seen": 68550656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007912858575727182, + "loss": 2.36, + "theoretical_loss": 5.063549862676011, + "tokens_seen": 68616192 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791269809428285, + "loss": 2.2466, + "theoretical_loss": 5.062847969142591, + "tokens_seen": 68681728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007912537612838516, + "loss": 2.3454, + "theoretical_loss": 5.062146932360616, + "tokens_seen": 68747264 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.9922741055488586, + "objective/train/avg_token_score": 0.007725887466222048, + "objective/train/docs_used": 32629, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 2.2269110679626465, + "objective/train/original_loss": 2.245553493499756, + "objective/train/theoretical_loss": 5.061446750469072, + "objective/train/tokens_used": 89272800, + "objective/train/unlikelihood_loss": 0.01065373606979847, + "theoretical_loss": 5.061446750469072, + "tokens_seen": 68812800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007912377131394184, + "loss": 2.2873, + "theoretical_loss": 5.061446750469072, + "tokens_seen": 68812800 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791221664994985, + "loss": 2.4085, + "theoretical_loss": 5.060747421612758, + "tokens_seen": 68878336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007912056168505518, + "loss": 2.3773, + "theoretical_loss": 5.060048943942254, + "tokens_seen": 68943872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007911895687061184, + "loss": 2.2431, + "theoretical_loss": 5.059351315613908, + "tokens_seen": 69009408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007911735205616852, + "loss": 2.2255, + "theoretical_loss": 5.058654534789804, + "tokens_seen": 69074944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007911574724172518, + "loss": 2.3293, + "theoretical_loss": 5.057958599637746, + "tokens_seen": 69140480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007911414242728186, + "loss": 2.3258, + "theoretical_loss": 5.057263508331225, + "tokens_seen": 69206016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007911253761283852, + "loss": 2.2629, + "theoretical_loss": 5.0565692590494065, + "tokens_seen": 69271552 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791109327983952, + "loss": 2.1985, + "theoretical_loss": 5.055875849977101, + "tokens_seen": 69337088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007910932798395186, + "loss": 2.4085, + "theoretical_loss": 5.055183279304744, + "tokens_seen": 69402624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007910772316950853, + "loss": 2.3093, + "theoretical_loss": 5.0544915452283705, + "tokens_seen": 69468160 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791061183550652, + "loss": 2.2673, + "theoretical_loss": 5.053800645949597, + "tokens_seen": 69533696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007910451354062187, + "loss": 2.2135, + "theoretical_loss": 5.053110579675593, + "tokens_seen": 69599232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007910290872617854, + "loss": 2.2717, + "theoretical_loss": 5.052421344619064, + "tokens_seen": 69664768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007910130391173521, + "loss": 2.3321, + "theoretical_loss": 5.051732938998227, + "tokens_seen": 69730304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007909969909729188, + "loss": 2.2251, + "theoretical_loss": 5.051045361036787, + "tokens_seen": 69795840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007909809428284855, + "loss": 2.2691, + "theoretical_loss": 5.0503586089639185, + "tokens_seen": 69861376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007909648946840522, + "loss": 2.2498, + "theoretical_loss": 5.04967268101424, + "tokens_seen": 69926912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007909488465396189, + "loss": 2.2659, + "theoretical_loss": 5.048987575427793, + "tokens_seen": 69992448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007909327983951855, + "loss": 2.3234, + "theoretical_loss": 5.048303290450023, + "tokens_seen": 70057984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007909167502507523, + "loss": 2.4077, + "theoretical_loss": 5.047619824331753, + "tokens_seen": 70123520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007909007021063189, + "loss": 2.2737, + "theoretical_loss": 5.046937175329167, + "tokens_seen": 70189056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007908846539618857, + "loss": 2.328, + "theoretical_loss": 5.046255341703786, + "tokens_seen": 70254592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007908686058174523, + "loss": 2.2842, + "theoretical_loss": 5.045574321722445, + "tokens_seen": 70320128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007908525576730191, + "loss": 2.2547, + "theoretical_loss": 5.0448941136572785, + "tokens_seen": 70385664 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007908365095285857, + "loss": 2.3253, + "theoretical_loss": 5.044214715785689, + "tokens_seen": 70451200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007908204613841525, + "loss": 2.2587, + "theoretical_loss": 5.043536126390334, + "tokens_seen": 70516736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007908044132397191, + "loss": 2.1585, + "theoretical_loss": 5.042858343759107, + "tokens_seen": 70582272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007907883650952859, + "loss": 2.2166, + "theoretical_loss": 5.0421813661851065, + "tokens_seen": 70647808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007907723169508525, + "loss": 2.2879, + "theoretical_loss": 5.041505191966625, + "tokens_seen": 70713344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007907562688064193, + "loss": 2.3694, + "theoretical_loss": 5.040829819407123, + "tokens_seen": 70778880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007907402206619859, + "loss": 2.302, + "theoretical_loss": 5.040155246815212, + "tokens_seen": 70844416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007907241725175527, + "loss": 2.4069, + "theoretical_loss": 5.039481472504633, + "tokens_seen": 70909952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007907081243731193, + "loss": 2.3293, + "theoretical_loss": 5.038808494794232, + "tokens_seen": 70975488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007906920762286861, + "loss": 2.1892, + "theoretical_loss": 5.038136312007948, + "tokens_seen": 71041024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007906760280842527, + "loss": 2.2077, + "theoretical_loss": 5.037464922474786, + "tokens_seen": 71106560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007906599799398195, + "loss": 2.2354, + "theoretical_loss": 5.036794324528802, + "tokens_seen": 71172096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007906439317953861, + "loss": 2.2096, + "theoretical_loss": 5.036124516509076, + "tokens_seen": 71237632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007906278836509529, + "loss": 2.3787, + "theoretical_loss": 5.035455496759701, + "tokens_seen": 71303168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007906118355065195, + "loss": 2.3516, + "theoretical_loss": 5.034787263629758, + "tokens_seen": 71368704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007905957873620863, + "loss": 2.3325, + "theoretical_loss": 5.0341198154732965, + "tokens_seen": 71434240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007905797392176529, + "loss": 2.3642, + "theoretical_loss": 5.033453150649318, + "tokens_seen": 71499776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007905636910732197, + "loss": 2.2304, + "theoretical_loss": 5.032787267521752, + "tokens_seen": 71565312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007905476429287863, + "loss": 2.2869, + "theoretical_loss": 5.032122164459443, + "tokens_seen": 71630848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007905315947843531, + "loss": 2.3174, + "theoretical_loss": 5.031457839836122, + "tokens_seen": 71696384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007905155466399197, + "loss": 2.1603, + "theoretical_loss": 5.0307942920304, + "tokens_seen": 71761920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007904994984954865, + "loss": 2.2568, + "theoretical_loss": 5.030131519425737, + "tokens_seen": 71827456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007904834503510531, + "loss": 2.3445, + "theoretical_loss": 5.02946952041043, + "tokens_seen": 71892992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007904674022066199, + "loss": 2.3039, + "theoretical_loss": 5.028808293377589, + "tokens_seen": 71958528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007904513540621865, + "loss": 2.2792, + "theoretical_loss": 5.028147836725127, + "tokens_seen": 72024064 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.9935893416404724, + "objective/train/avg_token_score": 0.0064107258804142475, + "objective/train/docs_used": 33987, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 2.2615766525268555, + "objective/train/original_loss": 2.2775356769561768, + "objective/train/theoretical_loss": 5.0274881488557295, + "objective/train/tokens_used": 92549600, + "objective/train/unlikelihood_loss": 0.008378447964787483, + "theoretical_loss": 5.0274881488557295, + "tokens_seen": 72089600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007904353059177533, + "loss": 2.1349, + "theoretical_loss": 5.0274881488557295, + "tokens_seen": 72089600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007904192577733199, + "loss": 2.2684, + "theoretical_loss": 5.026829228176848, + "tokens_seen": 72155136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007904032096288867, + "loss": 2.2821, + "theoretical_loss": 5.026171073100672, + "tokens_seen": 72220672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007903871614844534, + "loss": 2.2466, + "theoretical_loss": 5.025513682044114, + "tokens_seen": 72286208 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007903711133400201, + "loss": 2.4032, + "theoretical_loss": 5.024857053428793, + "tokens_seen": 72351744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007903550651955868, + "loss": 2.2415, + "theoretical_loss": 5.024201185681015, + "tokens_seen": 72417280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007903390170511535, + "loss": 2.3357, + "theoretical_loss": 5.023546077231752, + "tokens_seen": 72482816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007903229689067202, + "loss": 2.2287, + "theoretical_loss": 5.0228917265166295, + "tokens_seen": 72548352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007903069207622869, + "loss": 2.072, + "theoretical_loss": 5.022238131975905, + "tokens_seen": 72613888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007902908726178537, + "loss": 2.2757, + "theoretical_loss": 5.021585292054446, + "tokens_seen": 72679424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007902748244734203, + "loss": 2.262, + "theoretical_loss": 5.020933205201725, + "tokens_seen": 72744960 + }, + { + "epoch": 0.02, + "learning_rate": 0.000790258776328987, + "loss": 2.3092, + "theoretical_loss": 5.020281869871787, + "tokens_seen": 72810496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007902427281845537, + "loss": 2.2539, + "theoretical_loss": 5.019631284523241, + "tokens_seen": 72876032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007902266800401205, + "loss": 2.1785, + "theoretical_loss": 5.01898144761924, + "tokens_seen": 72941568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007902106318956871, + "loss": 2.1254, + "theoretical_loss": 5.018332357627462, + "tokens_seen": 73007104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007901945837512539, + "loss": 2.2276, + "theoretical_loss": 5.017684013020096, + "tokens_seen": 73072640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007901785356068205, + "loss": 2.3673, + "theoretical_loss": 5.017036412273823, + "tokens_seen": 73138176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007901624874623873, + "loss": 2.2792, + "theoretical_loss": 5.016389553869796, + "tokens_seen": 73203712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007901464393179539, + "loss": 2.279, + "theoretical_loss": 5.0157434362936275, + "tokens_seen": 73269248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007901303911735207, + "loss": 2.2735, + "theoretical_loss": 5.01509805803537, + "tokens_seen": 73334784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007901143430290873, + "loss": 2.2784, + "theoretical_loss": 5.0144534175894995, + "tokens_seen": 73400320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007900982948846541, + "loss": 2.0868, + "theoretical_loss": 5.013809513454898, + "tokens_seen": 73465856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007900822467402207, + "loss": 2.1692, + "theoretical_loss": 5.013166344134838, + "tokens_seen": 73531392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007900661985957875, + "loss": 2.2411, + "theoretical_loss": 5.012523908136963, + "tokens_seen": 73596928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007900501504513541, + "loss": 2.2818, + "theoretical_loss": 5.011882203973277, + "tokens_seen": 73662464 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007900341023069209, + "loss": 2.1751, + "theoretical_loss": 5.011241230160119, + "tokens_seen": 73728000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007900180541624875, + "loss": 2.4035, + "theoretical_loss": 5.0106009852181534, + "tokens_seen": 73793536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007900020060180543, + "loss": 2.3608, + "theoretical_loss": 5.009961467672353, + "tokens_seen": 73859072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007899859578736209, + "loss": 2.2726, + "theoretical_loss": 5.009322676051976, + "tokens_seen": 73924608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007899699097291877, + "loss": 2.2685, + "theoretical_loss": 5.008684608890562, + "tokens_seen": 73990144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007899538615847543, + "loss": 2.318, + "theoretical_loss": 5.008047264725901, + "tokens_seen": 74055680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007899378134403211, + "loss": 2.1789, + "theoretical_loss": 5.007410642100032, + "tokens_seen": 74121216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007899217652958877, + "loss": 2.4507, + "theoretical_loss": 5.006774739559216, + "tokens_seen": 74186752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007899057171514545, + "loss": 2.3053, + "theoretical_loss": 5.00613955565392, + "tokens_seen": 74252288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007898896690070211, + "loss": 2.1406, + "theoretical_loss": 5.005505088938815, + "tokens_seen": 74317824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007898736208625879, + "loss": 2.3079, + "theoretical_loss": 5.004871337972742, + "tokens_seen": 74383360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007898575727181545, + "loss": 2.2163, + "theoretical_loss": 5.004238301318706, + "tokens_seen": 74448896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007898415245737213, + "loss": 2.2546, + "theoretical_loss": 5.003605977543861, + "tokens_seen": 74514432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007898254764292879, + "loss": 2.2018, + "theoretical_loss": 5.00297436521949, + "tokens_seen": 74579968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007898094282848547, + "loss": 2.1693, + "theoretical_loss": 5.002343462920992, + "tokens_seen": 74645504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007897933801404213, + "loss": 2.354, + "theoretical_loss": 5.001713269227867, + "tokens_seen": 74711040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007897773319959881, + "loss": 2.2439, + "theoretical_loss": 5.0010837827237005, + "tokens_seen": 74776576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007897612838515547, + "loss": 2.258, + "theoretical_loss": 5.000455001996143, + "tokens_seen": 74842112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007897452357071215, + "loss": 2.2829, + "theoretical_loss": 4.999826925636905, + "tokens_seen": 74907648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007897291875626881, + "loss": 2.2925, + "theoretical_loss": 4.999199552241732, + "tokens_seen": 74973184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007897131394182549, + "loss": 2.3946, + "theoretical_loss": 4.998572880410396, + "tokens_seen": 75038720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007896970912738215, + "loss": 2.1599, + "theoretical_loss": 4.997946908746677, + "tokens_seen": 75104256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007896810431293882, + "loss": 2.2252, + "theoretical_loss": 4.997321635858347, + "tokens_seen": 75169792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007896649949849549, + "loss": 2.1829, + "theoretical_loss": 4.9966970603571585, + "tokens_seen": 75235328 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007896489468405216, + "loss": 2.3101, + "theoretical_loss": 4.996073180858831, + "tokens_seen": 75300864 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.9930024147033691, + "objective/train/avg_token_score": 0.006997568067163229, + "objective/train/docs_used": 35047, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 2.2650675773620605, + "objective/train/original_loss": 2.2807817459106445, + "objective/train/theoretical_loss": 4.995449995983028, + "objective/train/tokens_used": 95826400, + "objective/train/unlikelihood_loss": 0.010417983867228031, + "theoretical_loss": 4.995449995983028, + "tokens_seen": 75366400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007896328986960883, + "loss": 2.2765, + "theoretical_loss": 4.995449995983028, + "tokens_seen": 75366400 + }, + { + "epoch": 0.02, + "learning_rate": 0.000789616850551655, + "loss": 2.25, + "theoretical_loss": 4.994827504353353, + "tokens_seen": 75431936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007896008024072217, + "loss": 2.2747, + "theoretical_loss": 4.994205704597325, + "tokens_seen": 75497472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007895847542627884, + "loss": 2.1387, + "theoretical_loss": 4.993584595346373, + "tokens_seen": 75563008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007895687061183551, + "loss": 2.3114, + "theoretical_loss": 4.992964175235814, + "tokens_seen": 75628544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007895526579739218, + "loss": 2.301, + "theoretical_loss": 4.992344442904843, + "tokens_seen": 75694080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007895366098294884, + "loss": 2.1237, + "theoretical_loss": 4.991725396996515, + "tokens_seen": 75759616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007895205616850552, + "loss": 2.1103, + "theoretical_loss": 4.991107036157738, + "tokens_seen": 75825152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007895045135406218, + "loss": 2.3722, + "theoretical_loss": 4.990489359039248, + "tokens_seen": 75890688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007894884653961886, + "loss": 2.3342, + "theoretical_loss": 4.989872364295605, + "tokens_seen": 75956224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007894724172517552, + "loss": 2.2012, + "theoretical_loss": 4.989256050585174, + "tokens_seen": 76021760 + }, + { + "epoch": 0.02, + "learning_rate": 0.000789456369107322, + "loss": 2.3569, + "theoretical_loss": 4.9886404165701075, + "tokens_seen": 76087296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007894403209628886, + "loss": 2.339, + "theoretical_loss": 4.98802546091634, + "tokens_seen": 76152832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007894242728184554, + "loss": 2.2623, + "theoretical_loss": 4.9874111822935685, + "tokens_seen": 76218368 + }, + { + "epoch": 0.02, + "learning_rate": 0.000789408224674022, + "loss": 2.2928, + "theoretical_loss": 4.986797579375239, + "tokens_seen": 76283904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007893921765295888, + "loss": 2.2953, + "theoretical_loss": 4.986184650838532, + "tokens_seen": 76349440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007893761283851554, + "loss": 2.2088, + "theoretical_loss": 4.985572395364354, + "tokens_seen": 76414976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007893600802407222, + "loss": 2.2142, + "theoretical_loss": 4.98496081163732, + "tokens_seen": 76480512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007893440320962888, + "loss": 2.1894, + "theoretical_loss": 4.984349898345735, + "tokens_seen": 76546048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007893279839518556, + "loss": 2.1833, + "theoretical_loss": 4.98373965418159, + "tokens_seen": 76611584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007893119358074222, + "loss": 2.2505, + "theoretical_loss": 4.983130077840543, + "tokens_seen": 76677120 + }, + { + "epoch": 0.02, + "learning_rate": 0.000789295887662989, + "loss": 2.1998, + "theoretical_loss": 4.982521168021906, + "tokens_seen": 76742656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007892798395185556, + "loss": 2.1464, + "theoretical_loss": 4.981912923428634, + "tokens_seen": 76808192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007892637913741224, + "loss": 2.2322, + "theoretical_loss": 4.981305342767307, + "tokens_seen": 76873728 + }, + { + "epoch": 0.02, + "learning_rate": 0.000789247743229689, + "loss": 2.2903, + "theoretical_loss": 4.980698424748122, + "tokens_seen": 76939264 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007892316950852558, + "loss": 2.1918, + "theoretical_loss": 4.980092168084878, + "tokens_seen": 77004800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007892156469408224, + "loss": 2.2425, + "theoretical_loss": 4.97948657149496, + "tokens_seen": 77070336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007891995987963892, + "loss": 2.2676, + "theoretical_loss": 4.978881633699331, + "tokens_seen": 77135872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007891835506519559, + "loss": 2.1481, + "theoretical_loss": 4.9782773534225155, + "tokens_seen": 77201408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007891675025075226, + "loss": 2.2275, + "theoretical_loss": 4.977673729392586, + "tokens_seen": 77266944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007891514543630893, + "loss": 2.1834, + "theoretical_loss": 4.977070760341153, + "tokens_seen": 77332480 + }, + { + "epoch": 0.02, + "learning_rate": 0.000789135406218656, + "loss": 2.2274, + "theoretical_loss": 4.976468445003352, + "tokens_seen": 77398016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007891193580742227, + "loss": 2.2408, + "theoretical_loss": 4.975866782117828, + "tokens_seen": 77463552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007891033099297894, + "loss": 2.154, + "theoretical_loss": 4.975265770426724, + "tokens_seen": 77529088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007890872617853561, + "loss": 2.2512, + "theoretical_loss": 4.974665408675667, + "tokens_seen": 77594624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007890712136409228, + "loss": 2.2258, + "theoretical_loss": 4.974065695613763, + "tokens_seen": 77660160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007890551654964895, + "loss": 2.1901, + "theoretical_loss": 4.973466629993574, + "tokens_seen": 77725696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007890391173520562, + "loss": 2.1046, + "theoretical_loss": 4.972868210571109, + "tokens_seen": 77791232 + }, + { + "epoch": 0.02, + "learning_rate": 0.000789023069207623, + "loss": 2.4103, + "theoretical_loss": 4.972270436105815, + "tokens_seen": 77856768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007890070210631896, + "loss": 2.1735, + "theoretical_loss": 4.971673305360563, + "tokens_seen": 77922304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007889909729187563, + "loss": 2.1705, + "theoretical_loss": 4.971076817101633, + "tokens_seen": 77987840 + }, + { + "epoch": 0.02, + "learning_rate": 0.000788974924774323, + "loss": 2.1501, + "theoretical_loss": 4.970480970098703, + "tokens_seen": 78053376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007889588766298897, + "loss": 2.0908, + "theoretical_loss": 4.96988576312484, + "tokens_seen": 78118912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007889428284854564, + "loss": 2.2714, + "theoretical_loss": 4.969291194956485, + "tokens_seen": 78184448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007889267803410232, + "loss": 2.2267, + "theoretical_loss": 4.968697264373438, + "tokens_seen": 78249984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007889107321965898, + "loss": 2.0777, + "theoretical_loss": 4.968103970158854, + "tokens_seen": 78315520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007888946840521566, + "loss": 2.2308, + "theoretical_loss": 4.967511311099223, + "tokens_seen": 78381056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007888786359077232, + "loss": 2.083, + "theoretical_loss": 4.96691928598436, + "tokens_seen": 78446592 + }, + { + "epoch": 0.02, + "learning_rate": 0.00078886258776329, + "loss": 2.2209, + "theoretical_loss": 4.966327893607398, + "tokens_seen": 78512128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007888465396188566, + "loss": 2.2105, + "theoretical_loss": 4.96573713276477, + "tokens_seen": 78577664 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.9910324811935425, + "objective/train/avg_token_score": 0.008967493660748005, + "objective/train/docs_used": 36308, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 2.2980964183807373, + "objective/train/original_loss": 2.316588878631592, + "objective/train/theoretical_loss": 4.965147002256201, + "objective/train/tokens_used": 99103200, + "objective/train/unlikelihood_loss": 0.013726599514484406, + "theoretical_loss": 4.965147002256201, + "tokens_seen": 78643200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007888304914744234, + "loss": 2.3012, + "theoretical_loss": 4.965147002256201, + "tokens_seen": 78643200 + }, + { + "epoch": 0.02, + "learning_rate": 0.00078881444332999, + "loss": 2.2162, + "theoretical_loss": 4.964557500884695, + "tokens_seen": 78708736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007887983951855568, + "loss": 2.3191, + "theoretical_loss": 4.963968627456523, + "tokens_seen": 78774272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007887823470411234, + "loss": 2.0846, + "theoretical_loss": 4.96338038078121, + "tokens_seen": 78839808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007887662988966902, + "loss": 2.1655, + "theoretical_loss": 4.962792759671528, + "tokens_seen": 78905344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007887502507522568, + "loss": 2.1858, + "theoretical_loss": 4.962205762943478, + "tokens_seen": 78970880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007887342026078236, + "loss": 2.1178, + "theoretical_loss": 4.961619389416288, + "tokens_seen": 79036416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007887181544633902, + "loss": 2.117, + "theoretical_loss": 4.96103363791239, + "tokens_seen": 79101952 + }, + { + "epoch": 0.02, + "learning_rate": 0.000788702106318957, + "loss": 2.2244, + "theoretical_loss": 4.960448507257416, + "tokens_seen": 79167488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007886860581745236, + "loss": 2.2298, + "theoretical_loss": 4.959863996280188, + "tokens_seen": 79233024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007886700100300904, + "loss": 2.3418, + "theoretical_loss": 4.959280103812697, + "tokens_seen": 79298560 + }, + { + "epoch": 0.02, + "learning_rate": 0.000788653961885657, + "loss": 2.186, + "theoretical_loss": 4.958696828690105, + "tokens_seen": 79364096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007886379137412238, + "loss": 2.189, + "theoretical_loss": 4.958114169750726, + "tokens_seen": 79429632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007886218655967904, + "loss": 2.202, + "theoretical_loss": 4.957532125836012, + "tokens_seen": 79495168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007886058174523572, + "loss": 2.3011, + "theoretical_loss": 4.956950695790551, + "tokens_seen": 79560704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007885897693079238, + "loss": 2.2044, + "theoretical_loss": 4.95636987846205, + "tokens_seen": 79626240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007885737211634906, + "loss": 2.2323, + "theoretical_loss": 4.955789672701323, + "tokens_seen": 79691776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007885576730190572, + "loss": 2.2329, + "theoretical_loss": 4.955210077362285, + "tokens_seen": 79757312 + }, + { + "epoch": 0.02, + "learning_rate": 0.000788541624874624, + "loss": 2.3144, + "theoretical_loss": 4.954631091301934, + "tokens_seen": 79822848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007885255767301906, + "loss": 2.1484, + "theoretical_loss": 4.954052713380349, + "tokens_seen": 79888384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007885095285857574, + "loss": 2.109, + "theoretical_loss": 4.953474942460673, + "tokens_seen": 79953920 + }, + { + "epoch": 0.02, + "learning_rate": 0.000788493480441324, + "loss": 2.1564, + "theoretical_loss": 4.952897777409104, + "tokens_seen": 80019456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007884774322968908, + "loss": 2.1986, + "theoretical_loss": 4.952321217094882, + "tokens_seen": 80084992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007884613841524574, + "loss": 2.1007, + "theoretical_loss": 4.951745260390284, + "tokens_seen": 80150528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007884453360080242, + "loss": 2.1262, + "theoretical_loss": 4.951169906170607, + "tokens_seen": 80216064 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007884292878635908, + "loss": 2.2109, + "theoretical_loss": 4.950595153314161, + "tokens_seen": 80281600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007884132397191576, + "loss": 2.2132, + "theoretical_loss": 4.950021000702261, + "tokens_seen": 80347136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007883971915747242, + "loss": 2.1471, + "theoretical_loss": 4.949447447219208, + "tokens_seen": 80412672 + }, + { + "epoch": 0.02, + "learning_rate": 0.000788381143430291, + "loss": 2.1627, + "theoretical_loss": 4.948874491752287, + "tokens_seen": 80478208 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007883650952858576, + "loss": 2.072, + "theoretical_loss": 4.9483021331917545, + "tokens_seen": 80543744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007883490471414244, + "loss": 2.0755, + "theoretical_loss": 4.9477303704308255, + "tokens_seen": 80609280 + }, + { + "epoch": 0.02, + "learning_rate": 0.000788332998996991, + "loss": 2.1808, + "theoretical_loss": 4.947159202365664, + "tokens_seen": 80674816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007883169508525578, + "loss": 2.1547, + "theoretical_loss": 4.946588627895375, + "tokens_seen": 80740352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007883009027081244, + "loss": 2.1435, + "theoretical_loss": 4.946018645921994, + "tokens_seen": 80805888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007882848545636911, + "loss": 2.2508, + "theoretical_loss": 4.945449255350473, + "tokens_seen": 80871424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007882688064192578, + "loss": 2.2305, + "theoretical_loss": 4.944880455088677, + "tokens_seen": 80936960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007882527582748245, + "loss": 2.2551, + "theoretical_loss": 4.944312244047365, + "tokens_seen": 81002496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007882367101303912, + "loss": 2.1098, + "theoretical_loss": 4.943744621140189, + "tokens_seen": 81068032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007882206619859579, + "loss": 2.0886, + "theoretical_loss": 4.94317758528368, + "tokens_seen": 81133568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007882046138415246, + "loss": 2.2465, + "theoretical_loss": 4.942611135397234, + "tokens_seen": 81199104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007881885656970913, + "loss": 2.2651, + "theoretical_loss": 4.942045270403112, + "tokens_seen": 81264640 + }, + { + "epoch": 0.02, + "learning_rate": 0.000788172517552658, + "loss": 2.1852, + "theoretical_loss": 4.941479989226421, + "tokens_seen": 81330176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007881564694082247, + "loss": 2.2918, + "theoretical_loss": 4.940915290795106, + "tokens_seen": 81395712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007881404212637913, + "loss": 2.2126, + "theoretical_loss": 4.940351174039947, + "tokens_seen": 81461248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007881243731193581, + "loss": 2.2365, + "theoretical_loss": 4.939787637894537, + "tokens_seen": 81526784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007881083249749247, + "loss": 2.0599, + "theoretical_loss": 4.939224681295286, + "tokens_seen": 81592320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007880922768304915, + "loss": 2.2112, + "theoretical_loss": 4.938662303181399, + "tokens_seen": 81657856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007880762286860581, + "loss": 2.3535, + "theoretical_loss": 4.938100502494878, + "tokens_seen": 81723392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007880601805416249, + "loss": 2.2761, + "theoretical_loss": 4.9375392781805, + "tokens_seen": 81788928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007880441323971915, + "loss": 2.1805, + "theoretical_loss": 4.936978629185818, + "tokens_seen": 81854464 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.9886553287506104, + "objective/train/avg_token_score": 0.011344674974679947, + "objective/train/docs_used": 37569, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 2.1277360916137695, + "objective/train/original_loss": 2.1515581607818604, + "objective/train/theoretical_loss": 4.936418554461147, + "objective/train/tokens_used": 102380000, + "objective/train/unlikelihood_loss": 0.01817827858030796, + "theoretical_loss": 4.936418554461147, + "tokens_seen": 81920000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007880280842527583, + "loss": 2.1976, + "theoretical_loss": 4.936418554461147, + "tokens_seen": 81920000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007880120361083249, + "loss": 2.2223, + "theoretical_loss": 4.935859052959554, + "tokens_seen": 81985536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007879959879638917, + "loss": 2.0608, + "theoretical_loss": 4.935300123636851, + "tokens_seen": 82051072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007879799398194584, + "loss": 2.2098, + "theoretical_loss": 4.93474176545158, + "tokens_seen": 82116608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007879638916750251, + "loss": 2.1949, + "theoretical_loss": 4.934183977365015, + "tokens_seen": 82182144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007879478435305918, + "loss": 2.0944, + "theoretical_loss": 4.9336267583411395, + "tokens_seen": 82247680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007879317953861585, + "loss": 2.0858, + "theoretical_loss": 4.933070107346644, + "tokens_seen": 82313216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007879157472417252, + "loss": 2.1357, + "theoretical_loss": 4.932514023350921, + "tokens_seen": 82378752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007878996990972919, + "loss": 2.2423, + "theoretical_loss": 4.931958505326042, + "tokens_seen": 82444288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007878836509528586, + "loss": 2.1366, + "theoretical_loss": 4.931403552246769, + "tokens_seen": 82509824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007878676028084253, + "loss": 2.1863, + "theoretical_loss": 4.930849163090521, + "tokens_seen": 82575360 + }, + { + "epoch": 0.03, + "learning_rate": 0.000787851554663992, + "loss": 2.1683, + "theoretical_loss": 4.930295336837387, + "tokens_seen": 82640896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007878355065195587, + "loss": 2.1061, + "theoretical_loss": 4.9297420724701055, + "tokens_seen": 82706432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007878194583751254, + "loss": 2.1697, + "theoretical_loss": 4.929189368974054, + "tokens_seen": 82771968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007878034102306921, + "loss": 2.114, + "theoretical_loss": 4.928637225337246, + "tokens_seen": 82837504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007877873620862588, + "loss": 2.2332, + "theoretical_loss": 4.9280856405503215, + "tokens_seen": 82903040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007877713139418255, + "loss": 2.2597, + "theoretical_loss": 4.927534613606534, + "tokens_seen": 82968576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007877552657973922, + "loss": 2.0664, + "theoretical_loss": 4.926984143501745, + "tokens_seen": 83034112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007877392176529589, + "loss": 2.1709, + "theoretical_loss": 4.926434229234411, + "tokens_seen": 83099648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007877231695085256, + "loss": 2.1586, + "theoretical_loss": 4.925884869805586, + "tokens_seen": 83165184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007877071213640923, + "loss": 2.0936, + "theoretical_loss": 4.925336064218893, + "tokens_seen": 83230720 + }, + { + "epoch": 0.03, + "learning_rate": 0.000787691073219659, + "loss": 2.2232, + "theoretical_loss": 4.924787811480538, + "tokens_seen": 83296256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007876750250752257, + "loss": 2.2086, + "theoretical_loss": 4.9242401105992855, + "tokens_seen": 83361792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007876589769307924, + "loss": 2.2164, + "theoretical_loss": 4.923692960586452, + "tokens_seen": 83427328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007876429287863591, + "loss": 1.962, + "theoretical_loss": 4.923146360455906, + "tokens_seen": 83492864 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007876268806419258, + "loss": 2.0873, + "theoretical_loss": 4.92260030922405, + "tokens_seen": 83558400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007876108324974925, + "loss": 2.3114, + "theoretical_loss": 4.922054805909816, + "tokens_seen": 83623936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007875947843530592, + "loss": 2.1315, + "theoretical_loss": 4.921509849534658, + "tokens_seen": 83689472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007875787362086259, + "loss": 2.1392, + "theoretical_loss": 4.920965439122538, + "tokens_seen": 83755008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007875626880641927, + "loss": 2.1261, + "theoretical_loss": 4.9204215736999295, + "tokens_seen": 83820544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007875466399197593, + "loss": 2.09, + "theoretical_loss": 4.919878252295793, + "tokens_seen": 83886080 + }, + { + "epoch": 0.03, + "learning_rate": 0.000787530591775326, + "loss": 2.1917, + "theoretical_loss": 4.919335473941581, + "tokens_seen": 83951616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007875145436308927, + "loss": 2.2511, + "theoretical_loss": 4.918793237671224, + "tokens_seen": 84017152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007874984954864595, + "loss": 2.1743, + "theoretical_loss": 4.918251542521121, + "tokens_seen": 84082688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007874824473420261, + "loss": 2.2175, + "theoretical_loss": 4.9177103875301365, + "tokens_seen": 84148224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007874663991975929, + "loss": 2.1381, + "theoretical_loss": 4.917169771739586, + "tokens_seen": 84213760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007874503510531595, + "loss": 2.0871, + "theoretical_loss": 4.916629694193233, + "tokens_seen": 84279296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007874343029087263, + "loss": 2.1573, + "theoretical_loss": 4.916090153937276, + "tokens_seen": 84344832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007874182547642929, + "loss": 2.2751, + "theoretical_loss": 4.915551150020346, + "tokens_seen": 84410368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007874022066198597, + "loss": 2.3142, + "theoretical_loss": 4.915012681493495, + "tokens_seen": 84475904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007873861584754263, + "loss": 2.2333, + "theoretical_loss": 4.914474747410187, + "tokens_seen": 84541440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007873701103309931, + "loss": 2.056, + "theoretical_loss": 4.913937346826293, + "tokens_seen": 84606976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007873540621865597, + "loss": 2.1725, + "theoretical_loss": 4.91340047880008, + "tokens_seen": 84672512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007873380140421265, + "loss": 2.1363, + "theoretical_loss": 4.9128641423922055, + "tokens_seen": 84738048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007873219658976931, + "loss": 2.1384, + "theoretical_loss": 4.912328336665711, + "tokens_seen": 84803584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007873059177532599, + "loss": 2.1109, + "theoretical_loss": 4.911793060686007, + "tokens_seen": 84869120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007872898696088265, + "loss": 2.2359, + "theoretical_loss": 4.9112583135208725, + "tokens_seen": 84934656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007872738214643933, + "loss": 2.2505, + "theoretical_loss": 4.910724094240447, + "tokens_seen": 85000192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007872577733199599, + "loss": 2.1322, + "theoretical_loss": 4.910190401917214, + "tokens_seen": 85065728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007872417251755267, + "loss": 2.2587, + "theoretical_loss": 4.909657235626009, + "tokens_seen": 85131264 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.9955958127975464, + "objective/train/avg_token_score": 0.004404211882501841, + "objective/train/docs_used": 38678, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 2.164421319961548, + "objective/train/original_loss": 2.1756844520568848, + "objective/train/theoretical_loss": 4.909124594443993, + "objective/train/tokens_used": 105656800, + "objective/train/unlikelihood_loss": 0.005208347458392382, + "theoretical_loss": 4.909124594443993, + "tokens_seen": 85196800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007872256770310933, + "loss": 2.1698, + "theoretical_loss": 4.909124594443993, + "tokens_seen": 85196800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007872096288866601, + "loss": 2.1162, + "theoretical_loss": 4.908592477450659, + "tokens_seen": 85262336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007871935807422267, + "loss": 2.1879, + "theoretical_loss": 4.908060883727821, + "tokens_seen": 85327872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007871775325977935, + "loss": 2.1207, + "theoretical_loss": 4.9075298123596, + "tokens_seen": 85393408 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007871614844533601, + "loss": 2.063, + "theoretical_loss": 4.906999262432427, + "tokens_seen": 85458944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007871454363089269, + "loss": 2.1036, + "theoretical_loss": 4.906469233035025, + "tokens_seen": 85524480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007871293881644935, + "loss": 2.2808, + "theoretical_loss": 4.905939723258408, + "tokens_seen": 85590016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007871133400200603, + "loss": 2.1937, + "theoretical_loss": 4.905410732195875, + "tokens_seen": 85655552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007870972918756269, + "loss": 2.1843, + "theoretical_loss": 4.904882258942994, + "tokens_seen": 85721088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007870812437311937, + "loss": 2.0488, + "theoretical_loss": 4.904354302597603, + "tokens_seen": 85786624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007870651955867603, + "loss": 2.2286, + "theoretical_loss": 4.903826862259798, + "tokens_seen": 85852160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007870491474423271, + "loss": 2.1714, + "theoretical_loss": 4.90329993703193, + "tokens_seen": 85917696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007870330992978937, + "loss": 2.1496, + "theoretical_loss": 4.902773526018591, + "tokens_seen": 85983232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007870170511534605, + "loss": 2.1289, + "theoretical_loss": 4.902247628326613, + "tokens_seen": 86048768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007870010030090271, + "loss": 2.2067, + "theoretical_loss": 4.9017222430650556, + "tokens_seen": 86114304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007869849548645939, + "loss": 2.1971, + "theoretical_loss": 4.901197369345205, + "tokens_seen": 86179840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007869689067201605, + "loss": 2.0306, + "theoretical_loss": 4.9006730062805595, + "tokens_seen": 86245376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007869528585757273, + "loss": 2.1751, + "theoretical_loss": 4.9001491529868275, + "tokens_seen": 86310912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007869368104312939, + "loss": 2.0459, + "theoretical_loss": 4.899625808581918, + "tokens_seen": 86376448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007869207622868607, + "loss": 2.2633, + "theoretical_loss": 4.899102972185936, + "tokens_seen": 86441984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007869047141424273, + "loss": 2.0437, + "theoretical_loss": 4.898580642921171, + "tokens_seen": 86507520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007868886659979941, + "loss": 2.1046, + "theoretical_loss": 4.898058819912093, + "tokens_seen": 86573056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007868726178535607, + "loss": 2.2374, + "theoretical_loss": 4.897537502285347, + "tokens_seen": 86638592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007868565697091274, + "loss": 2.2583, + "theoretical_loss": 4.897016689169741, + "tokens_seen": 86704128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007868405215646941, + "loss": 2.0575, + "theoretical_loss": 4.896496379696243, + "tokens_seen": 86769664 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007868244734202608, + "loss": 2.1603, + "theoretical_loss": 4.895976572997973, + "tokens_seen": 86835200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007868084252758275, + "loss": 2.1153, + "theoretical_loss": 4.895457268210196, + "tokens_seen": 86900736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007867923771313942, + "loss": 2.1502, + "theoretical_loss": 4.8949384644703136, + "tokens_seen": 86966272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007867763289869609, + "loss": 2.0912, + "theoretical_loss": 4.894420160917861, + "tokens_seen": 87031808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007867602808425276, + "loss": 2.2543, + "theoretical_loss": 4.893902356694495, + "tokens_seen": 87097344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007867442326980943, + "loss": 2.0473, + "theoretical_loss": 4.8933850509439925, + "tokens_seen": 87162880 + }, + { + "epoch": 0.03, + "learning_rate": 0.000786728184553661, + "loss": 2.1747, + "theoretical_loss": 4.89286824281224, + "tokens_seen": 87228416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007867121364092277, + "loss": 2.0734, + "theoretical_loss": 4.892351931447224, + "tokens_seen": 87293952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007866960882647944, + "loss": 2.1326, + "theoretical_loss": 4.891836115999034, + "tokens_seen": 87359488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007866800401203611, + "loss": 2.0657, + "theoretical_loss": 4.891320795619847, + "tokens_seen": 87425024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007866639919759278, + "loss": 2.0946, + "theoretical_loss": 4.890805969463923, + "tokens_seen": 87490560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007866479438314945, + "loss": 2.1588, + "theoretical_loss": 4.8902916366876, + "tokens_seen": 87556096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007866318956870612, + "loss": 2.0818, + "theoretical_loss": 4.889777796449285, + "tokens_seen": 87621632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007866158475426279, + "loss": 2.0813, + "theoretical_loss": 4.889264447909451, + "tokens_seen": 87687168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007865997993981946, + "loss": 2.0959, + "theoretical_loss": 4.888751590230626, + "tokens_seen": 87752704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007865837512537613, + "loss": 2.1049, + "theoretical_loss": 4.888239222577389, + "tokens_seen": 87818240 + }, + { + "epoch": 0.03, + "learning_rate": 0.000786567703109328, + "loss": 2.209, + "theoretical_loss": 4.887727344116364, + "tokens_seen": 87883776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007865516549648947, + "loss": 2.1424, + "theoretical_loss": 4.8872159540162095, + "tokens_seen": 87949312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007865356068204614, + "loss": 2.148, + "theoretical_loss": 4.88670505144762, + "tokens_seen": 88014848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007865195586760281, + "loss": 2.1299, + "theoretical_loss": 4.8861946355833075, + "tokens_seen": 88080384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007865035105315948, + "loss": 2.1663, + "theoretical_loss": 4.885684705598009, + "tokens_seen": 88145920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007864874623871615, + "loss": 2.111, + "theoretical_loss": 4.885175260668468, + "tokens_seen": 88211456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007864714142427282, + "loss": 2.0947, + "theoretical_loss": 4.884666299973436, + "tokens_seen": 88276992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007864553660982949, + "loss": 2.2246, + "theoretical_loss": 4.884157822693663, + "tokens_seen": 88342528 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007864393179538616, + "loss": 2.0389, + "theoretical_loss": 4.88364982801189, + "tokens_seen": 88408064 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.9934666752815247, + "objective/train/avg_token_score": 0.006533364299684763, + "objective/train/docs_used": 39491, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 2.029850721359253, + "objective/train/original_loss": 2.0440855026245117, + "objective/train/theoretical_loss": 4.8831423151128455, + "objective/train/tokens_used": 108933600, + "objective/train/unlikelihood_loss": 0.012611349113285542, + "theoretical_loss": 4.8831423151128455, + "tokens_seen": 88473600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007864232698094283, + "loss": 2.1312, + "theoretical_loss": 4.8831423151128455, + "tokens_seen": 88473600 + }, + { + "epoch": 0.03, + "learning_rate": 0.000786407221664995, + "loss": 2.119, + "theoretical_loss": 4.8826352831832365, + "tokens_seen": 88539136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007863911735205617, + "loss": 2.1616, + "theoretical_loss": 4.882128731411745, + "tokens_seen": 88604672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007863751253761284, + "loss": 2.1898, + "theoretical_loss": 4.881622658989022, + "tokens_seen": 88670208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007863590772316951, + "loss": 2.1058, + "theoretical_loss": 4.881117065107675, + "tokens_seen": 88735744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007863430290872618, + "loss": 2.0423, + "theoretical_loss": 4.880611948962271, + "tokens_seen": 88801280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007863269809428285, + "loss": 2.1765, + "theoretical_loss": 4.880107309749322, + "tokens_seen": 88866816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007863109327983952, + "loss": 2.1499, + "theoretical_loss": 4.879603146667284, + "tokens_seen": 88932352 + }, + { + "epoch": 0.03, + "learning_rate": 0.000786294884653962, + "loss": 2.1527, + "theoretical_loss": 4.879099458916553, + "tokens_seen": 88997888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007862788365095286, + "loss": 2.0532, + "theoretical_loss": 4.878596245699449, + "tokens_seen": 89063424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007862627883650953, + "loss": 2.2642, + "theoretical_loss": 4.878093506220221, + "tokens_seen": 89128960 + }, + { + "epoch": 0.03, + "learning_rate": 0.000786246740220662, + "loss": 2.1951, + "theoretical_loss": 4.877591239685035, + "tokens_seen": 89194496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007862306920762287, + "loss": 2.0443, + "theoretical_loss": 4.877089445301969, + "tokens_seen": 89260032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007862146439317954, + "loss": 2.0163, + "theoretical_loss": 4.876588122281007, + "tokens_seen": 89325568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007861985957873622, + "loss": 2.1407, + "theoretical_loss": 4.876087269834034, + "tokens_seen": 89391104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007861825476429288, + "loss": 2.123, + "theoretical_loss": 4.875586887174828, + "tokens_seen": 89456640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007861664994984956, + "loss": 2.1566, + "theoretical_loss": 4.875086973519058, + "tokens_seen": 89522176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007861504513540622, + "loss": 2.1467, + "theoretical_loss": 4.874587528084271, + "tokens_seen": 89587712 + }, + { + "epoch": 0.03, + "learning_rate": 0.000786134403209629, + "loss": 2.1313, + "theoretical_loss": 4.874088550089896, + "tokens_seen": 89653248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007861183550651956, + "loss": 2.1495, + "theoretical_loss": 4.873590038757229, + "tokens_seen": 89718784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007861023069207624, + "loss": 1.9776, + "theoretical_loss": 4.87309199330943, + "tokens_seen": 89784320 + }, + { + "epoch": 0.03, + "learning_rate": 0.000786086258776329, + "loss": 1.9724, + "theoretical_loss": 4.872594412971521, + "tokens_seen": 89849856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007860702106318958, + "loss": 2.151, + "theoretical_loss": 4.872097296970376, + "tokens_seen": 89915392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007860541624874624, + "loss": 2.1734, + "theoretical_loss": 4.871600644534717, + "tokens_seen": 89980928 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007860381143430292, + "loss": 2.093, + "theoretical_loss": 4.871104454895106, + "tokens_seen": 90046464 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007860220661985958, + "loss": 2.1008, + "theoretical_loss": 4.870608727283942, + "tokens_seen": 90112000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007860060180541626, + "loss": 2.3905, + "theoretical_loss": 4.870113460935455, + "tokens_seen": 90177536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007859899699097292, + "loss": 2.0847, + "theoretical_loss": 4.869618655085697, + "tokens_seen": 90243072 + }, + { + "epoch": 0.03, + "learning_rate": 0.000785973921765296, + "loss": 2.0831, + "theoretical_loss": 4.869124308972543, + "tokens_seen": 90308608 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007859578736208626, + "loss": 2.2002, + "theoretical_loss": 4.868630421835677, + "tokens_seen": 90374144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007859418254764294, + "loss": 2.1868, + "theoretical_loss": 4.868136992916593, + "tokens_seen": 90439680 + }, + { + "epoch": 0.03, + "learning_rate": 0.000785925777331996, + "loss": 2.1773, + "theoretical_loss": 4.867644021458585, + "tokens_seen": 90505216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007859097291875628, + "loss": 1.9958, + "theoretical_loss": 4.8671515067067475, + "tokens_seen": 90570752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007858936810431294, + "loss": 2.2187, + "theoretical_loss": 4.866659447907962, + "tokens_seen": 90636288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007858776328986962, + "loss": 2.026, + "theoretical_loss": 4.866167844310896, + "tokens_seen": 90701824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007858615847542628, + "loss": 2.0685, + "theoretical_loss": 4.865676695165996, + "tokens_seen": 90767360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007858455366098296, + "loss": 2.0523, + "theoretical_loss": 4.865185999725487, + "tokens_seen": 90832896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007858294884653962, + "loss": 2.1554, + "theoretical_loss": 4.864695757243359, + "tokens_seen": 90898432 + }, + { + "epoch": 0.03, + "learning_rate": 0.000785813440320963, + "loss": 2.2206, + "theoretical_loss": 4.864205966975364, + "tokens_seen": 90963968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007857973921765296, + "loss": 2.0449, + "theoretical_loss": 4.8637166281790165, + "tokens_seen": 91029504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007857813440320964, + "loss": 2.1682, + "theoretical_loss": 4.86322774011358, + "tokens_seen": 91095040 + }, + { + "epoch": 0.03, + "learning_rate": 0.000785765295887663, + "loss": 2.1387, + "theoretical_loss": 4.862739302040067, + "tokens_seen": 91160576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007857492477432298, + "loss": 2.0271, + "theoretical_loss": 4.862251313221231, + "tokens_seen": 91226112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007857331995987964, + "loss": 2.2534, + "theoretical_loss": 4.861763772921561, + "tokens_seen": 91291648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007857171514543632, + "loss": 2.1393, + "theoretical_loss": 4.861276680407279, + "tokens_seen": 91357184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007857011033099298, + "loss": 2.2016, + "theoretical_loss": 4.860790034946331, + "tokens_seen": 91422720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007856850551654966, + "loss": 2.1345, + "theoretical_loss": 4.860303835808384, + "tokens_seen": 91488256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007856690070210632, + "loss": 2.0493, + "theoretical_loss": 4.859818082264822, + "tokens_seen": 91553792 + }, + { + "epoch": 0.03, + "learning_rate": 0.00078565295887663, + "loss": 2.1049, + "theoretical_loss": 4.859332773588736, + "tokens_seen": 91619328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007856369107321966, + "loss": 2.0786, + "theoretical_loss": 4.858847909054923, + "tokens_seen": 91684864 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.9918930530548096, + "objective/train/avg_token_score": 0.008106924593448639, + "objective/train/docs_used": 40748, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.9682199954986572, + "objective/train/original_loss": 1.9848847389221191, + "objective/train/theoretical_loss": 4.858363487939879, + "objective/train/tokens_used": 112210400, + "objective/train/unlikelihood_loss": 0.014112748205661774, + "theoretical_loss": 4.858363487939879, + "tokens_seen": 91750400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007856208625877634, + "loss": 2.0816, + "theoretical_loss": 4.858363487939879, + "tokens_seen": 91750400 + }, + { + "epoch": 0.03, + "learning_rate": 0.00078560481444333, + "loss": 2.0831, + "theoretical_loss": 4.857879509521796, + "tokens_seen": 91815936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007855887662988968, + "loss": 2.106, + "theoretical_loss": 4.857395973080552, + "tokens_seen": 91881472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007855727181544634, + "loss": 2.1506, + "theoretical_loss": 4.856912877897713, + "tokens_seen": 91947008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007855566700100302, + "loss": 2.0721, + "theoretical_loss": 4.85643022325652, + "tokens_seen": 92012544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007855406218655968, + "loss": 2.0513, + "theoretical_loss": 4.855948008441891, + "tokens_seen": 92078080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007855245737211636, + "loss": 2.1205, + "theoretical_loss": 4.85546623274041, + "tokens_seen": 92143616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007855085255767302, + "loss": 2.0176, + "theoretical_loss": 4.854984895440325, + "tokens_seen": 92209152 + }, + { + "epoch": 0.03, + "learning_rate": 0.000785492477432297, + "loss": 2.0978, + "theoretical_loss": 4.854503995831545, + "tokens_seen": 92274688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007854764292878636, + "loss": 2.2272, + "theoretical_loss": 4.854023533205627, + "tokens_seen": 92340224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007854603811434303, + "loss": 2.1435, + "theoretical_loss": 4.853543506855784, + "tokens_seen": 92405760 + }, + { + "epoch": 0.03, + "learning_rate": 0.000785444332998997, + "loss": 2.2103, + "theoretical_loss": 4.853063916076865, + "tokens_seen": 92471296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007854282848545637, + "loss": 2.0071, + "theoretical_loss": 4.852584760165362, + "tokens_seen": 92536832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007854122367101304, + "loss": 2.0019, + "theoretical_loss": 4.8521060384194, + "tokens_seen": 92602368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007853961885656971, + "loss": 2.0884, + "theoretical_loss": 4.851627750138731, + "tokens_seen": 92667904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007853801404212638, + "loss": 2.1393, + "theoretical_loss": 4.851149894624731, + "tokens_seen": 92733440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007853640922768305, + "loss": 2.1675, + "theoretical_loss": 4.8506724711803955, + "tokens_seen": 92798976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007853480441323972, + "loss": 1.9318, + "theoretical_loss": 4.850195479110333, + "tokens_seen": 92864512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007853319959879639, + "loss": 2.1589, + "theoretical_loss": 4.849718917720762, + "tokens_seen": 92930048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007853159478435306, + "loss": 2.1659, + "theoretical_loss": 4.849242786319503, + "tokens_seen": 92995584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007852998996990973, + "loss": 2.1021, + "theoretical_loss": 4.848767084215979, + "tokens_seen": 93061120 + }, + { + "epoch": 0.03, + "learning_rate": 0.000785283851554664, + "loss": 2.0478, + "theoretical_loss": 4.848291810721204, + "tokens_seen": 93126656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007852678034102307, + "loss": 2.0745, + "theoretical_loss": 4.847816965147783, + "tokens_seen": 93192192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007852517552657974, + "loss": 2.112, + "theoretical_loss": 4.847342546809907, + "tokens_seen": 93257728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007852357071213641, + "loss": 2.0364, + "theoretical_loss": 4.846868555023345, + "tokens_seen": 93323264 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007852196589769308, + "loss": 2.1241, + "theoretical_loss": 4.846394989105444, + "tokens_seen": 93388800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007852036108324975, + "loss": 2.0922, + "theoretical_loss": 4.845921848375118, + "tokens_seen": 93454336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007851875626880642, + "loss": 2.0786, + "theoretical_loss": 4.845449132152851, + "tokens_seen": 93519872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007851715145436309, + "loss": 2.1247, + "theoretical_loss": 4.844976839760683, + "tokens_seen": 93585408 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007851554663991976, + "loss": 2.0652, + "theoretical_loss": 4.844504970522217, + "tokens_seen": 93650944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007851394182547643, + "loss": 2.0667, + "theoretical_loss": 4.844033523762603, + "tokens_seen": 93716480 + }, + { + "epoch": 0.03, + "learning_rate": 0.000785123370110331, + "loss": 2.0677, + "theoretical_loss": 4.843562498808538, + "tokens_seen": 93782016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007851073219658977, + "loss": 2.2479, + "theoretical_loss": 4.843091894988265, + "tokens_seen": 93847552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007850912738214644, + "loss": 1.9428, + "theoretical_loss": 4.842621711631564, + "tokens_seen": 93913088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007850752256770311, + "loss": 2.1982, + "theoretical_loss": 4.842151948069745, + "tokens_seen": 93978624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007850591775325978, + "loss": 1.9916, + "theoretical_loss": 4.84168260363565, + "tokens_seen": 94044160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007850431293881645, + "loss": 2.052, + "theoretical_loss": 4.841213677663648, + "tokens_seen": 94109696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007850270812437312, + "loss": 2.0562, + "theoretical_loss": 4.840745169489621, + "tokens_seen": 94175232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007850110330992979, + "loss": 2.0412, + "theoretical_loss": 4.840277078450969, + "tokens_seen": 94240768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007849949849548646, + "loss": 2.0865, + "theoretical_loss": 4.839809403886606, + "tokens_seen": 94306304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007849789368104313, + "loss": 1.9452, + "theoretical_loss": 4.839342145136947, + "tokens_seen": 94371840 + }, + { + "epoch": 0.03, + "learning_rate": 0.000784962888665998, + "loss": 2.0456, + "theoretical_loss": 4.838875301543913, + "tokens_seen": 94437376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007849468405215647, + "loss": 2.0504, + "theoretical_loss": 4.838408872450919, + "tokens_seen": 94502912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007849307923771314, + "loss": 2.1553, + "theoretical_loss": 4.837942857202876, + "tokens_seen": 94568448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007849147442326981, + "loss": 1.9889, + "theoretical_loss": 4.837477255146178, + "tokens_seen": 94633984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007848986960882648, + "loss": 2.0932, + "theoretical_loss": 4.837012065628711, + "tokens_seen": 94699520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007848826479438315, + "loss": 1.9538, + "theoretical_loss": 4.836547287999833, + "tokens_seen": 94765056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007848665997993982, + "loss": 2.0259, + "theoretical_loss": 4.836082921610381, + "tokens_seen": 94830592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007848505516549649, + "loss": 2.0683, + "theoretical_loss": 4.835618965812664, + "tokens_seen": 94896128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007848345035105317, + "loss": 2.0804, + "theoretical_loss": 4.835155419960453, + "tokens_seen": 94961664 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.9844337105751038, + "objective/train/avg_token_score": 0.01556632574647665, + "objective/train/docs_used": 42039, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.872319221496582, + "objective/train/original_loss": 1.8973153829574585, + "objective/train/theoretical_loss": 4.8346922834089865, + "objective/train/tokens_used": 115487200, + "objective/train/unlikelihood_loss": 0.03274616226553917, + "theoretical_loss": 4.8346922834089865, + "tokens_seen": 95027200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007848184553660983, + "loss": 1.9787, + "theoretical_loss": 4.8346922834089865, + "tokens_seen": 95027200 + }, + { + "epoch": 0.03, + "learning_rate": 0.000784802407221665, + "loss": 2.0249, + "theoretical_loss": 4.834229555514956, + "tokens_seen": 95092736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007847863590772317, + "loss": 2.0252, + "theoretical_loss": 4.833767235636511, + "tokens_seen": 95158272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007847703109327985, + "loss": 2.0253, + "theoretical_loss": 4.833305323133246, + "tokens_seen": 95223808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007847542627883651, + "loss": 2.097, + "theoretical_loss": 4.832843817366204, + "tokens_seen": 95289344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007847382146439319, + "loss": 2.0668, + "theoretical_loss": 4.832382717697866, + "tokens_seen": 95354880 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007847221664994985, + "loss": 2.0777, + "theoretical_loss": 4.831922023492153, + "tokens_seen": 95420416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007847061183550653, + "loss": 2.057, + "theoretical_loss": 4.831461734114415, + "tokens_seen": 95485952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007846900702106319, + "loss": 2.1228, + "theoretical_loss": 4.831001848931429, + "tokens_seen": 95551488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007846740220661987, + "loss": 2.0962, + "theoretical_loss": 4.830542367311402, + "tokens_seen": 95617024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007846579739217653, + "loss": 2.1579, + "theoretical_loss": 4.830083288623955, + "tokens_seen": 95682560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007846419257773321, + "loss": 2.0384, + "theoretical_loss": 4.829624612240126, + "tokens_seen": 95748096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007846258776328987, + "loss": 2.1281, + "theoretical_loss": 4.829166337532367, + "tokens_seen": 95813632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007846098294884655, + "loss": 2.0431, + "theoretical_loss": 4.828708463874532, + "tokens_seen": 95879168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007845937813440321, + "loss": 2.0991, + "theoretical_loss": 4.828250990641886, + "tokens_seen": 95944704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007845777331995989, + "loss": 2.0856, + "theoretical_loss": 4.827793917211083, + "tokens_seen": 96010240 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007845616850551655, + "loss": 2.1343, + "theoretical_loss": 4.827337242960181, + "tokens_seen": 96075776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007845456369107323, + "loss": 1.9077, + "theoretical_loss": 4.8268809672686235, + "tokens_seen": 96141312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007845295887662989, + "loss": 1.9759, + "theoretical_loss": 4.826425089517245, + "tokens_seen": 96206848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007845135406218657, + "loss": 2.0163, + "theoretical_loss": 4.825969609088258, + "tokens_seen": 96272384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007844974924774323, + "loss": 2.1741, + "theoretical_loss": 4.825514525365258, + "tokens_seen": 96337920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007844814443329991, + "loss": 2.0374, + "theoretical_loss": 4.825059837733214, + "tokens_seen": 96403456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007844653961885657, + "loss": 2.047, + "theoretical_loss": 4.824605545578463, + "tokens_seen": 96468992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007844493480441325, + "loss": 2.0419, + "theoretical_loss": 4.824151648288714, + "tokens_seen": 96534528 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007844332998996991, + "loss": 1.9997, + "theoretical_loss": 4.823698145253035, + "tokens_seen": 96600064 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007844172517552659, + "loss": 2.1593, + "theoretical_loss": 4.823245035861852, + "tokens_seen": 96665600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007844012036108325, + "loss": 2.123, + "theoretical_loss": 4.822792319506952, + "tokens_seen": 96731136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007843851554663993, + "loss": 2.0436, + "theoretical_loss": 4.822339995581466, + "tokens_seen": 96796672 + }, + { + "epoch": 0.03, + "learning_rate": 0.000784369107321966, + "loss": 2.0226, + "theoretical_loss": 4.821888063479874, + "tokens_seen": 96862208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007843530591775327, + "loss": 2.1588, + "theoretical_loss": 4.8214365225980025, + "tokens_seen": 96927744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007843370110330994, + "loss": 2.0423, + "theoretical_loss": 4.8209853723330145, + "tokens_seen": 96993280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007843209628886661, + "loss": 1.9752, + "theoretical_loss": 4.820534612083408, + "tokens_seen": 97058816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007843049147442328, + "loss": 2.1115, + "theoretical_loss": 4.820084241249013, + "tokens_seen": 97124352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007842888665997995, + "loss": 2.0251, + "theoretical_loss": 4.819634259230991, + "tokens_seen": 97189888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007842728184553662, + "loss": 1.9593, + "theoretical_loss": 4.819184665431821, + "tokens_seen": 97255424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007842567703109329, + "loss": 1.9477, + "theoretical_loss": 4.818735459255306, + "tokens_seen": 97320960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007842407221664996, + "loss": 2.0983, + "theoretical_loss": 4.818286640106564, + "tokens_seen": 97386496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007842246740220663, + "loss": 2.2571, + "theoretical_loss": 4.817838207392027, + "tokens_seen": 97452032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007842086258776329, + "loss": 2.1768, + "theoretical_loss": 4.817390160519435, + "tokens_seen": 97517568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007841925777331997, + "loss": 2.195, + "theoretical_loss": 4.816942498897831, + "tokens_seen": 97583104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007841765295887663, + "loss": 2.3367, + "theoretical_loss": 4.816495221937563, + "tokens_seen": 97648640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007841604814443331, + "loss": 2.0875, + "theoretical_loss": 4.816048329050273, + "tokens_seen": 97714176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007841444332998997, + "loss": 2.1441, + "theoretical_loss": 4.815601819648897, + "tokens_seen": 97779712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007841283851554665, + "loss": 1.9817, + "theoretical_loss": 4.815155693147666, + "tokens_seen": 97845248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007841123370110331, + "loss": 2.0688, + "theoretical_loss": 4.81470994896209, + "tokens_seen": 97910784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007840962888665999, + "loss": 2.0937, + "theoretical_loss": 4.8142645865089655, + "tokens_seen": 97976320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007840802407221665, + "loss": 2.0717, + "theoretical_loss": 4.813819605206369, + "tokens_seen": 98041856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007840641925777332, + "loss": 2.0095, + "theoretical_loss": 4.813375004473651, + "tokens_seen": 98107392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007840481444332999, + "loss": 1.9526, + "theoretical_loss": 4.812930783731433, + "tokens_seen": 98172928 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007840320962888666, + "loss": 1.9116, + "theoretical_loss": 4.8124869424016055, + "tokens_seen": 98238464 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.9935272932052612, + "objective/train/avg_token_score": 0.006472676061093807, + "objective/train/docs_used": 43181, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 2.07255482673645, + "objective/train/original_loss": 2.087444543838501, + "objective/train/theoretical_loss": 4.812043479907323, + "objective/train/tokens_used": 118764000, + "objective/train/unlikelihood_loss": 0.009569038636982441, + "theoretical_loss": 4.812043479907323, + "tokens_seen": 98304000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007840160481444333, + "loss": 2.0712, + "theoretical_loss": 4.812043479907323, + "tokens_seen": 98304000 + }, + { + "epoch": 0.03, + "learning_rate": 0.000784, + "loss": 2.0137, + "theoretical_loss": 4.8116003956730005, + "tokens_seen": 98369536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007839839518555667, + "loss": 2.0458, + "theoretical_loss": 4.811157689124311, + "tokens_seen": 98435072 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007839679037111334, + "loss": 1.9083, + "theoretical_loss": 4.8107153596881815, + "tokens_seen": 98500608 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007839518555667001, + "loss": 1.9337, + "theoretical_loss": 4.810273406792788, + "tokens_seen": 98566144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007839358074222668, + "loss": 2.0424, + "theoretical_loss": 4.809831829867555, + "tokens_seen": 98631680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007839197592778335, + "loss": 2.1512, + "theoretical_loss": 4.809390628343146, + "tokens_seen": 98697216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007839037111334002, + "loss": 2.0589, + "theoretical_loss": 4.808949801651468, + "tokens_seen": 98762752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007838876629889669, + "loss": 2.0503, + "theoretical_loss": 4.8085093492256625, + "tokens_seen": 98828288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007838716148445336, + "loss": 1.9233, + "theoretical_loss": 4.808069270500102, + "tokens_seen": 98893824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007838555667001003, + "loss": 1.985, + "theoretical_loss": 4.80762956491039, + "tokens_seen": 98959360 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783839518555667, + "loss": 2.0833, + "theoretical_loss": 4.807190231893355, + "tokens_seen": 99024896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007838234704112337, + "loss": 2.1463, + "theoretical_loss": 4.806751270887048, + "tokens_seen": 99090432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007838074222668004, + "loss": 2.032, + "theoretical_loss": 4.806312681330736, + "tokens_seen": 99155968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007837913741223671, + "loss": 1.9805, + "theoretical_loss": 4.805874462664903, + "tokens_seen": 99221504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007837753259779338, + "loss": 2.0743, + "theoretical_loss": 4.805436614331247, + "tokens_seen": 99287040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007837592778335005, + "loss": 2.1073, + "theoretical_loss": 4.804999135772668, + "tokens_seen": 99352576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007837432296890672, + "loss": 2.1255, + "theoretical_loss": 4.804562026433278, + "tokens_seen": 99418112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007837271815446339, + "loss": 2.0525, + "theoretical_loss": 4.804125285758383, + "tokens_seen": 99483648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007837111334002006, + "loss": 2.0697, + "theoretical_loss": 4.803688913194495, + "tokens_seen": 99549184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007836950852557673, + "loss": 2.043, + "theoretical_loss": 4.8032529081893145, + "tokens_seen": 99614720 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783679037111334, + "loss": 2.0285, + "theoretical_loss": 4.802817270191736, + "tokens_seen": 99680256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007836629889669007, + "loss": 2.0385, + "theoretical_loss": 4.802381998651841, + "tokens_seen": 99745792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007836469408224674, + "loss": 1.9594, + "theoretical_loss": 4.801947093020896, + "tokens_seen": 99811328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007836308926780341, + "loss": 2.1904, + "theoretical_loss": 4.801512552751349, + "tokens_seen": 99876864 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007836148445336008, + "loss": 1.9369, + "theoretical_loss": 4.801078377296827, + "tokens_seen": 99942400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007835987963891675, + "loss": 1.9823, + "theoretical_loss": 4.800644566112128, + "tokens_seen": 100007936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007835827482447342, + "loss": 2.2047, + "theoretical_loss": 4.800211118653225, + "tokens_seen": 100073472 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783566700100301, + "loss": 2.069, + "theoretical_loss": 4.799778034377258, + "tokens_seen": 100139008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007835506519558676, + "loss": 2.065, + "theoretical_loss": 4.799345312742531, + "tokens_seen": 100204544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007835346038114343, + "loss": 2.049, + "theoretical_loss": 4.798912953208511, + "tokens_seen": 100270080 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783518555667001, + "loss": 2.0351, + "theoretical_loss": 4.798480955235822, + "tokens_seen": 100335616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007835025075225677, + "loss": 2.2408, + "theoretical_loss": 4.798049318286243, + "tokens_seen": 100401152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007834864593781344, + "loss": 2.148, + "theoretical_loss": 4.797618041822707, + "tokens_seen": 100466688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007834704112337011, + "loss": 1.9126, + "theoretical_loss": 4.797187125309293, + "tokens_seen": 100532224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007834543630892678, + "loss": 2.0517, + "theoretical_loss": 4.796756568211227, + "tokens_seen": 100597760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007834383149448346, + "loss": 1.9768, + "theoretical_loss": 4.796326369994876, + "tokens_seen": 100663296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007834222668004012, + "loss": 2.0132, + "theoretical_loss": 4.795896530127747, + "tokens_seen": 100728832 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783406218655968, + "loss": 2.132, + "theoretical_loss": 4.795467048078484, + "tokens_seen": 100794368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007833901705115346, + "loss": 1.9406, + "theoretical_loss": 4.79503792331686, + "tokens_seen": 100859904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007833741223671014, + "loss": 2.0628, + "theoretical_loss": 4.794609155313782, + "tokens_seen": 100925440 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783358074222668, + "loss": 2.0189, + "theoretical_loss": 4.79418074354128, + "tokens_seen": 100990976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007833420260782348, + "loss": 2.1237, + "theoretical_loss": 4.79375268747251, + "tokens_seen": 101056512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007833259779338014, + "loss": 2.1929, + "theoretical_loss": 4.793324986581746, + "tokens_seen": 101122048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007833099297893682, + "loss": 1.9621, + "theoretical_loss": 4.79289764034438, + "tokens_seen": 101187584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007832938816449348, + "loss": 1.938, + "theoretical_loss": 4.792470648236918, + "tokens_seen": 101253120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007832778335005016, + "loss": 1.9949, + "theoretical_loss": 4.792044009736978, + "tokens_seen": 101318656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007832617853560682, + "loss": 2.043, + "theoretical_loss": 4.791617724323283, + "tokens_seen": 101384192 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783245737211635, + "loss": 2.069, + "theoretical_loss": 4.791191791475663, + "tokens_seen": 101449728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007832296890672016, + "loss": 2.0345, + "theoretical_loss": 4.79076621067505, + "tokens_seen": 101515264 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.9919618964195251, + "objective/train/avg_token_score": 0.008038081228733063, + "objective/train/docs_used": 44310, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 2.0118792057037354, + "objective/train/original_loss": 2.030895709991455, + "objective/train/theoretical_loss": 4.790340981403474, + "objective/train/tokens_used": 122040800, + "objective/train/unlikelihood_loss": 0.012512128800153732, + "theoretical_loss": 4.790340981403474, + "tokens_seen": 101580800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007832136409227684, + "loss": 1.9773, + "theoretical_loss": 4.790340981403474, + "tokens_seen": 101580800 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783197592778335, + "loss": 2.051, + "theoretical_loss": 4.7899161031440585, + "tokens_seen": 101646336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007831815446339018, + "loss": 1.9968, + "theoretical_loss": 4.7894915753810245, + "tokens_seen": 101711872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007831654964894685, + "loss": 1.9352, + "theoretical_loss": 4.789067397599677, + "tokens_seen": 101777408 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007831494483450352, + "loss": 2.047, + "theoretical_loss": 4.788643569286411, + "tokens_seen": 101842944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007831334002006019, + "loss": 1.9497, + "theoretical_loss": 4.788220089928705, + "tokens_seen": 101908480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007831173520561686, + "loss": 1.9725, + "theoretical_loss": 4.787796959015118, + "tokens_seen": 101974016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007831013039117353, + "loss": 2.0298, + "theoretical_loss": 4.787374176035287, + "tokens_seen": 102039552 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783085255767302, + "loss": 2.1528, + "theoretical_loss": 4.78695174047992, + "tokens_seen": 102105088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007830692076228687, + "loss": 2.0716, + "theoretical_loss": 4.786529651840802, + "tokens_seen": 102170624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007830531594784354, + "loss": 1.9291, + "theoretical_loss": 4.786107909610784, + "tokens_seen": 102236160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007830371113340021, + "loss": 1.925, + "theoretical_loss": 4.785686513283783, + "tokens_seen": 102301696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007830210631895688, + "loss": 1.9338, + "theoretical_loss": 4.785265462354779, + "tokens_seen": 102367232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007830050150451355, + "loss": 1.9165, + "theoretical_loss": 4.7848447563198135, + "tokens_seen": 102432768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007829889669007022, + "loss": 2.0255, + "theoretical_loss": 4.784424394675983, + "tokens_seen": 102498304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007829729187562689, + "loss": 2.0497, + "theoretical_loss": 4.784004376921439, + "tokens_seen": 102563840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007829568706118356, + "loss": 1.9817, + "theoretical_loss": 4.783584702555387, + "tokens_seen": 102629376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007829408224674023, + "loss": 2.0133, + "theoretical_loss": 4.783165371078075, + "tokens_seen": 102694912 + }, + { + "epoch": 0.03, + "learning_rate": 0.000782924774322969, + "loss": 1.9813, + "theoretical_loss": 4.782746381990803, + "tokens_seen": 102760448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007829087261785357, + "loss": 2.1379, + "theoretical_loss": 4.78232773479591, + "tokens_seen": 102825984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007828926780341024, + "loss": 2.0815, + "theoretical_loss": 4.7819094289967765, + "tokens_seen": 102891520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007828766298896691, + "loss": 2.0199, + "theoretical_loss": 4.781491464097819, + "tokens_seen": 102957056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007828605817452358, + "loss": 1.8963, + "theoretical_loss": 4.781073839604488, + "tokens_seen": 103022592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007828445336008025, + "loss": 2.0069, + "theoretical_loss": 4.780656555023268, + "tokens_seen": 103088128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007828284854563692, + "loss": 2.051, + "theoretical_loss": 4.780239609861669, + "tokens_seen": 103153664 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007828124373119358, + "loss": 2.0262, + "theoretical_loss": 4.77982300362823, + "tokens_seen": 103219200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007827963891675026, + "loss": 2.0423, + "theoretical_loss": 4.779406735832509, + "tokens_seen": 103284736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007827803410230692, + "loss": 2.0629, + "theoretical_loss": 4.778990805985089, + "tokens_seen": 103350272 + }, + { + "epoch": 0.03, + "learning_rate": 0.000782764292878636, + "loss": 1.9473, + "theoretical_loss": 4.778575213597568, + "tokens_seen": 103415808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007827482447342026, + "loss": 2.0208, + "theoretical_loss": 4.778159958182557, + "tokens_seen": 103481344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007827321965897694, + "loss": 1.895, + "theoretical_loss": 4.777745039253682, + "tokens_seen": 103546880 + }, + { + "epoch": 0.03, + "learning_rate": 0.000782716148445336, + "loss": 1.9761, + "theoretical_loss": 4.777330456325579, + "tokens_seen": 103612416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007827001003009028, + "loss": 2.0272, + "theoretical_loss": 4.776916208913887, + "tokens_seen": 103677952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007826840521564694, + "loss": 2.0525, + "theoretical_loss": 4.776502296535252, + "tokens_seen": 103743488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007826680040120361, + "loss": 1.9849, + "theoretical_loss": 4.776088718707317, + "tokens_seen": 103809024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007826519558676028, + "loss": 1.9448, + "theoretical_loss": 4.77567547494873, + "tokens_seen": 103874560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007826359077231695, + "loss": 2.1045, + "theoretical_loss": 4.775262564779128, + "tokens_seen": 103940096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007826198595787362, + "loss": 2.1495, + "theoretical_loss": 4.774849987719145, + "tokens_seen": 104005632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007826038114343029, + "loss": 2.1373, + "theoretical_loss": 4.774437743290404, + "tokens_seen": 104071168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007825877632898696, + "loss": 2.0749, + "theoretical_loss": 4.774025831015517, + "tokens_seen": 104136704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007825717151454363, + "loss": 1.8764, + "theoretical_loss": 4.7736142504180785, + "tokens_seen": 104202240 + }, + { + "epoch": 0.03, + "learning_rate": 0.000782555667001003, + "loss": 1.825, + "theoretical_loss": 4.773203001022668, + "tokens_seen": 104267776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007825396188565697, + "loss": 2.0694, + "theoretical_loss": 4.772792082354843, + "tokens_seen": 104333312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007825235707121364, + "loss": 2.0405, + "theoretical_loss": 4.77238149394114, + "tokens_seen": 104398848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007825075225677031, + "loss": 2.07, + "theoretical_loss": 4.771971235309065, + "tokens_seen": 104464384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007824914744232698, + "loss": 1.971, + "theoretical_loss": 4.771561305987104, + "tokens_seen": 104529920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007824754262788365, + "loss": 2.055, + "theoretical_loss": 4.771151705504707, + "tokens_seen": 104595456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007824593781344032, + "loss": 2.0387, + "theoretical_loss": 4.770742433392289, + "tokens_seen": 104660992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007824433299899699, + "loss": 1.9287, + "theoretical_loss": 4.770333489181233, + "tokens_seen": 104726528 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007824272818455366, + "loss": 1.8892, + "theoretical_loss": 4.769924872403883, + "tokens_seen": 104792064 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.99358731508255, + "objective/train/avg_token_score": 0.006412716582417488, + "objective/train/docs_used": 45444, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 2.176731586456299, + "objective/train/original_loss": 2.191204071044922, + "objective/train/theoretical_loss": 4.769516582593539, + "objective/train/tokens_used": 125317600, + "objective/train/unlikelihood_loss": 0.009412930347025394, + "theoretical_loss": 4.769516582593539, + "tokens_seen": 104857600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007824112337011033, + "loss": 2.0789, + "theoretical_loss": 4.769516582593539, + "tokens_seen": 104857600 + }, + { + "epoch": 0.03, + "learning_rate": 0.00078239518555667, + "loss": 1.9824, + "theoretical_loss": 4.769108619284461, + "tokens_seen": 104923136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007823791374122367, + "loss": 1.8608, + "theoretical_loss": 4.768700982011859, + "tokens_seen": 104988672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007823630892678034, + "loss": 1.9963, + "theoretical_loss": 4.768293670311899, + "tokens_seen": 105054208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007823470411233701, + "loss": 1.9125, + "theoretical_loss": 4.7678866837216916, + "tokens_seen": 105119744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007823309929789368, + "loss": 1.9932, + "theoretical_loss": 4.767480021779294, + "tokens_seen": 105185280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007823149448345035, + "loss": 1.9655, + "theoretical_loss": 4.76707368402371, + "tokens_seen": 105250816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007822988966900702, + "loss": 1.9587, + "theoretical_loss": 4.766667669994881, + "tokens_seen": 105316352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007822828485456369, + "loss": 1.9557, + "theoretical_loss": 4.76626197923369, + "tokens_seen": 105381888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007822668004012036, + "loss": 1.9842, + "theoretical_loss": 4.765856611281953, + "tokens_seen": 105447424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007822507522567703, + "loss": 1.9699, + "theoretical_loss": 4.765451565682423, + "tokens_seen": 105512960 + }, + { + "epoch": 0.03, + "learning_rate": 0.000782234704112337, + "loss": 2.076, + "theoretical_loss": 4.765046841978783, + "tokens_seen": 105578496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007822186559679037, + "loss": 1.9584, + "theoretical_loss": 4.764642439715642, + "tokens_seen": 105644032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007822026078234704, + "loss": 2.2071, + "theoretical_loss": 4.764238358438539, + "tokens_seen": 105709568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007821865596790371, + "loss": 2.0728, + "theoretical_loss": 4.763834597693935, + "tokens_seen": 105775104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007821705115346038, + "loss": 1.9223, + "theoretical_loss": 4.763431157029213, + "tokens_seen": 105840640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007821544633901705, + "loss": 1.9132, + "theoretical_loss": 4.763028035992675, + "tokens_seen": 105906176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007821384152457372, + "loss": 2.0886, + "theoretical_loss": 4.762625234133537, + "tokens_seen": 105971712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007821223671013039, + "loss": 1.9649, + "theoretical_loss": 4.762222751001933, + "tokens_seen": 106037248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007821063189568706, + "loss": 1.9117, + "theoretical_loss": 4.761820586148906, + "tokens_seen": 106102784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007820902708124373, + "loss": 2.0322, + "theoretical_loss": 4.761418739126407, + "tokens_seen": 106168320 + }, + { + "epoch": 0.03, + "learning_rate": 0.000782074222668004, + "loss": 1.9047, + "theoretical_loss": 4.761017209487298, + "tokens_seen": 106233856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007820581745235707, + "loss": 2.001, + "theoretical_loss": 4.760615996785342, + "tokens_seen": 106299392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007820421263791375, + "loss": 2.0488, + "theoretical_loss": 4.760215100575204, + "tokens_seen": 106364928 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007820260782347041, + "loss": 2.0329, + "theoretical_loss": 4.759814520412451, + "tokens_seen": 106430464 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007820100300902709, + "loss": 1.9832, + "theoretical_loss": 4.759414255853547, + "tokens_seen": 106496000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007819939819458375, + "loss": 2.0522, + "theoretical_loss": 4.759014306455848, + "tokens_seen": 106561536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007819779338014043, + "loss": 1.9461, + "theoretical_loss": 4.758614671777604, + "tokens_seen": 106627072 + }, + { + "epoch": 0.03, + "learning_rate": 0.000781961885656971, + "loss": 1.9814, + "theoretical_loss": 4.758215351377956, + "tokens_seen": 106692608 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007819458375125377, + "loss": 1.9185, + "theoretical_loss": 4.757816344816933, + "tokens_seen": 106758144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007819297893681044, + "loss": 2.0124, + "theoretical_loss": 4.75741765165545, + "tokens_seen": 106823680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007819137412236711, + "loss": 2.0323, + "theoretical_loss": 4.7570192714553015, + "tokens_seen": 106889216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007818976930792378, + "loss": 1.9395, + "theoretical_loss": 4.756621203779169, + "tokens_seen": 106954752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007818816449348045, + "loss": 2.0326, + "theoretical_loss": 4.756223448190605, + "tokens_seen": 107020288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007818655967903712, + "loss": 2.1202, + "theoretical_loss": 4.755826004254046, + "tokens_seen": 107085824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007818495486459379, + "loss": 2.0305, + "theoretical_loss": 4.755428871534798, + "tokens_seen": 107151360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007818335005015046, + "loss": 1.9645, + "theoretical_loss": 4.755032049599039, + "tokens_seen": 107216896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007818174523570713, + "loss": 2.0161, + "theoretical_loss": 4.754635538013817, + "tokens_seen": 107282432 + }, + { + "epoch": 0.03, + "learning_rate": 0.000781801404212638, + "loss": 2.01, + "theoretical_loss": 4.754239336347048, + "tokens_seen": 107347968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007817853560682047, + "loss": 1.9774, + "theoretical_loss": 4.753843444167513, + "tokens_seen": 107413504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007817693079237714, + "loss": 1.9871, + "theoretical_loss": 4.753447861044853, + "tokens_seen": 107479040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007817532597793381, + "loss": 2.038, + "theoretical_loss": 4.753052586549572, + "tokens_seen": 107544576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007817372116349048, + "loss": 2.0042, + "theoretical_loss": 4.75265762025303, + "tokens_seen": 107610112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007817211634904715, + "loss": 1.9858, + "theoretical_loss": 4.752262961727446, + "tokens_seen": 107675648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007817051153460382, + "loss": 2.0504, + "theoretical_loss": 4.75186861054589, + "tokens_seen": 107741184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007816890672016049, + "loss": 1.9533, + "theoretical_loss": 4.751474566282285, + "tokens_seen": 107806720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007816730190571716, + "loss": 2.1084, + "theoretical_loss": 4.751080828511402, + "tokens_seen": 107872256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007816569709127383, + "loss": 2.0569, + "theoretical_loss": 4.750687396808859, + "tokens_seen": 107937792 + }, + { + "epoch": 0.03, + "learning_rate": 0.000781640922768305, + "loss": 1.9002, + "theoretical_loss": 4.7502942707511195, + "tokens_seen": 108003328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007816248746238717, + "loss": 2.0098, + "theoretical_loss": 4.749901449915489, + "tokens_seen": 108068864 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.993815004825592, + "objective/train/avg_token_score": 0.006184975150972605, + "objective/train/docs_used": 46682, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 2.1743500232696533, + "objective/train/original_loss": 2.187734842300415, + "objective/train/theoretical_loss": 4.749508933880117, + "objective/train/tokens_used": 128594400, + "objective/train/unlikelihood_loss": 0.008992877788841724, + "theoretical_loss": 4.749508933880117, + "tokens_seen": 108134400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007816088264794384, + "loss": 2.0878, + "theoretical_loss": 4.749508933880117, + "tokens_seen": 108134400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007815927783350051, + "loss": 1.9908, + "theoretical_loss": 4.7491167222239845, + "tokens_seen": 108199936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007815767301905718, + "loss": 1.9829, + "theoretical_loss": 4.7487248145269145, + "tokens_seen": 108265472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007815606820461385, + "loss": 2.0147, + "theoretical_loss": 4.7483332103695615, + "tokens_seen": 108331008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007815446339017052, + "loss": 2.0711, + "theoretical_loss": 4.747941909333411, + "tokens_seen": 108396544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007815285857572719, + "loss": 1.9638, + "theoretical_loss": 4.747550911000781, + "tokens_seen": 108462080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007815125376128386, + "loss": 1.9358, + "theoretical_loss": 4.747160214954813, + "tokens_seen": 108527616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007814964894684053, + "loss": 2.0648, + "theoretical_loss": 4.746769820779477, + "tokens_seen": 108593152 + }, + { + "epoch": 0.03, + "learning_rate": 0.000781480441323972, + "loss": 1.984, + "theoretical_loss": 4.746379728059566, + "tokens_seen": 108658688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007814643931795387, + "loss": 2.0401, + "theoretical_loss": 4.745989936380692, + "tokens_seen": 108724224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007814483450351054, + "loss": 2.0517, + "theoretical_loss": 4.745600445329288, + "tokens_seen": 108789760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007814322968906721, + "loss": 2.0145, + "theoretical_loss": 4.745211254492602, + "tokens_seen": 108855296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007814162487462388, + "loss": 1.9913, + "theoretical_loss": 4.744822363458699, + "tokens_seen": 108920832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007814002006018055, + "loss": 2.0054, + "theoretical_loss": 4.744433771816455, + "tokens_seen": 108986368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007813841524573721, + "loss": 2.0711, + "theoretical_loss": 4.744045479155557, + "tokens_seen": 109051904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007813681043129389, + "loss": 1.977, + "theoretical_loss": 4.7436574850665005, + "tokens_seen": 109117440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007813520561685055, + "loss": 1.9227, + "theoretical_loss": 4.743269789140587, + "tokens_seen": 109182976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007813360080240723, + "loss": 1.9192, + "theoretical_loss": 4.742882390969925, + "tokens_seen": 109248512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007813199598796389, + "loss": 1.9957, + "theoretical_loss": 4.742495290147421, + "tokens_seen": 109314048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007813039117352057, + "loss": 1.8807, + "theoretical_loss": 4.742108486266783, + "tokens_seen": 109379584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007812878635907723, + "loss": 2.0039, + "theoretical_loss": 4.74172197892252, + "tokens_seen": 109445120 + }, + { + "epoch": 0.03, + "learning_rate": 0.000781271815446339, + "loss": 1.9524, + "theoretical_loss": 4.741335767709934, + "tokens_seen": 109510656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007812557673019057, + "loss": 2.0055, + "theoretical_loss": 4.740949852225122, + "tokens_seen": 109576192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007812397191574724, + "loss": 2.0217, + "theoretical_loss": 4.740564232064973, + "tokens_seen": 109641728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007812236710130391, + "loss": 2.0621, + "theoretical_loss": 4.7401789068271665, + "tokens_seen": 109707264 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007812076228686058, + "loss": 1.9996, + "theoretical_loss": 4.73979387611017, + "tokens_seen": 109772800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007811915747241725, + "loss": 1.9185, + "theoretical_loss": 4.739409139513235, + "tokens_seen": 109838336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007811755265797392, + "loss": 2.0456, + "theoretical_loss": 4.739024696636401, + "tokens_seen": 109903872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007811594784353059, + "loss": 2.0277, + "theoretical_loss": 4.738640547080485, + "tokens_seen": 109969408 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007811434302908726, + "loss": 1.9292, + "theoretical_loss": 4.738256690447088, + "tokens_seen": 110034944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007811273821464393, + "loss": 2.0519, + "theoretical_loss": 4.7378731263385845, + "tokens_seen": 110100480 + }, + { + "epoch": 0.03, + "learning_rate": 0.000781111334002006, + "loss": 1.975, + "theoretical_loss": 4.737489854358129, + "tokens_seen": 110166016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007810952858575727, + "loss": 1.9108, + "theoretical_loss": 4.737106874109649, + "tokens_seen": 110231552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007810792377131394, + "loss": 1.9994, + "theoretical_loss": 4.736724185197844, + "tokens_seen": 110297088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007810631895687061, + "loss": 2.0914, + "theoretical_loss": 4.7363417872281826, + "tokens_seen": 110362624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007810471414242728, + "loss": 2.0445, + "theoretical_loss": 4.735959679806903, + "tokens_seen": 110428160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007810310932798395, + "loss": 2.0212, + "theoretical_loss": 4.735577862541007, + "tokens_seen": 110493696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007810150451354062, + "loss": 1.9663, + "theoretical_loss": 4.735196335038264, + "tokens_seen": 110559232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007809989969909729, + "loss": 2.1233, + "theoretical_loss": 4.734815096907203, + "tokens_seen": 110624768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007809829488465396, + "loss": 1.8978, + "theoretical_loss": 4.734434147757117, + "tokens_seen": 110690304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007809669007021063, + "loss": 1.9825, + "theoretical_loss": 4.734053487198054, + "tokens_seen": 110755840 + }, + { + "epoch": 0.03, + "learning_rate": 0.000780950852557673, + "loss": 1.9634, + "theoretical_loss": 4.733673114840817, + "tokens_seen": 110821376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007809348044132397, + "loss": 1.9984, + "theoretical_loss": 4.73329303029697, + "tokens_seen": 110886912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007809187562688064, + "loss": 2.0753, + "theoretical_loss": 4.732913233178825, + "tokens_seen": 110952448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007809027081243731, + "loss": 1.8978, + "theoretical_loss": 4.732533723099444, + "tokens_seen": 111017984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007808866599799398, + "loss": 1.9905, + "theoretical_loss": 4.732154499672642, + "tokens_seen": 111083520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007808706118355065, + "loss": 1.9255, + "theoretical_loss": 4.731775562512977, + "tokens_seen": 111149056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007808545636910732, + "loss": 1.9471, + "theoretical_loss": 4.7313969112357555, + "tokens_seen": 111214592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007808385155466399, + "loss": 1.8552, + "theoretical_loss": 4.731018545457024, + "tokens_seen": 111280128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007808224674022066, + "loss": 2.0184, + "theoretical_loss": 4.730640464793574, + "tokens_seen": 111345664 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.9922261834144592, + "objective/train/avg_token_score": 0.007773855701088905, + "objective/train/docs_used": 47788, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.9350111484527588, + "objective/train/original_loss": 1.954054594039917, + "objective/train/theoretical_loss": 4.730262668862931, + "objective/train/tokens_used": 131871200, + "objective/train/unlikelihood_loss": 0.010545310564339161, + "theoretical_loss": 4.730262668862931, + "tokens_seen": 111411200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007808064192577733, + "loss": 1.9797, + "theoretical_loss": 4.730262668862931, + "tokens_seen": 111411200 + }, + { + "epoch": 0.03, + "learning_rate": 0.00078079037111334, + "loss": 2.0699, + "theoretical_loss": 4.729885157283366, + "tokens_seen": 111476736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007807743229689067, + "loss": 1.9518, + "theoretical_loss": 4.729507929673879, + "tokens_seen": 111542272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007807582748244735, + "loss": 1.9749, + "theoretical_loss": 4.729130985654208, + "tokens_seen": 111607808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007807422266800401, + "loss": 1.9893, + "theoretical_loss": 4.72875432484482, + "tokens_seen": 111673344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007807261785356069, + "loss": 2.0288, + "theoretical_loss": 4.728377946866915, + "tokens_seen": 111738880 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007807101303911736, + "loss": 2.0694, + "theoretical_loss": 4.72800185134242, + "tokens_seen": 111804416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007806940822467403, + "loss": 2.0618, + "theoretical_loss": 4.727626037893989, + "tokens_seen": 111869952 + }, + { + "epoch": 0.03, + "learning_rate": 0.000780678034102307, + "loss": 1.9279, + "theoretical_loss": 4.727250506145001, + "tokens_seen": 111935488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007806619859578737, + "loss": 1.96, + "theoretical_loss": 4.726875255719557, + "tokens_seen": 112001024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007806459378134404, + "loss": 2.132, + "theoretical_loss": 4.726500286242482, + "tokens_seen": 112066560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007806298896690071, + "loss": 2.0129, + "theoretical_loss": 4.7261255973393155, + "tokens_seen": 112132096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007806138415245738, + "loss": 2.0158, + "theoretical_loss": 4.725751188636319, + "tokens_seen": 112197632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007805977933801405, + "loss": 1.9654, + "theoretical_loss": 4.725377059760468, + "tokens_seen": 112263168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007805817452357072, + "loss": 1.9534, + "theoretical_loss": 4.725003210339452, + "tokens_seen": 112328704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007805656970912739, + "loss": 2.0351, + "theoretical_loss": 4.724629640001672, + "tokens_seen": 112394240 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007805496489468406, + "loss": 1.9868, + "theoretical_loss": 4.724256348376242, + "tokens_seen": 112459776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007805336008024073, + "loss": 1.8331, + "theoretical_loss": 4.723883335092982, + "tokens_seen": 112525312 + }, + { + "epoch": 0.03, + "learning_rate": 0.000780517552657974, + "loss": 2.0438, + "theoretical_loss": 4.723510599782419, + "tokens_seen": 112590848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007805015045135407, + "loss": 2.025, + "theoretical_loss": 4.723138142075788, + "tokens_seen": 112656384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007804854563691074, + "loss": 1.9763, + "theoretical_loss": 4.722765961605024, + "tokens_seen": 112721920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007804694082246741, + "loss": 2.0574, + "theoretical_loss": 4.722394058002765, + "tokens_seen": 112787456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007804533600802408, + "loss": 2.0758, + "theoretical_loss": 4.7220224309023475, + "tokens_seen": 112852992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007804373119358075, + "loss": 1.9396, + "theoretical_loss": 4.721651079937811, + "tokens_seen": 112918528 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007804212637913742, + "loss": 2.0622, + "theoretical_loss": 4.7212800047438845, + "tokens_seen": 112984064 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007804052156469409, + "loss": 1.9564, + "theoretical_loss": 4.720909204955996, + "tokens_seen": 113049600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007803891675025076, + "loss": 1.9568, + "theoretical_loss": 4.720538680210268, + "tokens_seen": 113115136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007803731193580743, + "loss": 1.9995, + "theoretical_loss": 4.720168430143508, + "tokens_seen": 113180672 + }, + { + "epoch": 0.03, + "learning_rate": 0.000780357071213641, + "loss": 1.9725, + "theoretical_loss": 4.719798454393219, + "tokens_seen": 113246208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007803410230692077, + "loss": 2.0178, + "theoretical_loss": 4.719428752597588, + "tokens_seen": 113311744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007803249749247744, + "loss": 1.8998, + "theoretical_loss": 4.719059324395493, + "tokens_seen": 113377280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007803089267803411, + "loss": 1.959, + "theoretical_loss": 4.71869016942649, + "tokens_seen": 113442816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007802928786359078, + "loss": 1.9718, + "theoretical_loss": 4.718321287330823, + "tokens_seen": 113508352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007802768304914745, + "loss": 2.0147, + "theoretical_loss": 4.7179526777494125, + "tokens_seen": 113573888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007802607823470412, + "loss": 1.9792, + "theoretical_loss": 4.717584340323864, + "tokens_seen": 113639424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007802447342026079, + "loss": 1.8801, + "theoretical_loss": 4.7172162746964545, + "tokens_seen": 113704960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007802286860581746, + "loss": 2.0341, + "theoretical_loss": 4.716848480510142, + "tokens_seen": 113770496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007802126379137413, + "loss": 1.9524, + "theoretical_loss": 4.716480957408556, + "tokens_seen": 113836032 + }, + { + "epoch": 0.03, + "learning_rate": 0.000780196589769308, + "loss": 1.8471, + "theoretical_loss": 4.716113705035999, + "tokens_seen": 113901568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007801805416248747, + "loss": 1.9279, + "theoretical_loss": 4.715746723037448, + "tokens_seen": 113967104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007801644934804414, + "loss": 2.0102, + "theoretical_loss": 4.715380011058542, + "tokens_seen": 114032640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007801484453360081, + "loss": 1.9055, + "theoretical_loss": 4.715013568745597, + "tokens_seen": 114098176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007801323971915748, + "loss": 1.965, + "theoretical_loss": 4.714647395745587, + "tokens_seen": 114163712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007801163490471415, + "loss": 1.9298, + "theoretical_loss": 4.714281491706154, + "tokens_seen": 114229248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007801003009027082, + "loss": 2.0152, + "theoretical_loss": 4.713915856275603, + "tokens_seen": 114294784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007800842527582749, + "loss": 1.9015, + "theoretical_loss": 4.713550489102901, + "tokens_seen": 114360320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007800682046138416, + "loss": 1.8637, + "theoretical_loss": 4.7131853898376725, + "tokens_seen": 114425856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007800521564694083, + "loss": 1.9677, + "theoretical_loss": 4.7128205581302005, + "tokens_seen": 114491392 + }, + { + "epoch": 0.03, + "learning_rate": 0.000780036108324975, + "loss": 1.9585, + "theoretical_loss": 4.712455993631425, + "tokens_seen": 114556928 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007800200601805417, + "loss": 2.0187, + "theoretical_loss": 4.712091695992941, + "tokens_seen": 114622464 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.995287299156189, + "objective/train/avg_token_score": 0.004712699446827173, + "objective/train/docs_used": 48994, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.974547266960144, + "objective/train/original_loss": 1.9833984375, + "objective/train/theoretical_loss": 4.711727664866995, + "objective/train/tokens_used": 135148000, + "objective/train/unlikelihood_loss": 0.009570197202265263, + "theoretical_loss": 4.711727664866995, + "tokens_seen": 114688000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007800040120361084, + "loss": 1.9197, + "theoretical_loss": 4.711727664866995, + "tokens_seen": 114688000 + }, + { + "epoch": 0.03, + "learning_rate": 0.000779987963891675, + "loss": 1.9546, + "theoretical_loss": 4.7113638999064875, + "tokens_seen": 114753536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007799719157472418, + "loss": 2.0039, + "theoretical_loss": 4.711000400764967, + "tokens_seen": 114819072 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007799558676028084, + "loss": 1.8988, + "theoretical_loss": 4.71063716709663, + "tokens_seen": 114884608 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007799398194583752, + "loss": 2.1003, + "theoretical_loss": 4.7102741985563235, + "tokens_seen": 114950144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007799237713139418, + "loss": 1.9992, + "theoretical_loss": 4.709911494799533, + "tokens_seen": 115015680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007799077231695086, + "loss": 2.0823, + "theoretical_loss": 4.709549055482395, + "tokens_seen": 115081216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007798916750250752, + "loss": 1.9933, + "theoretical_loss": 4.709186880261683, + "tokens_seen": 115146752 + }, + { + "epoch": 0.03, + "learning_rate": 0.000779875626880642, + "loss": 1.9298, + "theoretical_loss": 4.708824968794813, + "tokens_seen": 115212288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007798595787362086, + "loss": 1.761, + "theoretical_loss": 4.70846332073984, + "tokens_seen": 115277824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007798435305917753, + "loss": 1.948, + "theoretical_loss": 4.708101935755455, + "tokens_seen": 115343360 + }, + { + "epoch": 0.03, + "learning_rate": 0.000779827482447342, + "loss": 1.9683, + "theoretical_loss": 4.707740813500988, + "tokens_seen": 115408896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007798114343029087, + "loss": 2.0637, + "theoretical_loss": 4.707379953636399, + "tokens_seen": 115474432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007797953861584754, + "loss": 1.8723, + "theoretical_loss": 4.7070193558222835, + "tokens_seen": 115539968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007797793380140421, + "loss": 1.9926, + "theoretical_loss": 4.7066590197198686, + "tokens_seen": 115605504 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007797632898696088, + "loss": 1.9238, + "theoretical_loss": 4.706298944991008, + "tokens_seen": 115671040 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007797472417251755, + "loss": 1.9326, + "theoretical_loss": 4.705939131298189, + "tokens_seen": 115736576 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007797311935807422, + "loss": 2.1148, + "theoretical_loss": 4.705579578304519, + "tokens_seen": 115802112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007797151454363089, + "loss": 2.0022, + "theoretical_loss": 4.705220285673736, + "tokens_seen": 115867648 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007796990972918756, + "loss": 2.0581, + "theoretical_loss": 4.704861253070198, + "tokens_seen": 115933184 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007796830491474423, + "loss": 2.0226, + "theoretical_loss": 4.704502480158887, + "tokens_seen": 115998720 + }, + { + "epoch": 0.04, + "learning_rate": 0.000779667001003009, + "loss": 2.0915, + "theoretical_loss": 4.704143966605406, + "tokens_seen": 116064256 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007796509528585757, + "loss": 1.9097, + "theoretical_loss": 4.703785712075973, + "tokens_seen": 116129792 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007796349047141424, + "loss": 2.182, + "theoretical_loss": 4.703427716237432, + "tokens_seen": 116195328 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007796188565697091, + "loss": 2.004, + "theoretical_loss": 4.703069978757233, + "tokens_seen": 116260864 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007796028084252758, + "loss": 1.9416, + "theoretical_loss": 4.702712499303448, + "tokens_seen": 116326400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007795867602808425, + "loss": 2.062, + "theoretical_loss": 4.7023552775447595, + "tokens_seen": 116391936 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007795707121364092, + "loss": 2.0896, + "theoretical_loss": 4.701998313150462, + "tokens_seen": 116457472 + }, + { + "epoch": 0.04, + "learning_rate": 0.000779554663991976, + "loss": 1.9399, + "theoretical_loss": 4.701641605790459, + "tokens_seen": 116523008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007795386158475426, + "loss": 2.0257, + "theoretical_loss": 4.701285155135265, + "tokens_seen": 116588544 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007795225677031094, + "loss": 1.9313, + "theoretical_loss": 4.700928960856, + "tokens_seen": 116654080 + }, + { + "epoch": 0.04, + "learning_rate": 0.000779506519558676, + "loss": 1.9688, + "theoretical_loss": 4.70057302262439, + "tokens_seen": 116719616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007794904714142428, + "loss": 1.8724, + "theoretical_loss": 4.700217340112767, + "tokens_seen": 116785152 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007794744232698094, + "loss": 1.9668, + "theoretical_loss": 4.699861912994063, + "tokens_seen": 116850688 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007794583751253762, + "loss": 1.955, + "theoretical_loss": 4.699506740941813, + "tokens_seen": 116916224 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007794423269809428, + "loss": 2.016, + "theoretical_loss": 4.699151823630155, + "tokens_seen": 116981760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007794262788365096, + "loss": 1.8386, + "theoretical_loss": 4.698797160733818, + "tokens_seen": 117047296 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007794102306920762, + "loss": 1.9407, + "theoretical_loss": 4.6984427519281375, + "tokens_seen": 117112832 + }, + { + "epoch": 0.04, + "learning_rate": 0.000779394182547643, + "loss": 2.1882, + "theoretical_loss": 4.698088596889037, + "tokens_seen": 117178368 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007793781344032096, + "loss": 1.9077, + "theoretical_loss": 4.697734695293039, + "tokens_seen": 117243904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007793620862587764, + "loss": 2.0422, + "theoretical_loss": 4.697381046817256, + "tokens_seen": 117309440 + }, + { + "epoch": 0.04, + "learning_rate": 0.000779346038114343, + "loss": 1.8936, + "theoretical_loss": 4.697027651139393, + "tokens_seen": 117374976 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007793299899699098, + "loss": 1.9867, + "theoretical_loss": 4.696674507937746, + "tokens_seen": 117440512 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007793139418254765, + "loss": 1.8322, + "theoretical_loss": 4.696321616891199, + "tokens_seen": 117506048 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007792978936810432, + "loss": 2.0275, + "theoretical_loss": 4.695968977679221, + "tokens_seen": 117571584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007792818455366099, + "loss": 2.0173, + "theoretical_loss": 4.695616589981871, + "tokens_seen": 117637120 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007792657973921766, + "loss": 1.9012, + "theoretical_loss": 4.695264453479789, + "tokens_seen": 117702656 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007792497492477433, + "loss": 1.883, + "theoretical_loss": 4.6949125678542, + "tokens_seen": 117768192 + }, + { + "epoch": 0.04, + "learning_rate": 0.00077923370110331, + "loss": 2.0783, + "theoretical_loss": 4.69456093278691, + "tokens_seen": 117833728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007792176529588767, + "loss": 1.9534, + "theoretical_loss": 4.694209547960304, + "tokens_seen": 117899264 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.9920607209205627, + "objective/train/avg_token_score": 0.007939248345792294, + "objective/train/docs_used": 50256, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 2.0465433597564697, + "objective/train/original_loss": 2.0645923614501953, + "objective/train/theoretical_loss": 4.693858413057347, + "objective/train/tokens_used": 138424800, + "objective/train/unlikelihood_loss": 0.012900648638606071, + "theoretical_loss": 4.693858413057347, + "tokens_seen": 117964800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007792016048144434, + "loss": 2.0261, + "theoretical_loss": 4.693858413057347, + "tokens_seen": 117964800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007791855566700101, + "loss": 2.0586, + "theoretical_loss": 4.693507527761584, + "tokens_seen": 118030336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007791695085255768, + "loss": 2.02, + "theoretical_loss": 4.6931568917571305, + "tokens_seen": 118095872 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007791534603811435, + "loss": 1.9103, + "theoretical_loss": 4.692806504728683, + "tokens_seen": 118161408 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007791374122367102, + "loss": 2.0529, + "theoretical_loss": 4.692456366361506, + "tokens_seen": 118226944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007791213640922769, + "loss": 2.0604, + "theoretical_loss": 4.692106476341438, + "tokens_seen": 118292480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007791053159478436, + "loss": 1.8558, + "theoretical_loss": 4.691756834354891, + "tokens_seen": 118358016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007790892678034103, + "loss": 1.9873, + "theoretical_loss": 4.691407440088843, + "tokens_seen": 118423552 + }, + { + "epoch": 0.04, + "learning_rate": 0.000779073219658977, + "loss": 1.8984, + "theoretical_loss": 4.69105829323084, + "tokens_seen": 118489088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007790571715145437, + "loss": 2.0271, + "theoretical_loss": 4.690709393468995, + "tokens_seen": 118554624 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007790411233701104, + "loss": 1.9443, + "theoretical_loss": 4.690360740491989, + "tokens_seen": 118620160 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007790250752256771, + "loss": 1.8848, + "theoretical_loss": 4.690012333989063, + "tokens_seen": 118685696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007790090270812438, + "loss": 1.9767, + "theoretical_loss": 4.689664173650023, + "tokens_seen": 118751232 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007789929789368105, + "loss": 1.9493, + "theoretical_loss": 4.689316259165235, + "tokens_seen": 118816768 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007789769307923772, + "loss": 1.9381, + "theoretical_loss": 4.6889685902256275, + "tokens_seen": 118882304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007789608826479439, + "loss": 1.9142, + "theoretical_loss": 4.688621166522684, + "tokens_seen": 118947840 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007789448345035106, + "loss": 1.9908, + "theoretical_loss": 4.688273987748448, + "tokens_seen": 119013376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007789287863590773, + "loss": 2.0286, + "theoretical_loss": 4.687927053595518, + "tokens_seen": 119078912 + }, + { + "epoch": 0.04, + "learning_rate": 0.000778912738214644, + "loss": 2.052, + "theoretical_loss": 4.687580363757048, + "tokens_seen": 119144448 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007788966900702107, + "loss": 1.9248, + "theoretical_loss": 4.6872339179267435, + "tokens_seen": 119209984 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007788806419257774, + "loss": 1.9845, + "theoretical_loss": 4.686887715798864, + "tokens_seen": 119275520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007788645937813441, + "loss": 1.8534, + "theoretical_loss": 4.6865417570682215, + "tokens_seen": 119341056 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007788485456369108, + "loss": 1.8646, + "theoretical_loss": 4.686196041430172, + "tokens_seen": 119406592 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007788324974924775, + "loss": 1.98, + "theoretical_loss": 4.685850568580624, + "tokens_seen": 119472128 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007788164493480442, + "loss": 1.876, + "theoretical_loss": 4.685505338216032, + "tokens_seen": 119537664 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007788004012036109, + "loss": 2.0501, + "theoretical_loss": 4.685160350033398, + "tokens_seen": 119603200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007787843530591776, + "loss": 1.8518, + "theoretical_loss": 4.684815603730263, + "tokens_seen": 119668736 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007787683049147443, + "loss": 1.9016, + "theoretical_loss": 4.684471099004718, + "tokens_seen": 119734272 + }, + { + "epoch": 0.04, + "learning_rate": 0.000778752256770311, + "loss": 1.9405, + "theoretical_loss": 4.684126835555389, + "tokens_seen": 119799808 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007787362086258777, + "loss": 2.1534, + "theoretical_loss": 4.683782813081448, + "tokens_seen": 119865344 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007787201604814444, + "loss": 1.9657, + "theoretical_loss": 4.683439031282606, + "tokens_seen": 119930880 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007787041123370111, + "loss": 2.013, + "theoretical_loss": 4.683095489859107, + "tokens_seen": 119996416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007786880641925778, + "loss": 1.8332, + "theoretical_loss": 4.682752188511737, + "tokens_seen": 120061952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007786720160481445, + "loss": 1.93, + "theoretical_loss": 4.682409126941815, + "tokens_seen": 120127488 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007786559679037112, + "loss": 2.0016, + "theoretical_loss": 4.682066304851195, + "tokens_seen": 120193024 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007786399197592779, + "loss": 1.9397, + "theoretical_loss": 4.681723721942264, + "tokens_seen": 120258560 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007786238716148446, + "loss": 2.0819, + "theoretical_loss": 4.6813813779179405, + "tokens_seen": 120324096 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007786078234704113, + "loss": 1.9585, + "theoretical_loss": 4.681039272481674, + "tokens_seen": 120389632 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007785917753259779, + "loss": 2.026, + "theoretical_loss": 4.680697405337443, + "tokens_seen": 120455168 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007785757271815447, + "loss": 2.0493, + "theoretical_loss": 4.680355776189755, + "tokens_seen": 120520704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007785596790371113, + "loss": 1.9649, + "theoretical_loss": 4.680014384743641, + "tokens_seen": 120586240 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007785436308926781, + "loss": 1.7459, + "theoretical_loss": 4.679673230704664, + "tokens_seen": 120651776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007785275827482447, + "loss": 2.0063, + "theoretical_loss": 4.679332313778904, + "tokens_seen": 120717312 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007785115346038115, + "loss": 2.0276, + "theoretical_loss": 4.678991633672968, + "tokens_seen": 120782848 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007784954864593781, + "loss": 1.9186, + "theoretical_loss": 4.678651190093988, + "tokens_seen": 120848384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007784794383149449, + "loss": 1.8818, + "theoretical_loss": 4.67831098274961, + "tokens_seen": 120913920 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007784633901705115, + "loss": 2.1124, + "theoretical_loss": 4.677971011348005, + "tokens_seen": 120979456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007784473420260782, + "loss": 2.0425, + "theoretical_loss": 4.677631275597859, + "tokens_seen": 121044992 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007784312938816449, + "loss": 1.8941, + "theoretical_loss": 4.677291775208378, + "tokens_seen": 121110528 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007784152457372116, + "loss": 1.9936, + "theoretical_loss": 4.676952509889281, + "tokens_seen": 121176064 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.9931885004043579, + "objective/train/avg_token_score": 0.0068114441819489, + "objective/train/docs_used": 51395, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.9956903457641602, + "objective/train/original_loss": 2.0097174644470215, + "objective/train/theoretical_loss": 4.676613479350804, + "objective/train/tokens_used": 141701600, + "objective/train/unlikelihood_loss": 0.011152202263474464, + "theoretical_loss": 4.676613479350804, + "tokens_seen": 121241600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007783991975927783, + "loss": 1.9862, + "theoretical_loss": 4.676613479350804, + "tokens_seen": 121241600 + }, + { + "epoch": 0.04, + "learning_rate": 0.000778383149448345, + "loss": 1.8904, + "theoretical_loss": 4.6762746833036966, + "tokens_seen": 121307136 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007783671013039117, + "loss": 1.9319, + "theoretical_loss": 4.675936121459218, + "tokens_seen": 121372672 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007783510531594785, + "loss": 1.8464, + "theoretical_loss": 4.675597793529143, + "tokens_seen": 121438208 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007783350050150451, + "loss": 1.9173, + "theoretical_loss": 4.675259699225752, + "tokens_seen": 121503744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007783189568706119, + "loss": 1.9266, + "theoretical_loss": 4.674921838261836, + "tokens_seen": 121569280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007783029087261785, + "loss": 1.982, + "theoretical_loss": 4.674584210350695, + "tokens_seen": 121634816 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007782868605817453, + "loss": 1.9324, + "theoretical_loss": 4.674246815206136, + "tokens_seen": 121700352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007782708124373119, + "loss": 2.0043, + "theoretical_loss": 4.673909652542466, + "tokens_seen": 121765888 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007782547642928787, + "loss": 1.8703, + "theoretical_loss": 4.673572722074502, + "tokens_seen": 121831424 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007782387161484453, + "loss": 1.9976, + "theoretical_loss": 4.673236023517562, + "tokens_seen": 121896960 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007782226680040121, + "loss": 1.8657, + "theoretical_loss": 4.672899556587465, + "tokens_seen": 121962496 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007782066198595787, + "loss": 1.9397, + "theoretical_loss": 4.672563321000533, + "tokens_seen": 122028032 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007781905717151455, + "loss": 1.8994, + "theoretical_loss": 4.672227316473584, + "tokens_seen": 122093568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007781745235707121, + "loss": 1.9494, + "theoretical_loss": 4.671891542723937, + "tokens_seen": 122159104 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007781584754262789, + "loss": 1.9126, + "theoretical_loss": 4.671555999469408, + "tokens_seen": 122224640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007781424272818455, + "loss": 2.129, + "theoretical_loss": 4.671220686428308, + "tokens_seen": 122290176 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007781263791374123, + "loss": 1.9488, + "theoretical_loss": 4.670885603319445, + "tokens_seen": 122355712 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007781103309929789, + "loss": 1.8551, + "theoretical_loss": 4.670550749862119, + "tokens_seen": 122421248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007780942828485457, + "loss": 1.8591, + "theoretical_loss": 4.670216125776122, + "tokens_seen": 122486784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007780782347041123, + "loss": 1.99, + "theoretical_loss": 4.669881730781741, + "tokens_seen": 122552320 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007780621865596791, + "loss": 1.8899, + "theoretical_loss": 4.66954756459975, + "tokens_seen": 122617856 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007780461384152457, + "loss": 1.8902, + "theoretical_loss": 4.669213626951413, + "tokens_seen": 122683392 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007780300902708125, + "loss": 1.9371, + "theoretical_loss": 4.668879917558484, + "tokens_seen": 122748928 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007780140421263791, + "loss": 1.9274, + "theoretical_loss": 4.6685464361432025, + "tokens_seen": 122814464 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007779979939819459, + "loss": 1.9736, + "theoretical_loss": 4.668213182428295, + "tokens_seen": 122880000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007779819458375126, + "loss": 2.0389, + "theoretical_loss": 4.667880156136971, + "tokens_seen": 122945536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007779658976930793, + "loss": 1.9523, + "theoretical_loss": 4.667547356992927, + "tokens_seen": 123011072 + }, + { + "epoch": 0.04, + "learning_rate": 0.000777949849548646, + "loss": 1.9356, + "theoretical_loss": 4.667214784720338, + "tokens_seen": 123076608 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007779338014042127, + "loss": 1.8716, + "theoretical_loss": 4.666882439043865, + "tokens_seen": 123142144 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007779177532597794, + "loss": 2.0173, + "theoretical_loss": 4.666550319688646, + "tokens_seen": 123207680 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007779017051153461, + "loss": 1.9753, + "theoretical_loss": 4.6662184263803, + "tokens_seen": 123273216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007778856569709128, + "loss": 1.9164, + "theoretical_loss": 4.665886758844923, + "tokens_seen": 123338752 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007778696088264795, + "loss": 1.9536, + "theoretical_loss": 4.66555531680909, + "tokens_seen": 123404288 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007778535606820462, + "loss": 2.0146, + "theoretical_loss": 4.665224099999852, + "tokens_seen": 123469824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007778375125376129, + "loss": 1.8394, + "theoretical_loss": 4.664893108144735, + "tokens_seen": 123535360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007778214643931796, + "loss": 1.9771, + "theoretical_loss": 4.664562340971735, + "tokens_seen": 123600896 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007778054162487463, + "loss": 1.8866, + "theoretical_loss": 4.664231798209327, + "tokens_seen": 123666432 + }, + { + "epoch": 0.04, + "learning_rate": 0.000777789368104313, + "loss": 2.1227, + "theoretical_loss": 4.663901479586454, + "tokens_seen": 123731968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007777733199598797, + "loss": 1.8939, + "theoretical_loss": 4.663571384832531, + "tokens_seen": 123797504 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007777572718154464, + "loss": 1.9165, + "theoretical_loss": 4.663241513677443, + "tokens_seen": 123863040 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007777412236710131, + "loss": 1.9573, + "theoretical_loss": 4.662911865851541, + "tokens_seen": 123928576 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007777251755265798, + "loss": 1.8683, + "theoretical_loss": 4.662582441085648, + "tokens_seen": 123994112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007777091273821465, + "loss": 1.8389, + "theoretical_loss": 4.662253239111049, + "tokens_seen": 124059648 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007776930792377132, + "loss": 1.9649, + "theoretical_loss": 4.661924259659497, + "tokens_seen": 124125184 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007776770310932799, + "loss": 1.9505, + "theoretical_loss": 4.661595502463211, + "tokens_seen": 124190720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007776609829488466, + "loss": 1.9084, + "theoretical_loss": 4.661266967254868, + "tokens_seen": 124256256 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007776449348044133, + "loss": 1.8643, + "theoretical_loss": 4.660938653767614, + "tokens_seen": 124321792 + }, + { + "epoch": 0.04, + "learning_rate": 0.00077762888665998, + "loss": 2.0201, + "theoretical_loss": 4.660610561735049, + "tokens_seen": 124387328 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007776128385155467, + "loss": 1.9448, + "theoretical_loss": 4.66028269089124, + "tokens_seen": 124452864 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.9943526387214661, + "objective/train/avg_token_score": 0.005647291895002127, + "objective/train/docs_used": 52572, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.8987584114074707, + "objective/train/original_loss": 1.909789800643921, + "objective/train/theoretical_loss": 4.659955040970709, + "objective/train/tokens_used": 144978400, + "objective/train/unlikelihood_loss": 0.01116106752306223, + "theoretical_loss": 4.659955040970709, + "tokens_seen": 124518400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007775967903711134, + "loss": 1.8916, + "theoretical_loss": 4.659955040970709, + "tokens_seen": 124518400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007775807422266801, + "loss": 1.8951, + "theoretical_loss": 4.659627611708437, + "tokens_seen": 124583936 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007775646940822468, + "loss": 2.0506, + "theoretical_loss": 4.659300402839863, + "tokens_seen": 124649472 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007775486459378135, + "loss": 1.9576, + "theoretical_loss": 4.658973414100881, + "tokens_seen": 124715008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007775325977933802, + "loss": 2.0815, + "theoretical_loss": 4.658646645227842, + "tokens_seen": 124780544 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007775165496489469, + "loss": 1.9385, + "theoretical_loss": 4.6583200959575475, + "tokens_seen": 124846080 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007775005015045136, + "loss": 1.9266, + "theoretical_loss": 4.657993766027255, + "tokens_seen": 124911616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007774844533600803, + "loss": 2.0223, + "theoretical_loss": 4.657667655174674, + "tokens_seen": 124977152 + }, + { + "epoch": 0.04, + "learning_rate": 0.000777468405215647, + "loss": 2.0204, + "theoretical_loss": 4.657341763137963, + "tokens_seen": 125042688 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007774523570712137, + "loss": 2.0073, + "theoretical_loss": 4.657016089655732, + "tokens_seen": 125108224 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007774363089267804, + "loss": 2.0001, + "theoretical_loss": 4.65669063446704, + "tokens_seen": 125173760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007774202607823471, + "loss": 1.9731, + "theoretical_loss": 4.656365397311393, + "tokens_seen": 125239296 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007774042126379138, + "loss": 1.9551, + "theoretical_loss": 4.656040377928743, + "tokens_seen": 125304832 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007773881644934805, + "loss": 1.7977, + "theoretical_loss": 4.655715576059493, + "tokens_seen": 125370368 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007773721163490472, + "loss": 1.9434, + "theoretical_loss": 4.655390991444484, + "tokens_seen": 125435904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007773560682046139, + "loss": 1.9629, + "theoretical_loss": 4.6550666238250065, + "tokens_seen": 125501440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007773400200601806, + "loss": 1.9035, + "theoretical_loss": 4.65474247294279, + "tokens_seen": 125566976 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007773239719157473, + "loss": 1.9207, + "theoretical_loss": 4.654418538540009, + "tokens_seen": 125632512 + }, + { + "epoch": 0.04, + "learning_rate": 0.000777307923771314, + "loss": 1.9941, + "theoretical_loss": 4.654094820359278, + "tokens_seen": 125698048 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007772918756268807, + "loss": 2.0066, + "theoretical_loss": 4.653771318143651, + "tokens_seen": 125763584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007772758274824474, + "loss": 1.9337, + "theoretical_loss": 4.65344803163662, + "tokens_seen": 125829120 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007772597793380141, + "loss": 1.9354, + "theoretical_loss": 4.653124960582118, + "tokens_seen": 125894656 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007772437311935808, + "loss": 1.994, + "theoretical_loss": 4.652802104724513, + "tokens_seen": 125960192 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007772276830491475, + "loss": 1.9473, + "theoretical_loss": 4.652479463808609, + "tokens_seen": 126025728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007772116349047142, + "loss": 1.876, + "theoretical_loss": 4.652157037579647, + "tokens_seen": 126091264 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007771955867602808, + "loss": 1.9148, + "theoretical_loss": 4.6518348257833, + "tokens_seen": 126156800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007771795386158476, + "loss": 1.8408, + "theoretical_loss": 4.651512828165675, + "tokens_seen": 126222336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007771634904714142, + "loss": 2.0042, + "theoretical_loss": 4.651191044473311, + "tokens_seen": 126287872 + }, + { + "epoch": 0.04, + "learning_rate": 0.000777147442326981, + "loss": 1.8935, + "theoretical_loss": 4.65086947445318, + "tokens_seen": 126353408 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007771313941825476, + "loss": 1.7249, + "theoretical_loss": 4.650548117852681, + "tokens_seen": 126418944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007771153460381144, + "loss": 1.9621, + "theoretical_loss": 4.650226974419646, + "tokens_seen": 126484480 + }, + { + "epoch": 0.04, + "learning_rate": 0.000777099297893681, + "loss": 1.982, + "theoretical_loss": 4.649906043902332, + "tokens_seen": 126550016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007770832497492478, + "loss": 2.019, + "theoretical_loss": 4.6495853260494275, + "tokens_seen": 126615552 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007770672016048144, + "loss": 1.9446, + "theoretical_loss": 4.649264820610043, + "tokens_seen": 126681088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007770511534603812, + "loss": 1.8728, + "theoretical_loss": 4.648944527333717, + "tokens_seen": 126746624 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007770351053159478, + "loss": 1.9343, + "theoretical_loss": 4.648624445970414, + "tokens_seen": 126812160 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007770190571715146, + "loss": 1.8517, + "theoretical_loss": 4.648304576270519, + "tokens_seen": 126877696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007770030090270812, + "loss": 2.0128, + "theoretical_loss": 4.647984917984841, + "tokens_seen": 126943232 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776986960882648, + "loss": 2.0079, + "theoretical_loss": 4.647665470864613, + "tokens_seen": 127008768 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007769709127382146, + "loss": 1.9076, + "theoretical_loss": 4.6473462346614856, + "tokens_seen": 127074304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007769548645937814, + "loss": 2.0027, + "theoretical_loss": 4.647027209127531, + "tokens_seen": 127139840 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776938816449348, + "loss": 1.9153, + "theoretical_loss": 4.646708394015242, + "tokens_seen": 127205376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007769227683049148, + "loss": 1.9035, + "theoretical_loss": 4.646389789077524, + "tokens_seen": 127270912 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007769067201604814, + "loss": 1.9625, + "theoretical_loss": 4.646071394067707, + "tokens_seen": 127336448 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007768906720160482, + "loss": 1.9056, + "theoretical_loss": 4.645753208739533, + "tokens_seen": 127401984 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007768746238716148, + "loss": 1.8727, + "theoretical_loss": 4.645435232847159, + "tokens_seen": 127467520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007768585757271816, + "loss": 1.8746, + "theoretical_loss": 4.645117466145157, + "tokens_seen": 127533056 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007768425275827482, + "loss": 1.8143, + "theoretical_loss": 4.6447999083885145, + "tokens_seen": 127598592 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776826479438315, + "loss": 1.9902, + "theoretical_loss": 4.644482559332629, + "tokens_seen": 127664128 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007768104312938816, + "loss": 1.9712, + "theoretical_loss": 4.644165418733312, + "tokens_seen": 127729664 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.9920344948768616, + "objective/train/avg_token_score": 0.007965493947267532, + "objective/train/docs_used": 53741, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.9757416248321533, + "objective/train/original_loss": 1.9907881021499634, + "objective/train/theoretical_loss": 4.6438484863467835, + "objective/train/tokens_used": 148255200, + "objective/train/unlikelihood_loss": 0.015104726888239384, + "theoretical_loss": 4.6438484863467835, + "tokens_seen": 127795200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007767943831494484, + "loss": 2.0286, + "theoretical_loss": 4.6438484863467835, + "tokens_seen": 127795200 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776778335005015, + "loss": 2.0723, + "theoretical_loss": 4.643531761929674, + "tokens_seen": 127860736 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007767622868605818, + "loss": 1.8505, + "theoretical_loss": 4.643215245239025, + "tokens_seen": 127926272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007767462387161484, + "loss": 1.8488, + "theoretical_loss": 4.642898936032285, + "tokens_seen": 127991808 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007767301905717152, + "loss": 1.8663, + "theoretical_loss": 4.6425828340673085, + "tokens_seen": 128057344 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007767141424272818, + "loss": 2.0139, + "theoretical_loss": 4.642266939102357, + "tokens_seen": 128122880 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007766980942828486, + "loss": 1.8888, + "theoretical_loss": 4.641951250896098, + "tokens_seen": 128188416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007766820461384152, + "loss": 1.9302, + "theoretical_loss": 4.641635769207603, + "tokens_seen": 128253952 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776665997993982, + "loss": 2.0022, + "theoretical_loss": 4.641320493796349, + "tokens_seen": 128319488 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007766499498495486, + "loss": 1.8524, + "theoretical_loss": 4.641005424422213, + "tokens_seen": 128385024 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007766339017051154, + "loss": 1.8995, + "theoretical_loss": 4.640690560845474, + "tokens_seen": 128450560 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776617853560682, + "loss": 1.962, + "theoretical_loss": 4.640375902826815, + "tokens_seen": 128516096 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007766018054162488, + "loss": 2.0749, + "theoretical_loss": 4.640061450127316, + "tokens_seen": 128581632 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007765857572718155, + "loss": 1.857, + "theoretical_loss": 4.639747202508459, + "tokens_seen": 128647168 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007765697091273822, + "loss": 1.956, + "theoretical_loss": 4.639433159732121, + "tokens_seen": 128712704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007765536609829489, + "loss": 2.0137, + "theoretical_loss": 4.639119321560581, + "tokens_seen": 128778240 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007765376128385156, + "loss": 1.9457, + "theoretical_loss": 4.638805687756511, + "tokens_seen": 128843776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007765215646940823, + "loss": 1.8712, + "theoretical_loss": 4.63849225808298, + "tokens_seen": 128909312 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776505516549649, + "loss": 1.9718, + "theoretical_loss": 4.6381790323034515, + "tokens_seen": 128974848 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007764894684052157, + "loss": 2.0709, + "theoretical_loss": 4.637866010181784, + "tokens_seen": 129040384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007764734202607824, + "loss": 1.9456, + "theoretical_loss": 4.63755319148223, + "tokens_seen": 129105920 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007764573721163491, + "loss": 1.9577, + "theoretical_loss": 4.637240575969432, + "tokens_seen": 129171456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007764413239719158, + "loss": 1.8774, + "theoretical_loss": 4.636928163408426, + "tokens_seen": 129236992 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007764252758274825, + "loss": 2.0029, + "theoretical_loss": 4.636615953564637, + "tokens_seen": 129302528 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007764092276830492, + "loss": 1.7833, + "theoretical_loss": 4.636303946203881, + "tokens_seen": 129368064 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007763931795386159, + "loss": 1.9804, + "theoretical_loss": 4.635992141092364, + "tokens_seen": 129433600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007763771313941826, + "loss": 1.8699, + "theoretical_loss": 4.635680537996679, + "tokens_seen": 129499136 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007763610832497493, + "loss": 1.9588, + "theoretical_loss": 4.635369136683805, + "tokens_seen": 129564672 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776345035105316, + "loss": 1.8819, + "theoretical_loss": 4.6350579369211085, + "tokens_seen": 129630208 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007763289869608827, + "loss": 1.9609, + "theoretical_loss": 4.634746938476344, + "tokens_seen": 129695744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007763129388164494, + "loss": 1.8506, + "theoretical_loss": 4.634436141117645, + "tokens_seen": 129761280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007762968906720161, + "loss": 2.0182, + "theoretical_loss": 4.634125544613536, + "tokens_seen": 129826816 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007762808425275828, + "loss": 1.912, + "theoretical_loss": 4.63381514873292, + "tokens_seen": 129892352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007762647943831495, + "loss": 2.0762, + "theoretical_loss": 4.633504953245085, + "tokens_seen": 129957888 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007762487462387162, + "loss": 1.9946, + "theoretical_loss": 4.633194957919697, + "tokens_seen": 130023424 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007762326980942829, + "loss": 2.0315, + "theoretical_loss": 4.632885162526806, + "tokens_seen": 130088960 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007762166499498496, + "loss": 1.9456, + "theoretical_loss": 4.63257556683684, + "tokens_seen": 130154496 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007762006018054163, + "loss": 1.9542, + "theoretical_loss": 4.632266170620607, + "tokens_seen": 130220032 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776184553660983, + "loss": 1.9965, + "theoretical_loss": 4.631956973649293, + "tokens_seen": 130285568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007761685055165497, + "loss": 2.0554, + "theoretical_loss": 4.631647975694461, + "tokens_seen": 130351104 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007761524573721164, + "loss": 2.0269, + "theoretical_loss": 4.631339176528051, + "tokens_seen": 130416640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007761364092276831, + "loss": 1.9062, + "theoretical_loss": 4.631030575922379, + "tokens_seen": 130482176 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007761203610832498, + "loss": 1.8487, + "theoretical_loss": 4.630722173650135, + "tokens_seen": 130547712 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007761043129388165, + "loss": 1.8513, + "theoretical_loss": 4.630413969484383, + "tokens_seen": 130613248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007760882647943832, + "loss": 1.7749, + "theoretical_loss": 4.630105963198563, + "tokens_seen": 130678784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007760722166499499, + "loss": 1.913, + "theoretical_loss": 4.629798154566485, + "tokens_seen": 130744320 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007760561685055166, + "loss": 1.8777, + "theoretical_loss": 4.62949054336233, + "tokens_seen": 130809856 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007760401203610833, + "loss": 2.0089, + "theoretical_loss": 4.629183129360653, + "tokens_seen": 130875392 + }, + { + "epoch": 0.04, + "learning_rate": 0.00077602407221665, + "loss": 1.9128, + "theoretical_loss": 4.628875912336378, + "tokens_seen": 130940928 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007760080240722167, + "loss": 1.894, + "theoretical_loss": 4.628568892064797, + "tokens_seen": 131006464 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.9924710392951965, + "objective/train/avg_token_score": 0.007528969086706638, + "objective/train/docs_used": 54897, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.9529361724853516, + "objective/train/original_loss": 1.9693180322647095, + "objective/train/theoretical_loss": 4.62826206832157, + "objective/train/tokens_used": 151532000, + "objective/train/unlikelihood_loss": 0.01245919056236744, + "theoretical_loss": 4.62826206832157, + "tokens_seen": 131072000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007759919759277834, + "loss": 1.8685, + "theoretical_loss": 4.62826206832157, + "tokens_seen": 131072000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007759759277833502, + "loss": 2.1258, + "theoretical_loss": 4.6279554408827295, + "tokens_seen": 131137536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007759598796389168, + "loss": 1.8474, + "theoretical_loss": 4.62764900952467, + "tokens_seen": 131203072 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007759438314944836, + "loss": 1.8349, + "theoretical_loss": 4.627342774024155, + "tokens_seen": 131268608 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007759277833500502, + "loss": 1.9859, + "theoretical_loss": 4.6270367341583105, + "tokens_seen": 131334144 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007759117352056169, + "loss": 1.8775, + "theoretical_loss": 4.626730889704632, + "tokens_seen": 131399680 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007758956870611836, + "loss": 1.8084, + "theoretical_loss": 4.626425240440973, + "tokens_seen": 131465216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007758796389167503, + "loss": 2.0331, + "theoretical_loss": 4.626119786145557, + "tokens_seen": 131530752 + }, + { + "epoch": 0.04, + "learning_rate": 0.000775863590772317, + "loss": 1.8946, + "theoretical_loss": 4.625814526596962, + "tokens_seen": 131596288 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007758475426278837, + "loss": 1.8936, + "theoretical_loss": 4.625509461574135, + "tokens_seen": 131661824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007758314944834504, + "loss": 2.0907, + "theoretical_loss": 4.625204590856377, + "tokens_seen": 131727360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007758154463390171, + "loss": 1.9491, + "theoretical_loss": 4.624899914223355, + "tokens_seen": 131792896 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007757993981945837, + "loss": 1.916, + "theoretical_loss": 4.624595431455091, + "tokens_seen": 131858432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007757833500501505, + "loss": 1.9313, + "theoretical_loss": 4.62429114233197, + "tokens_seen": 131923968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007757673019057171, + "loss": 1.9995, + "theoretical_loss": 4.623987046634729, + "tokens_seen": 131989504 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007757512537612839, + "loss": 1.8763, + "theoretical_loss": 4.623683144144465, + "tokens_seen": 132055040 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007757352056168505, + "loss": 1.907, + "theoretical_loss": 4.623379434642632, + "tokens_seen": 132120576 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007757191574724173, + "loss": 1.9078, + "theoretical_loss": 4.623075917911039, + "tokens_seen": 132186112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007757031093279839, + "loss": 1.9701, + "theoretical_loss": 4.622772593731848, + "tokens_seen": 132251648 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007756870611835507, + "loss": 1.9743, + "theoretical_loss": 4.622469461887577, + "tokens_seen": 132317184 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007756710130391173, + "loss": 1.9982, + "theoretical_loss": 4.622166522161096, + "tokens_seen": 132382720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007756549648946841, + "loss": 1.9007, + "theoretical_loss": 4.621863774335628, + "tokens_seen": 132448256 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007756389167502507, + "loss": 2.0438, + "theoretical_loss": 4.621561218194747, + "tokens_seen": 132513792 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007756228686058175, + "loss": 2.0228, + "theoretical_loss": 4.62125885352238, + "tokens_seen": 132579328 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007756068204613841, + "loss": 1.9653, + "theoretical_loss": 4.620956680102801, + "tokens_seen": 132644864 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007755907723169509, + "loss": 1.986, + "theoretical_loss": 4.620654697720637, + "tokens_seen": 132710400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007755747241725175, + "loss": 2.1176, + "theoretical_loss": 4.620352906160861, + "tokens_seen": 132775936 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007755586760280843, + "loss": 2.0907, + "theoretical_loss": 4.6200513052087935, + "tokens_seen": 132841472 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007755426278836509, + "loss": 2.0075, + "theoretical_loss": 4.6197498946501065, + "tokens_seen": 132907008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007755265797392177, + "loss": 1.9896, + "theoretical_loss": 4.6194486742708145, + "tokens_seen": 132972544 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007755105315947843, + "loss": 1.9629, + "theoretical_loss": 4.6191476438572785, + "tokens_seen": 133038080 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007754944834503511, + "loss": 1.9154, + "theoretical_loss": 4.618846803196206, + "tokens_seen": 133103616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007754784353059177, + "loss": 1.9091, + "theoretical_loss": 4.618546152074648, + "tokens_seen": 133169152 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007754623871614845, + "loss": 1.9298, + "theoretical_loss": 4.61824569028, + "tokens_seen": 133234688 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007754463390170511, + "loss": 1.9487, + "theoretical_loss": 4.617945417599998, + "tokens_seen": 133300224 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007754302908726179, + "loss": 1.8226, + "theoretical_loss": 4.617645333822722, + "tokens_seen": 133365760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007754142427281845, + "loss": 1.7814, + "theoretical_loss": 4.617345438736594, + "tokens_seen": 133431296 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007753981945837513, + "loss": 1.8846, + "theoretical_loss": 4.617045732130375, + "tokens_seen": 133496832 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007753821464393179, + "loss": 1.8339, + "theoretical_loss": 4.616746213793167, + "tokens_seen": 133562368 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007753660982948847, + "loss": 1.9894, + "theoretical_loss": 4.616446883514412, + "tokens_seen": 133627904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007753500501504513, + "loss": 1.8743, + "theoretical_loss": 4.616147741083891, + "tokens_seen": 133693440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007753340020060181, + "loss": 1.9071, + "theoretical_loss": 4.6158487862917195, + "tokens_seen": 133758976 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007753179538615847, + "loss": 1.8686, + "theoretical_loss": 4.615550018928353, + "tokens_seen": 133824512 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007753019057171515, + "loss": 2.0109, + "theoretical_loss": 4.615251438784584, + "tokens_seen": 133890048 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007752858575727181, + "loss": 1.9894, + "theoretical_loss": 4.61495304565154, + "tokens_seen": 133955584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007752698094282849, + "loss": 2.0029, + "theoretical_loss": 4.614654839320682, + "tokens_seen": 134021120 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007752537612838515, + "loss": 1.9146, + "theoretical_loss": 4.614356819583808, + "tokens_seen": 134086656 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007752377131394183, + "loss": 1.9992, + "theoretical_loss": 4.614058986233047, + "tokens_seen": 134152192 + }, + { + "epoch": 0.04, + "learning_rate": 0.000775221664994985, + "loss": 1.9831, + "theoretical_loss": 4.613761339060863, + "tokens_seen": 134217728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007752056168505517, + "loss": 1.8791, + "theoretical_loss": 4.613463877860051, + "tokens_seen": 134283264 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.9936538934707642, + "objective/train/avg_token_score": 0.006346104200929403, + "objective/train/docs_used": 56110, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.8996678590774536, + "objective/train/original_loss": 1.9137049913406372, + "objective/train/theoretical_loss": 4.61316660242374, + "objective/train/tokens_used": 154808800, + "objective/train/unlikelihood_loss": 0.010605230927467346, + "theoretical_loss": 4.61316660242374, + "tokens_seen": 134348800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007751895687061184, + "loss": 2.046, + "theoretical_loss": 4.61316660242374, + "tokens_seen": 134348800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007751735205616851, + "loss": 2.0054, + "theoretical_loss": 4.6128695125453865, + "tokens_seen": 134414336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007751574724172518, + "loss": 1.9307, + "theoretical_loss": 4.612572608018779, + "tokens_seen": 134479872 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007751414242728185, + "loss": 1.9035, + "theoretical_loss": 4.612275888638036, + "tokens_seen": 134545408 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007751253761283852, + "loss": 1.8852, + "theoretical_loss": 4.611979354197603, + "tokens_seen": 134610944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007751093279839519, + "loss": 1.8293, + "theoretical_loss": 4.6116830044922565, + "tokens_seen": 134676480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007750932798395186, + "loss": 1.942, + "theoretical_loss": 4.611386839317097, + "tokens_seen": 134742016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007750772316950853, + "loss": 1.9274, + "theoretical_loss": 4.611090858467554, + "tokens_seen": 134807552 + }, + { + "epoch": 0.04, + "learning_rate": 0.000775061183550652, + "loss": 1.7654, + "theoretical_loss": 4.610795061739384, + "tokens_seen": 134873088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007750451354062187, + "loss": 1.9261, + "theoretical_loss": 4.610499448928666, + "tokens_seen": 134938624 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007750290872617854, + "loss": 2.0755, + "theoretical_loss": 4.610204019831805, + "tokens_seen": 135004160 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007750130391173521, + "loss": 1.993, + "theoretical_loss": 4.609908774245532, + "tokens_seen": 135069696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007749969909729188, + "loss": 2.0332, + "theoretical_loss": 4.609613711966898, + "tokens_seen": 135135232 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007749809428284855, + "loss": 1.8812, + "theoretical_loss": 4.60931883279328, + "tokens_seen": 135200768 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007749648946840522, + "loss": 1.8438, + "theoretical_loss": 4.6090241365223745, + "tokens_seen": 135266304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007749488465396189, + "loss": 1.8721, + "theoretical_loss": 4.6087296229522, + "tokens_seen": 135331840 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007749327983951856, + "loss": 1.875, + "theoretical_loss": 4.608435291881099, + "tokens_seen": 135397376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007749167502507523, + "loss": 1.9265, + "theoretical_loss": 4.608141143107728, + "tokens_seen": 135462912 + }, + { + "epoch": 0.04, + "learning_rate": 0.000774900702106319, + "loss": 1.9401, + "theoretical_loss": 4.607847176431067, + "tokens_seen": 135528448 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007748846539618857, + "loss": 1.8482, + "theoretical_loss": 4.6075533916504146, + "tokens_seen": 135593984 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007748686058174524, + "loss": 1.9054, + "theoretical_loss": 4.607259788565385, + "tokens_seen": 135659520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007748525576730191, + "loss": 1.9677, + "theoretical_loss": 4.606966366975914, + "tokens_seen": 135725056 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007748365095285858, + "loss": 2.1538, + "theoretical_loss": 4.606673126682249, + "tokens_seen": 135790592 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007748204613841525, + "loss": 1.8823, + "theoretical_loss": 4.6063800674849595, + "tokens_seen": 135856128 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007748044132397192, + "loss": 1.926, + "theoretical_loss": 4.606087189184924, + "tokens_seen": 135921664 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007747883650952859, + "loss": 1.9474, + "theoretical_loss": 4.6057944915833415, + "tokens_seen": 135987200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007747723169508527, + "loss": 1.9564, + "theoretical_loss": 4.6055019744817205, + "tokens_seen": 136052736 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007747562688064193, + "loss": 1.9914, + "theoretical_loss": 4.605209637681886, + "tokens_seen": 136118272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007747402206619861, + "loss": 1.8008, + "theoretical_loss": 4.604917480985975, + "tokens_seen": 136183808 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007747241725175527, + "loss": 1.7928, + "theoretical_loss": 4.604625504196438, + "tokens_seen": 136249344 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007747081243731195, + "loss": 1.8896, + "theoretical_loss": 4.604333707116033, + "tokens_seen": 136314880 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007746920762286861, + "loss": 2.0578, + "theoretical_loss": 4.604042089547834, + "tokens_seen": 136380416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007746760280842529, + "loss": 1.9681, + "theoretical_loss": 4.603750651295222, + "tokens_seen": 136445952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007746599799398195, + "loss": 1.8947, + "theoretical_loss": 4.603459392161888, + "tokens_seen": 136511488 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007746439317953863, + "loss": 1.9493, + "theoretical_loss": 4.603168311951835, + "tokens_seen": 136577024 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007746278836509529, + "loss": 2.1153, + "theoretical_loss": 4.60287741046937, + "tokens_seen": 136642560 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007746118355065197, + "loss": 1.884, + "theoretical_loss": 4.602586687519111, + "tokens_seen": 136708096 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007745957873620863, + "loss": 1.9621, + "theoretical_loss": 4.602296142905981, + "tokens_seen": 136773632 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007745797392176531, + "loss": 2.0595, + "theoretical_loss": 4.602005776435211, + "tokens_seen": 136839168 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007745636910732197, + "loss": 1.8353, + "theoretical_loss": 4.601715587912338, + "tokens_seen": 136904704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007745476429287865, + "loss": 1.9267, + "theoretical_loss": 4.601425577143203, + "tokens_seen": 136970240 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007745315947843531, + "loss": 1.8376, + "theoretical_loss": 4.601135743933952, + "tokens_seen": 137035776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007745155466399198, + "loss": 1.9296, + "theoretical_loss": 4.600846088091036, + "tokens_seen": 137101312 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007744994984954865, + "loss": 1.8857, + "theoretical_loss": 4.600556609421206, + "tokens_seen": 137166848 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007744834503510532, + "loss": 1.9969, + "theoretical_loss": 4.600267307731521, + "tokens_seen": 137232384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007744674022066199, + "loss": 1.9751, + "theoretical_loss": 4.5999781828293385, + "tokens_seen": 137297920 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007744513540621866, + "loss": 1.9137, + "theoretical_loss": 4.5996892345223195, + "tokens_seen": 137363456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007744353059177533, + "loss": 1.9401, + "theoretical_loss": 4.599400462618423, + "tokens_seen": 137428992 + }, + { + "epoch": 0.04, + "learning_rate": 0.00077441925777332, + "loss": 1.8599, + "theoretical_loss": 4.59911186692591, + "tokens_seen": 137494528 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007744032096288867, + "loss": 1.8279, + "theoretical_loss": 4.598823447253343, + "tokens_seen": 137560064 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.9924926161766052, + "objective/train/avg_token_score": 0.007507418282330036, + "objective/train/docs_used": 57180, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 2.1149203777313232, + "objective/train/original_loss": 2.1318821907043457, + "objective/train/theoretical_loss": 4.598535203409582, + "objective/train/tokens_used": 158085600, + "objective/train/unlikelihood_loss": 0.011760061606764793, + "theoretical_loss": 4.598535203409582, + "tokens_seen": 137625600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007743871614844534, + "loss": 2.0501, + "theoretical_loss": 4.598535203409582, + "tokens_seen": 137625600 + }, + { + "epoch": 0.04, + "learning_rate": 0.00077437111334002, + "loss": 1.8612, + "theoretical_loss": 4.598247135203783, + "tokens_seen": 137691136 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007743550651955868, + "loss": 1.8331, + "theoretical_loss": 4.5979592424454045, + "tokens_seen": 137756672 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007743390170511534, + "loss": 1.9365, + "theoretical_loss": 4.597671524944198, + "tokens_seen": 137822208 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007743229689067202, + "loss": 2.0339, + "theoretical_loss": 4.597383982510216, + "tokens_seen": 137887744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007743069207622868, + "loss": 1.8519, + "theoretical_loss": 4.597096614953802, + "tokens_seen": 137953280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007742908726178536, + "loss": 1.8675, + "theoretical_loss": 4.596809422085599, + "tokens_seen": 138018816 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007742748244734202, + "loss": 1.9475, + "theoretical_loss": 4.596522403716542, + "tokens_seen": 138084352 + }, + { + "epoch": 0.04, + "learning_rate": 0.000774258776328987, + "loss": 2.0608, + "theoretical_loss": 4.596235559657863, + "tokens_seen": 138149888 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007742427281845536, + "loss": 1.778, + "theoretical_loss": 4.595948889721086, + "tokens_seen": 138215424 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007742266800401204, + "loss": 2.0132, + "theoretical_loss": 4.595662393718026, + "tokens_seen": 138280960 + }, + { + "epoch": 0.04, + "learning_rate": 0.000774210631895687, + "loss": 2.0208, + "theoretical_loss": 4.595376071460796, + "tokens_seen": 138346496 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007741945837512538, + "loss": 1.8543, + "theoretical_loss": 4.595089922761796, + "tokens_seen": 138412032 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007741785356068204, + "loss": 1.8982, + "theoretical_loss": 4.594803947433717, + "tokens_seen": 138477568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007741624874623872, + "loss": 1.92, + "theoretical_loss": 4.594518145289545, + "tokens_seen": 138543104 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007741464393179538, + "loss": 2.002, + "theoretical_loss": 4.594232516142553, + "tokens_seen": 138608640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007741303911735206, + "loss": 2.0396, + "theoretical_loss": 4.593947059806302, + "tokens_seen": 138674176 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007741143430290872, + "loss": 1.8713, + "theoretical_loss": 4.593661776094647, + "tokens_seen": 138739712 + }, + { + "epoch": 0.04, + "learning_rate": 0.000774098294884654, + "loss": 1.816, + "theoretical_loss": 4.593376664821726, + "tokens_seen": 138805248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007740822467402206, + "loss": 1.7807, + "theoretical_loss": 4.593091725801967, + "tokens_seen": 138870784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007740661985957874, + "loss": 1.9264, + "theoretical_loss": 4.592806958850087, + "tokens_seen": 138936320 + }, + { + "epoch": 0.04, + "learning_rate": 0.000774050150451354, + "loss": 1.9176, + "theoretical_loss": 4.592522363781086, + "tokens_seen": 139001856 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007740341023069208, + "loss": 1.913, + "theoretical_loss": 4.592237940410252, + "tokens_seen": 139067392 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007740180541624874, + "loss": 1.8496, + "theoretical_loss": 4.591953688553159, + "tokens_seen": 139132928 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007740020060180542, + "loss": 1.8921, + "theoretical_loss": 4.591669608025663, + "tokens_seen": 139198464 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007739859578736208, + "loss": 2.0069, + "theoretical_loss": 4.591385698643908, + "tokens_seen": 139264000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007739699097291876, + "loss": 1.9744, + "theoretical_loss": 4.591101960224321, + "tokens_seen": 139329536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007739538615847542, + "loss": 1.9027, + "theoretical_loss": 4.590818392583608, + "tokens_seen": 139395072 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773937813440321, + "loss": 1.7888, + "theoretical_loss": 4.590534995538762, + "tokens_seen": 139460608 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007739217652958876, + "loss": 1.9595, + "theoretical_loss": 4.590251768907057, + "tokens_seen": 139526144 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007739057171514544, + "loss": 1.8298, + "theoretical_loss": 4.5899687125060495, + "tokens_seen": 139591680 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773889669007021, + "loss": 2.004, + "theoretical_loss": 4.589685826153573, + "tokens_seen": 139657216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007738736208625878, + "loss": 1.8561, + "theoretical_loss": 4.589403109667746, + "tokens_seen": 139722752 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007738575727181545, + "loss": 1.9771, + "theoretical_loss": 4.589120562866961, + "tokens_seen": 139788288 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007738415245737212, + "loss": 1.9729, + "theoretical_loss": 4.588838185569897, + "tokens_seen": 139853824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007738254764292879, + "loss": 1.8545, + "theoretical_loss": 4.588555977595506, + "tokens_seen": 139919360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007738094282848546, + "loss": 1.912, + "theoretical_loss": 4.58827393876302, + "tokens_seen": 139984896 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007737933801404213, + "loss": 1.9119, + "theoretical_loss": 4.587992068891948, + "tokens_seen": 140050432 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773777331995988, + "loss": 1.7839, + "theoretical_loss": 4.587710367802076, + "tokens_seen": 140115968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007737612838515547, + "loss": 1.8934, + "theoretical_loss": 4.587428835313469, + "tokens_seen": 140181504 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007737452357071214, + "loss": 1.926, + "theoretical_loss": 4.587147471246463, + "tokens_seen": 140247040 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007737291875626881, + "loss": 1.8712, + "theoretical_loss": 4.586866275421673, + "tokens_seen": 140312576 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007737131394182548, + "loss": 1.9554, + "theoretical_loss": 4.586585247659986, + "tokens_seen": 140378112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007736970912738215, + "loss": 1.8402, + "theoretical_loss": 4.586304387782567, + "tokens_seen": 140443648 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007736810431293882, + "loss": 1.8778, + "theoretical_loss": 4.5860236956108515, + "tokens_seen": 140509184 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007736649949849549, + "loss": 1.9049, + "theoretical_loss": 4.585743170966548, + "tokens_seen": 140574720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007736489468405216, + "loss": 1.8614, + "theoretical_loss": 4.58546281367164, + "tokens_seen": 140640256 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007736328986960883, + "loss": 2.0853, + "theoretical_loss": 4.585182623548381, + "tokens_seen": 140705792 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773616850551655, + "loss": 1.8803, + "theoretical_loss": 4.584902600419295, + "tokens_seen": 140771328 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007736008024072217, + "loss": 1.9185, + "theoretical_loss": 4.584622744107181, + "tokens_seen": 140836864 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.9958894848823547, + "objective/train/avg_token_score": 0.004110530950129032, + "objective/train/docs_used": 58312, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.7865874767303467, + "objective/train/original_loss": 1.7942376136779785, + "objective/train/theoretical_loss": 4.584343054435103, + "objective/train/tokens_used": 161362400, + "objective/train/unlikelihood_loss": 0.009206429123878479, + "theoretical_loss": 4.584343054435103, + "tokens_seen": 140902400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007735847542627884, + "loss": 1.7756, + "theoretical_loss": 4.584343054435103, + "tokens_seen": 140902400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007735687061183552, + "loss": 1.913, + "theoretical_loss": 4.5840635312264, + "tokens_seen": 140967936 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007735526579739218, + "loss": 1.8622, + "theoretical_loss": 4.583784174304677, + "tokens_seen": 141033472 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007735366098294886, + "loss": 1.7518, + "theoretical_loss": 4.583504983493809, + "tokens_seen": 141099008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007735205616850552, + "loss": 1.9856, + "theoretical_loss": 4.583225958617938, + "tokens_seen": 141164544 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773504513540622, + "loss": 1.921, + "theoretical_loss": 4.582947099501475, + "tokens_seen": 141230080 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007734884653961886, + "loss": 1.9109, + "theoretical_loss": 4.5826684059691, + "tokens_seen": 141295616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007734724172517554, + "loss": 1.9573, + "theoretical_loss": 4.582389877845754, + "tokens_seen": 141361152 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773456369107322, + "loss": 1.9266, + "theoretical_loss": 4.58211151495665, + "tokens_seen": 141426688 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007734403209628888, + "loss": 1.6935, + "theoretical_loss": 4.581833317127263, + "tokens_seen": 141492224 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007734242728184554, + "loss": 1.889, + "theoretical_loss": 4.581555284183335, + "tokens_seen": 141557760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007734082246740222, + "loss": 1.9469, + "theoretical_loss": 4.581277415950872, + "tokens_seen": 141623296 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007733921765295888, + "loss": 1.9246, + "theoretical_loss": 4.580999712256142, + "tokens_seen": 141688832 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007733761283851556, + "loss": 1.8793, + "theoretical_loss": 4.5807221729256815, + "tokens_seen": 141754368 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007733600802407222, + "loss": 1.9182, + "theoretical_loss": 4.580444797786283, + "tokens_seen": 141819904 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773344032096289, + "loss": 1.9182, + "theoretical_loss": 4.5801675866650085, + "tokens_seen": 141885440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007733279839518556, + "loss": 1.9807, + "theoretical_loss": 4.579890539389177, + "tokens_seen": 141950976 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007733119358074224, + "loss": 1.9275, + "theoretical_loss": 4.579613655786369, + "tokens_seen": 142016512 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773295887662989, + "loss": 1.8608, + "theoretical_loss": 4.579336935684431, + "tokens_seen": 142082048 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007732798395185558, + "loss": 1.9126, + "theoretical_loss": 4.579060378911464, + "tokens_seen": 142147584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007732637913741224, + "loss": 1.8496, + "theoretical_loss": 4.578783985295832, + "tokens_seen": 142213120 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007732477432296892, + "loss": 1.9495, + "theoretical_loss": 4.578507754666157, + "tokens_seen": 142278656 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007732316950852558, + "loss": 2.0276, + "theoretical_loss": 4.57823168685132, + "tokens_seen": 142344192 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007732156469408226, + "loss": 1.9717, + "theoretical_loss": 4.577955781680462, + "tokens_seen": 142409728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007731995987963892, + "loss": 1.8427, + "theoretical_loss": 4.57768003898298, + "tokens_seen": 142475264 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773183550651956, + "loss": 1.8562, + "theoretical_loss": 4.577404458588529, + "tokens_seen": 142540800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007731675025075226, + "loss": 1.9256, + "theoretical_loss": 4.577129040327021, + "tokens_seen": 142606336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007731514543630894, + "loss": 1.8801, + "theoretical_loss": 4.576853784028622, + "tokens_seen": 142671872 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773135406218656, + "loss": 1.8152, + "theoretical_loss": 4.576578689523759, + "tokens_seen": 142737408 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007731193580742227, + "loss": 1.9075, + "theoretical_loss": 4.576303756643108, + "tokens_seen": 142802944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007731033099297894, + "loss": 1.865, + "theoretical_loss": 4.576028985217606, + "tokens_seen": 142868480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007730872617853561, + "loss": 1.9526, + "theoretical_loss": 4.5757543750784375, + "tokens_seen": 142934016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007730712136409228, + "loss": 1.9311, + "theoretical_loss": 4.5754799260570485, + "tokens_seen": 142999552 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007730551654964895, + "loss": 2.0036, + "theoretical_loss": 4.575205637985132, + "tokens_seen": 143065088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007730391173520562, + "loss": 1.9806, + "theoretical_loss": 4.574931510694635, + "tokens_seen": 143130624 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007730230692076229, + "loss": 1.8185, + "theoretical_loss": 4.574657544017761, + "tokens_seen": 143196160 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007730070210631896, + "loss": 1.9725, + "theoretical_loss": 4.57438373778696, + "tokens_seen": 143261696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007729909729187563, + "loss": 1.9204, + "theoretical_loss": 4.574110091834936, + "tokens_seen": 143327232 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007729749247743229, + "loss": 1.8841, + "theoretical_loss": 4.573836605994645, + "tokens_seen": 143392768 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007729588766298897, + "loss": 1.931, + "theoretical_loss": 4.573563280099288, + "tokens_seen": 143458304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007729428284854563, + "loss": 1.9829, + "theoretical_loss": 4.573290113982324, + "tokens_seen": 143523840 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007729267803410231, + "loss": 1.7419, + "theoretical_loss": 4.573017107477455, + "tokens_seen": 143589376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007729107321965897, + "loss": 1.9131, + "theoretical_loss": 4.572744260418633, + "tokens_seen": 143654912 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007728946840521565, + "loss": 1.9555, + "theoretical_loss": 4.572471572640059, + "tokens_seen": 143720448 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007728786359077231, + "loss": 1.9719, + "theoretical_loss": 4.572199043976185, + "tokens_seen": 143785984 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007728625877632899, + "loss": 1.8193, + "theoretical_loss": 4.571926674261705, + "tokens_seen": 143851520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007728465396188565, + "loss": 1.7724, + "theoretical_loss": 4.571654463331563, + "tokens_seen": 143917056 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007728304914744233, + "loss": 1.9073, + "theoretical_loss": 4.571382411020949, + "tokens_seen": 143982592 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007728144433299899, + "loss": 1.9162, + "theoretical_loss": 4.5711105171653, + "tokens_seen": 144048128 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007727983951855567, + "loss": 1.9131, + "theoretical_loss": 4.570838781600296, + "tokens_seen": 144113664 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.9903619289398193, + "objective/train/avg_token_score": 0.009638032875955105, + "objective/train/docs_used": 59583, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.8158360719680786, + "objective/train/original_loss": 1.835557222366333, + "objective/train/theoretical_loss": 4.570567204161865, + "objective/train/tokens_used": 164639200, + "objective/train/unlikelihood_loss": 0.016580061987042427, + "theoretical_loss": 4.570567204161865, + "tokens_seen": 144179200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007727823470411233, + "loss": 1.9016, + "theoretical_loss": 4.570567204161865, + "tokens_seen": 144179200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007727662988966901, + "loss": 1.8062, + "theoretical_loss": 4.570295784686176, + "tokens_seen": 144244736 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007727502507522567, + "loss": 1.9354, + "theoretical_loss": 4.570024523009646, + "tokens_seen": 144310272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007727342026078235, + "loss": 1.8987, + "theoretical_loss": 4.569753418968933, + "tokens_seen": 144375808 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007727181544633901, + "loss": 1.8603, + "theoretical_loss": 4.569482472400939, + "tokens_seen": 144441344 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007727021063189569, + "loss": 1.9167, + "theoretical_loss": 4.5692116831428065, + "tokens_seen": 144506880 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007726860581745235, + "loss": 1.8588, + "theoretical_loss": 4.568941051031925, + "tokens_seen": 144572416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007726700100300903, + "loss": 1.8974, + "theoretical_loss": 4.568670575905921, + "tokens_seen": 144637952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007726539618856569, + "loss": 1.9576, + "theoretical_loss": 4.568400257602666, + "tokens_seen": 144703488 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007726379137412237, + "loss": 1.9551, + "theoretical_loss": 4.568130095960268, + "tokens_seen": 144769024 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007726218655967903, + "loss": 1.9518, + "theoretical_loss": 4.56786009081708, + "tokens_seen": 144834560 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007726058174523571, + "loss": 1.8845, + "theoretical_loss": 4.567590242011692, + "tokens_seen": 144900096 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007725897693079237, + "loss": 1.9027, + "theoretical_loss": 4.567320549382931, + "tokens_seen": 144965632 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007725737211634905, + "loss": 1.9267, + "theoretical_loss": 4.56705101276987, + "tokens_seen": 145031168 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007725576730190571, + "loss": 1.8435, + "theoretical_loss": 4.566781632011814, + "tokens_seen": 145096704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007725416248746239, + "loss": 1.7877, + "theoretical_loss": 4.566512406948309, + "tokens_seen": 145162240 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007725255767301905, + "loss": 1.8818, + "theoretical_loss": 4.566243337419137, + "tokens_seen": 145227776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007725095285857573, + "loss": 1.8339, + "theoretical_loss": 4.5659744232643185, + "tokens_seen": 145293312 + }, + { + "epoch": 0.04, + "learning_rate": 0.000772493480441324, + "loss": 2.0771, + "theoretical_loss": 4.565705664324111, + "tokens_seen": 145358848 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007724774322968907, + "loss": 1.7793, + "theoretical_loss": 4.565437060439006, + "tokens_seen": 145424384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007724613841524574, + "loss": 1.9339, + "theoretical_loss": 4.565168611449732, + "tokens_seen": 145489920 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007724453360080241, + "loss": 1.9012, + "theoretical_loss": 4.564900317197251, + "tokens_seen": 145555456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007724292878635908, + "loss": 2.031, + "theoretical_loss": 4.564632177522766, + "tokens_seen": 145620992 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007724132397191575, + "loss": 1.8686, + "theoretical_loss": 4.564364192267705, + "tokens_seen": 145686528 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007723971915747242, + "loss": 1.9031, + "theoretical_loss": 4.564096361273736, + "tokens_seen": 145752064 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007723811434302909, + "loss": 1.9404, + "theoretical_loss": 4.563828684382761, + "tokens_seen": 145817600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007723650952858577, + "loss": 1.9713, + "theoretical_loss": 4.563561161436912, + "tokens_seen": 145883136 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007723490471414243, + "loss": 1.732, + "theoretical_loss": 4.563293792278554, + "tokens_seen": 145948672 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007723329989969911, + "loss": 1.8343, + "theoretical_loss": 4.563026576750286, + "tokens_seen": 146014208 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007723169508525577, + "loss": 1.8793, + "theoretical_loss": 4.562759514694935, + "tokens_seen": 146079744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007723009027081245, + "loss": 1.9062, + "theoretical_loss": 4.5624926059555655, + "tokens_seen": 146145280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007722848545636911, + "loss": 1.8717, + "theoretical_loss": 4.562225850375466, + "tokens_seen": 146210816 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007722688064192579, + "loss": 1.887, + "theoretical_loss": 4.561959247798159, + "tokens_seen": 146276352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007722527582748245, + "loss": 1.8801, + "theoretical_loss": 4.561692798067396, + "tokens_seen": 146341888 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007722367101303913, + "loss": 1.7995, + "theoretical_loss": 4.561426501027158, + "tokens_seen": 146407424 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007722206619859579, + "loss": 1.9046, + "theoretical_loss": 4.561160356521656, + "tokens_seen": 146472960 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007722046138415247, + "loss": 1.839, + "theoretical_loss": 4.5608943643953275, + "tokens_seen": 146538496 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007721885656970913, + "loss": 1.9649, + "theoretical_loss": 4.56062852449284, + "tokens_seen": 146604032 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007721725175526581, + "loss": 1.8176, + "theoretical_loss": 4.5603628366590865, + "tokens_seen": 146669568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007721564694082247, + "loss": 2.0132, + "theoretical_loss": 4.560097300739191, + "tokens_seen": 146735104 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007721404212637915, + "loss": 1.8937, + "theoretical_loss": 4.5598319165785, + "tokens_seen": 146800640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007721243731193581, + "loss": 1.8877, + "theoretical_loss": 4.55956668402259, + "tokens_seen": 146866176 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007721083249749249, + "loss": 1.7429, + "theoretical_loss": 4.559301602917261, + "tokens_seen": 146931712 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007720922768304915, + "loss": 1.864, + "theoretical_loss": 4.55903667310854, + "tokens_seen": 146997248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007720762286860583, + "loss": 1.8139, + "theoretical_loss": 4.558771894442677, + "tokens_seen": 147062784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007720601805416249, + "loss": 1.9065, + "theoretical_loss": 4.558507266766151, + "tokens_seen": 147128320 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007720441323971917, + "loss": 1.8996, + "theoretical_loss": 4.558242789925659, + "tokens_seen": 147193856 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007720280842527583, + "loss": 1.8014, + "theoretical_loss": 4.557978463768127, + "tokens_seen": 147259392 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007720120361083251, + "loss": 1.8902, + "theoretical_loss": 4.557714288140703, + "tokens_seen": 147324928 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007719959879638917, + "loss": 1.8273, + "theoretical_loss": 4.557450262890757, + "tokens_seen": 147390464 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.9939889311790466, + "objective/train/avg_token_score": 0.006011018063873053, + "objective/train/docs_used": 60747, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.7478055953979492, + "objective/train/original_loss": 1.7586967945098877, + "objective/train/theoretical_loss": 4.557186387865883, + "objective/train/tokens_used": 167916000, + "objective/train/unlikelihood_loss": 0.012668152339756489, + "theoretical_loss": 4.557186387865883, + "tokens_seen": 147456000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007719799398194585, + "loss": 1.7835, + "theoretical_loss": 4.557186387865883, + "tokens_seen": 147456000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007719638916750251, + "loss": 1.8504, + "theoretical_loss": 4.556922662913895, + "tokens_seen": 147521536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007719478435305919, + "loss": 1.8742, + "theoretical_loss": 4.55665908788283, + "tokens_seen": 147587072 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007719317953861585, + "loss": 1.9705, + "theoretical_loss": 4.556395662620945, + "tokens_seen": 147652608 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007719157472417253, + "loss": 1.802, + "theoretical_loss": 4.5561323869767225, + "tokens_seen": 147718144 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007718996990972919, + "loss": 1.8675, + "theoretical_loss": 4.555869260798859, + "tokens_seen": 147783680 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007718836509528587, + "loss": 1.904, + "theoretical_loss": 4.5556062839362745, + "tokens_seen": 147849216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007718676028084253, + "loss": 1.9315, + "theoretical_loss": 4.555343456238109, + "tokens_seen": 147914752 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007718515546639921, + "loss": 1.9557, + "theoretical_loss": 4.55508077755372, + "tokens_seen": 147980288 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007718355065195587, + "loss": 1.8796, + "theoretical_loss": 4.554818247732685, + "tokens_seen": 148045824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007718194583751255, + "loss": 1.8855, + "theoretical_loss": 4.554555866624798, + "tokens_seen": 148111360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007718034102306921, + "loss": 1.9108, + "theoretical_loss": 4.554293634080074, + "tokens_seen": 148176896 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007717873620862589, + "loss": 1.7972, + "theoretical_loss": 4.554031549948743, + "tokens_seen": 148242432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007717713139418255, + "loss": 1.8378, + "theoretical_loss": 4.5537696140812525, + "tokens_seen": 148307968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007717552657973923, + "loss": 1.9344, + "theoretical_loss": 4.553507826328268, + "tokens_seen": 148373504 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007717392176529589, + "loss": 1.7871, + "theoretical_loss": 4.553246186540671, + "tokens_seen": 148439040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007717231695085256, + "loss": 1.7761, + "theoretical_loss": 4.552984694569555, + "tokens_seen": 148504576 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007717071213640923, + "loss": 1.9321, + "theoretical_loss": 4.552723350266236, + "tokens_seen": 148570112 + }, + { + "epoch": 0.05, + "learning_rate": 0.000771691073219659, + "loss": 1.8746, + "theoretical_loss": 4.552462153482239, + "tokens_seen": 148635648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007716750250752257, + "loss": 1.8619, + "theoretical_loss": 4.552201104069307, + "tokens_seen": 148701184 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007716589769307924, + "loss": 1.9188, + "theoretical_loss": 4.551940201879396, + "tokens_seen": 148766720 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007716429287863591, + "loss": 1.7483, + "theoretical_loss": 4.551679446764677, + "tokens_seen": 148832256 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007716268806419258, + "loss": 1.9777, + "theoretical_loss": 4.551418838577531, + "tokens_seen": 148897792 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007716108324974925, + "loss": 1.8373, + "theoretical_loss": 4.551158377170557, + "tokens_seen": 148963328 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007715947843530592, + "loss": 1.6799, + "theoretical_loss": 4.5508980623965645, + "tokens_seen": 149028864 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007715787362086258, + "loss": 1.9743, + "theoretical_loss": 4.550637894108572, + "tokens_seen": 149094400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007715626880641926, + "loss": 1.7532, + "theoretical_loss": 4.5503778721598165, + "tokens_seen": 149159936 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007715466399197592, + "loss": 1.7374, + "theoretical_loss": 4.55011799640374, + "tokens_seen": 149225472 + }, + { + "epoch": 0.05, + "learning_rate": 0.000771530591775326, + "loss": 1.8833, + "theoretical_loss": 4.549858266694001, + "tokens_seen": 149291008 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007715145436308926, + "loss": 1.8269, + "theoretical_loss": 4.549598682884463, + "tokens_seen": 149356544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007714984954864594, + "loss": 1.8933, + "theoretical_loss": 4.549339244829204, + "tokens_seen": 149422080 + }, + { + "epoch": 0.05, + "learning_rate": 0.000771482447342026, + "loss": 1.7835, + "theoretical_loss": 4.549079952382511, + "tokens_seen": 149487616 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007714663991975928, + "loss": 1.8732, + "theoretical_loss": 4.548820805398881, + "tokens_seen": 149553152 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007714503510531594, + "loss": 1.7255, + "theoretical_loss": 4.548561803733017, + "tokens_seen": 149618688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007714343029087262, + "loss": 2.0778, + "theoretical_loss": 4.548302947239834, + "tokens_seen": 149684224 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007714182547642928, + "loss": 1.8462, + "theoretical_loss": 4.548044235774454, + "tokens_seen": 149749760 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007714022066198596, + "loss": 1.894, + "theoretical_loss": 4.547785669192207, + "tokens_seen": 149815296 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007713861584754262, + "loss": 1.9064, + "theoretical_loss": 4.547527247348631, + "tokens_seen": 149880832 + }, + { + "epoch": 0.05, + "learning_rate": 0.000771370110330993, + "loss": 1.8842, + "theoretical_loss": 4.547268970099471, + "tokens_seen": 149946368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007713540621865596, + "loss": 1.8532, + "theoretical_loss": 4.547010837300677, + "tokens_seen": 150011904 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007713380140421264, + "loss": 1.8773, + "theoretical_loss": 4.546752848808408, + "tokens_seen": 150077440 + }, + { + "epoch": 0.05, + "learning_rate": 0.000771321965897693, + "loss": 1.8481, + "theoretical_loss": 4.546495004479027, + "tokens_seen": 150142976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007713059177532598, + "loss": 1.7721, + "theoretical_loss": 4.546237304169104, + "tokens_seen": 150208512 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007712898696088264, + "loss": 1.92, + "theoretical_loss": 4.545979747735413, + "tokens_seen": 150274048 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007712738214643932, + "loss": 1.7174, + "theoretical_loss": 4.5457223350349345, + "tokens_seen": 150339584 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007712577733199598, + "loss": 1.7839, + "theoretical_loss": 4.545465065924851, + "tokens_seen": 150405120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007712417251755266, + "loss": 1.7595, + "theoretical_loss": 4.54520794026255, + "tokens_seen": 150470656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007712256770310932, + "loss": 1.7673, + "theoretical_loss": 4.544950957905625, + "tokens_seen": 150536192 + }, + { + "epoch": 0.05, + "learning_rate": 0.00077120962888666, + "loss": 1.8138, + "theoretical_loss": 4.54469411871187, + "tokens_seen": 150601728 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007711935807422266, + "loss": 1.7793, + "theoretical_loss": 4.544437422539282, + "tokens_seen": 150667264 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.9940516948699951, + "objective/train/avg_token_score": 0.005948349367827177, + "objective/train/docs_used": 61752, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.8874962329864502, + "objective/train/original_loss": 1.90000581741333, + "objective/train/theoretical_loss": 4.544180869246063, + "objective/train/tokens_used": 171192800, + "objective/train/unlikelihood_loss": 0.009729099459946156, + "theoretical_loss": 4.544180869246063, + "tokens_seen": 150732800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007711775325977934, + "loss": 1.8754, + "theoretical_loss": 4.544180869246063, + "tokens_seen": 150732800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007711614844533602, + "loss": 1.9919, + "theoretical_loss": 4.543924458690612, + "tokens_seen": 150798336 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007711454363089268, + "loss": 1.9418, + "theoretical_loss": 4.5436681907315375, + "tokens_seen": 150863872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007711293881644936, + "loss": 1.9547, + "theoretical_loss": 4.543412065227642, + "tokens_seen": 150929408 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007711133400200602, + "loss": 1.806, + "theoretical_loss": 4.5431560820379335, + "tokens_seen": 150994944 + }, + { + "epoch": 0.05, + "learning_rate": 0.000771097291875627, + "loss": 1.8205, + "theoretical_loss": 4.54290024102162, + "tokens_seen": 151060480 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007710812437311936, + "loss": 1.8722, + "theoretical_loss": 4.542644542038108, + "tokens_seen": 151126016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007710651955867604, + "loss": 1.9658, + "theoretical_loss": 4.542388984947005, + "tokens_seen": 151191552 + }, + { + "epoch": 0.05, + "learning_rate": 0.000771049147442327, + "loss": 1.8213, + "theoretical_loss": 4.5421335696081195, + "tokens_seen": 151257088 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007710330992978938, + "loss": 1.7912, + "theoretical_loss": 4.541878295881456, + "tokens_seen": 151322624 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007710170511534604, + "loss": 1.7949, + "theoretical_loss": 4.541623163627221, + "tokens_seen": 151388160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007710010030090272, + "loss": 1.7578, + "theoretical_loss": 4.541368172705817, + "tokens_seen": 151453696 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007709849548645938, + "loss": 1.8059, + "theoretical_loss": 4.541113322977846, + "tokens_seen": 151519232 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007709689067201606, + "loss": 1.8677, + "theoretical_loss": 4.540858614304108, + "tokens_seen": 151584768 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007709528585757272, + "loss": 1.7894, + "theoretical_loss": 4.5406040465455995, + "tokens_seen": 151650304 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770936810431294, + "loss": 1.7219, + "theoretical_loss": 4.5403496195635125, + "tokens_seen": 151715840 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007709207622868606, + "loss": 1.908, + "theoretical_loss": 4.54009533321924, + "tokens_seen": 151781376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007709047141424274, + "loss": 1.7504, + "theoretical_loss": 4.539841187374366, + "tokens_seen": 151846912 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770888665997994, + "loss": 1.8332, + "theoretical_loss": 4.539587181890674, + "tokens_seen": 151912448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007708726178535608, + "loss": 1.8409, + "theoretical_loss": 4.539333316630143, + "tokens_seen": 151977984 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007708565697091274, + "loss": 1.7253, + "theoretical_loss": 4.539079591454943, + "tokens_seen": 152043520 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007708405215646942, + "loss": 1.9503, + "theoretical_loss": 4.5388260062274455, + "tokens_seen": 152109056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007708244734202608, + "loss": 1.8074, + "theoretical_loss": 4.538572560810211, + "tokens_seen": 152174592 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007708084252758276, + "loss": 1.7944, + "theoretical_loss": 4.538319255065995, + "tokens_seen": 152240128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007707923771313942, + "loss": 1.7696, + "theoretical_loss": 4.53806608885775, + "tokens_seen": 152305664 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770776328986961, + "loss": 2.0208, + "theoretical_loss": 4.537813062048618, + "tokens_seen": 152371200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007707602808425276, + "loss": 1.766, + "theoretical_loss": 4.537560174501936, + "tokens_seen": 152436736 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007707442326980944, + "loss": 1.8546, + "theoretical_loss": 4.537307426081232, + "tokens_seen": 152502272 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770728184553661, + "loss": 1.9237, + "theoretical_loss": 4.53705481665023, + "tokens_seen": 152567808 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007707121364092278, + "loss": 1.9036, + "theoretical_loss": 4.536802346072841, + "tokens_seen": 152633344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007706960882647944, + "loss": 1.8243, + "theoretical_loss": 4.536550014213171, + "tokens_seen": 152698880 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007706800401203612, + "loss": 1.877, + "theoretical_loss": 4.536297820935517, + "tokens_seen": 152764416 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007706639919759278, + "loss": 1.7833, + "theoretical_loss": 4.536045766104366, + "tokens_seen": 152829952 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007706479438314946, + "loss": 1.9463, + "theoretical_loss": 4.535793849584396, + "tokens_seen": 152895488 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007706318956870612, + "loss": 1.9752, + "theoretical_loss": 4.535542071240473, + "tokens_seen": 152961024 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770615847542628, + "loss": 1.8543, + "theoretical_loss": 4.535290430937657, + "tokens_seen": 153026560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007705997993981946, + "loss": 1.832, + "theoretical_loss": 4.535038928541194, + "tokens_seen": 153092096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007705837512537614, + "loss": 1.8194, + "theoretical_loss": 4.534787563916522, + "tokens_seen": 153157632 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770567703109328, + "loss": 1.7254, + "theoretical_loss": 4.534536336929264, + "tokens_seen": 153223168 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007705516549648948, + "loss": 1.9164, + "theoretical_loss": 4.534285247445236, + "tokens_seen": 153288704 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007705356068204614, + "loss": 1.8605, + "theoretical_loss": 4.534034295330439, + "tokens_seen": 153354240 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007705195586760282, + "loss": 1.7785, + "theoretical_loss": 4.5337834804510635, + "tokens_seen": 153419776 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007705035105315948, + "loss": 1.9926, + "theoretical_loss": 4.533532802673484, + "tokens_seen": 153485312 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007704874623871616, + "loss": 1.7955, + "theoretical_loss": 4.533282261864267, + "tokens_seen": 153550848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007704714142427282, + "loss": 1.958, + "theoretical_loss": 4.533031857890164, + "tokens_seen": 153616384 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770455366098295, + "loss": 1.8343, + "theoretical_loss": 4.532781590618109, + "tokens_seen": 153681920 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007704393179538616, + "loss": 1.9699, + "theoretical_loss": 4.532531459915229, + "tokens_seen": 153747456 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007704232698094284, + "loss": 1.8782, + "theoretical_loss": 4.532281465648831, + "tokens_seen": 153812992 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770407221664995, + "loss": 1.8715, + "theoretical_loss": 4.532031607686411, + "tokens_seen": 153878528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007703911735205618, + "loss": 1.8509, + "theoretical_loss": 4.531781885895647, + "tokens_seen": 153944064 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.995068371295929, + "objective/train/avg_token_score": 0.004931598901748657, + "objective/train/docs_used": 63027, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 2.0622313022613525, + "objective/train/original_loss": 2.0729503631591797, + "objective/train/theoretical_loss": 4.531532300144404, + "objective/train/tokens_used": 174469600, + "objective/train/unlikelihood_loss": 0.007933967746794224, + "theoretical_loss": 4.531532300144404, + "tokens_seen": 154009600 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007703751253761284, + "loss": 1.951, + "theoretical_loss": 4.531532300144404, + "tokens_seen": 154009600 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007703590772316952, + "loss": 1.9024, + "theoretical_loss": 4.531282850300731, + "tokens_seen": 154075136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007703430290872618, + "loss": 2.0512, + "theoretical_loss": 4.53103353623286, + "tokens_seen": 154140672 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007703269809428285, + "loss": 1.8798, + "theoretical_loss": 4.530784357809209, + "tokens_seen": 154206208 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007703109327983952, + "loss": 1.7564, + "theoretical_loss": 4.5305353148983745, + "tokens_seen": 154271744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007702948846539619, + "loss": 1.869, + "theoretical_loss": 4.530286407369143, + "tokens_seen": 154337280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007702788365095286, + "loss": 1.9817, + "theoretical_loss": 4.530037635090476, + "tokens_seen": 154402816 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007702627883650953, + "loss": 1.927, + "theoretical_loss": 4.529788997931526, + "tokens_seen": 154468352 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770246740220662, + "loss": 1.8227, + "theoretical_loss": 4.52954049576162, + "tokens_seen": 154533888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007702306920762287, + "loss": 1.92, + "theoretical_loss": 4.529292128450271, + "tokens_seen": 154599424 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007702146439317954, + "loss": 1.8698, + "theoretical_loss": 4.52904389586717, + "tokens_seen": 154664960 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007701985957873621, + "loss": 1.8554, + "theoretical_loss": 4.528795797882193, + "tokens_seen": 154730496 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007701825476429287, + "loss": 1.8379, + "theoretical_loss": 4.528547834365394, + "tokens_seen": 154796032 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007701664994984955, + "loss": 1.7732, + "theoretical_loss": 4.52830000518701, + "tokens_seen": 154861568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007701504513540621, + "loss": 1.8345, + "theoretical_loss": 4.528052310217453, + "tokens_seen": 154927104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007701344032096289, + "loss": 1.776, + "theoretical_loss": 4.527804749327321, + "tokens_seen": 154992640 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007701183550651955, + "loss": 1.9707, + "theoretical_loss": 4.527557322387387, + "tokens_seen": 155058176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007701023069207623, + "loss": 1.9029, + "theoretical_loss": 4.527310029268604, + "tokens_seen": 155123712 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007700862587763289, + "loss": 1.8465, + "theoretical_loss": 4.527062869842107, + "tokens_seen": 155189248 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007700702106318957, + "loss": 1.9072, + "theoretical_loss": 4.526815843979204, + "tokens_seen": 155254784 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007700541624874623, + "loss": 1.9693, + "theoretical_loss": 4.5265689515513845, + "tokens_seen": 155320320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007700381143430291, + "loss": 1.8802, + "theoretical_loss": 4.526322192430316, + "tokens_seen": 155385856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007700220661985957, + "loss": 1.8066, + "theoretical_loss": 4.526075566487842, + "tokens_seen": 155451392 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007700060180541625, + "loss": 1.752, + "theoretical_loss": 4.525829073595984, + "tokens_seen": 155516928 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007699899699097291, + "loss": 1.6873, + "theoretical_loss": 4.5255827136269415, + "tokens_seen": 155582464 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007699739217652959, + "loss": 1.9107, + "theoretical_loss": 4.525336486453088, + "tokens_seen": 155648000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007699578736208627, + "loss": 1.9467, + "theoretical_loss": 4.525090391946974, + "tokens_seen": 155713536 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007699418254764293, + "loss": 1.7811, + "theoretical_loss": 4.524844429981327, + "tokens_seen": 155779072 + }, + { + "epoch": 0.05, + "learning_rate": 0.000769925777331996, + "loss": 1.8171, + "theoretical_loss": 4.52459860042905, + "tokens_seen": 155844608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007699097291875627, + "loss": 2.021, + "theoretical_loss": 4.524352903163219, + "tokens_seen": 155910144 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007698936810431295, + "loss": 1.7189, + "theoretical_loss": 4.524107338057089, + "tokens_seen": 155975680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007698776328986961, + "loss": 1.8769, + "theoretical_loss": 4.523861904984086, + "tokens_seen": 156041216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007698615847542629, + "loss": 1.7096, + "theoretical_loss": 4.5236166038178105, + "tokens_seen": 156106752 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007698455366098295, + "loss": 1.8113, + "theoretical_loss": 4.52337143443204, + "tokens_seen": 156172288 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007698294884653963, + "loss": 1.8991, + "theoretical_loss": 4.523126396700722, + "tokens_seen": 156237824 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007698134403209629, + "loss": 1.9613, + "theoretical_loss": 4.522881490497981, + "tokens_seen": 156303360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007697973921765297, + "loss": 1.8044, + "theoretical_loss": 4.522636715698111, + "tokens_seen": 156368896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007697813440320963, + "loss": 1.8466, + "theoretical_loss": 4.522392072175582, + "tokens_seen": 156434432 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007697652958876631, + "loss": 1.8718, + "theoretical_loss": 4.522147559805034, + "tokens_seen": 156499968 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007697492477432297, + "loss": 1.8317, + "theoretical_loss": 4.5219031784612795, + "tokens_seen": 156565504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007697331995987965, + "loss": 1.7033, + "theoretical_loss": 4.521658928019305, + "tokens_seen": 156631040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007697171514543631, + "loss": 1.8528, + "theoretical_loss": 4.521414808354266, + "tokens_seen": 156696576 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007697011033099299, + "loss": 1.7451, + "theoretical_loss": 4.52117081934149, + "tokens_seen": 156762112 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007696850551654965, + "loss": 1.7098, + "theoretical_loss": 4.520926960856476, + "tokens_seen": 156827648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007696690070210633, + "loss": 1.8559, + "theoretical_loss": 4.520683232774893, + "tokens_seen": 156893184 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007696529588766299, + "loss": 1.84, + "theoretical_loss": 4.520439634972579, + "tokens_seen": 156958720 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007696369107321967, + "loss": 1.7512, + "theoretical_loss": 4.520196167325546, + "tokens_seen": 157024256 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007696208625877633, + "loss": 1.9103, + "theoretical_loss": 4.519952829709972, + "tokens_seen": 157089792 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007696048144433301, + "loss": 1.7673, + "theoretical_loss": 4.519709622002205, + "tokens_seen": 157155328 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007695887662988967, + "loss": 1.787, + "theoretical_loss": 4.519466544078762, + "tokens_seen": 157220864 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.9935777187347412, + "objective/train/avg_token_score": 0.0064222924411296844, + "objective/train/docs_used": 64097, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 2.0235674381256104, + "objective/train/original_loss": 2.0362772941589355, + "objective/train/theoretical_loss": 4.519223595816332, + "objective/train/tokens_used": 177746400, + "objective/train/unlikelihood_loss": 0.012138007208704948, + "theoretical_loss": 4.519223595816332, + "tokens_seen": 157286400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007695727181544635, + "loss": 2.032, + "theoretical_loss": 4.519223595816332, + "tokens_seen": 157286400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007695566700100301, + "loss": 1.7331, + "theoretical_loss": 4.518980777091768, + "tokens_seen": 157351936 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007695406218655969, + "loss": 1.9133, + "theoretical_loss": 4.518738087782093, + "tokens_seen": 157417472 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007695245737211635, + "loss": 1.8362, + "theoretical_loss": 4.5184955277644985, + "tokens_seen": 157483008 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007695085255767303, + "loss": 1.9074, + "theoretical_loss": 4.518253096916341, + "tokens_seen": 157548544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007694924774322969, + "loss": 1.8433, + "theoretical_loss": 4.518010795115151, + "tokens_seen": 157614080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007694764292878637, + "loss": 1.8295, + "theoretical_loss": 4.517768622238616, + "tokens_seen": 157679616 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007694603811434303, + "loss": 1.7845, + "theoretical_loss": 4.5175265781646, + "tokens_seen": 157745152 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007694443329989971, + "loss": 1.7241, + "theoretical_loss": 4.517284662771125, + "tokens_seen": 157810688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007694282848545637, + "loss": 1.8813, + "theoretical_loss": 4.5170428759363865, + "tokens_seen": 157876224 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007694122367101305, + "loss": 1.8688, + "theoretical_loss": 4.516801217538739, + "tokens_seen": 157941760 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007693961885656971, + "loss": 1.7965, + "theoretical_loss": 4.516559687456708, + "tokens_seen": 158007296 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007693801404212639, + "loss": 1.85, + "theoretical_loss": 4.516318285568982, + "tokens_seen": 158072832 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007693640922768305, + "loss": 1.8427, + "theoretical_loss": 4.516077011754414, + "tokens_seen": 158138368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007693480441323973, + "loss": 1.8505, + "theoretical_loss": 4.51583586589202, + "tokens_seen": 158203904 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007693319959879639, + "loss": 1.9334, + "theoretical_loss": 4.515594847860985, + "tokens_seen": 158269440 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007693159478435307, + "loss": 1.8609, + "theoretical_loss": 4.515353957540656, + "tokens_seen": 158334976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007692998996990973, + "loss": 1.9027, + "theoretical_loss": 4.515113194810541, + "tokens_seen": 158400512 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007692838515546641, + "loss": 1.8073, + "theoretical_loss": 4.514872559550316, + "tokens_seen": 158466048 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007692678034102307, + "loss": 1.8004, + "theoretical_loss": 4.514632051639815, + "tokens_seen": 158531584 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007692517552657975, + "loss": 1.8263, + "theoretical_loss": 4.51439167095904, + "tokens_seen": 158597120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007692357071213641, + "loss": 1.7873, + "theoretical_loss": 4.5141514173881525, + "tokens_seen": 158662656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007692196589769309, + "loss": 1.8113, + "theoretical_loss": 4.513911290807478, + "tokens_seen": 158728192 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007692036108324975, + "loss": 1.8606, + "theoretical_loss": 4.513671291097502, + "tokens_seen": 158793728 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007691875626880643, + "loss": 1.8731, + "theoretical_loss": 4.513431418138874, + "tokens_seen": 158859264 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007691715145436309, + "loss": 1.8097, + "theoretical_loss": 4.513191671812404, + "tokens_seen": 158924800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007691554663991977, + "loss": 1.8096, + "theoretical_loss": 4.512952051999061, + "tokens_seen": 158990336 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007691394182547643, + "loss": 1.708, + "theoretical_loss": 4.512712558579979, + "tokens_seen": 159055872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007691233701103311, + "loss": 1.8166, + "theoretical_loss": 4.512473191436451, + "tokens_seen": 159121408 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007691073219658977, + "loss": 1.7873, + "theoretical_loss": 4.512233950449927, + "tokens_seen": 159186944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007690912738214645, + "loss": 1.871, + "theoretical_loss": 4.511994835502023, + "tokens_seen": 159252480 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007690752256770311, + "loss": 1.8079, + "theoretical_loss": 4.511755846474511, + "tokens_seen": 159318016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007690591775325979, + "loss": 1.8808, + "theoretical_loss": 4.511516983249322, + "tokens_seen": 159383552 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007690431293881645, + "loss": 1.7733, + "theoretical_loss": 4.511278245708548, + "tokens_seen": 159449088 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007690270812437313, + "loss": 1.7933, + "theoretical_loss": 4.511039633734439, + "tokens_seen": 159514624 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007690110330992979, + "loss": 1.8927, + "theoretical_loss": 4.5108011472094045, + "tokens_seen": 159580160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007689949849548647, + "loss": 1.8424, + "theoretical_loss": 4.510562786016012, + "tokens_seen": 159645696 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007689789368104313, + "loss": 1.8546, + "theoretical_loss": 4.5103245500369855, + "tokens_seen": 159711232 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007689628886659981, + "loss": 1.8883, + "theoretical_loss": 4.510086439155209, + "tokens_seen": 159776768 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007689468405215647, + "loss": 1.7483, + "theoretical_loss": 4.509848453253722, + "tokens_seen": 159842304 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007689307923771315, + "loss": 1.8422, + "theoretical_loss": 4.509610592215726, + "tokens_seen": 159907840 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007689147442326981, + "loss": 1.7965, + "theoretical_loss": 4.509372855924571, + "tokens_seen": 159973376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007688986960882648, + "loss": 1.6782, + "theoretical_loss": 4.5091352442637715, + "tokens_seen": 160038912 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007688826479438315, + "loss": 1.8529, + "theoretical_loss": 4.508897757116995, + "tokens_seen": 160104448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007688665997993982, + "loss": 1.8202, + "theoretical_loss": 4.508660394368066, + "tokens_seen": 160169984 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007688505516549649, + "loss": 1.7907, + "theoretical_loss": 4.508423155900964, + "tokens_seen": 160235520 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007688345035105316, + "loss": 1.8968, + "theoretical_loss": 4.508186041599823, + "tokens_seen": 160301056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007688184553660983, + "loss": 1.8184, + "theoretical_loss": 4.5079490513489375, + "tokens_seen": 160366592 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768802407221665, + "loss": 2.0172, + "theoretical_loss": 4.50771218503275, + "tokens_seen": 160432128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007687863590772316, + "loss": 1.981, + "theoretical_loss": 4.507475442535863, + "tokens_seen": 160497664 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.9968739748001099, + "objective/train/avg_token_score": 0.003126047784462571, + "objective/train/docs_used": 65316, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.873626947402954, + "objective/train/original_loss": 1.8801671266555786, + "objective/train/theoretical_loss": 4.507238823743033, + "objective/train/tokens_used": 181023200, + "objective/train/unlikelihood_loss": 0.0062182326801121235, + "theoretical_loss": 4.507238823743033, + "tokens_seen": 160563200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007687703109327984, + "loss": 1.8803, + "theoretical_loss": 4.507238823743033, + "tokens_seen": 160563200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007687542627883651, + "loss": 1.7564, + "theoretical_loss": 4.507002328539167, + "tokens_seen": 160628736 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007687382146439318, + "loss": 1.7356, + "theoretical_loss": 4.506765956809331, + "tokens_seen": 160694272 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007687221664994985, + "loss": 1.9768, + "theoretical_loss": 4.50652970843874, + "tokens_seen": 160759808 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007687061183550652, + "loss": 1.7962, + "theoretical_loss": 4.506293583312766, + "tokens_seen": 160825344 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768690070210632, + "loss": 1.8625, + "theoretical_loss": 4.506057581316933, + "tokens_seen": 160890880 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007686740220661986, + "loss": 1.7673, + "theoretical_loss": 4.505821702336917, + "tokens_seen": 160956416 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007686579739217653, + "loss": 1.8423, + "theoretical_loss": 4.505585946258547, + "tokens_seen": 161021952 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768641925777332, + "loss": 1.836, + "theoretical_loss": 4.505350312967805, + "tokens_seen": 161087488 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007686258776328987, + "loss": 1.7907, + "theoretical_loss": 4.5051148023508265, + "tokens_seen": 161153024 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007686098294884654, + "loss": 1.6889, + "theoretical_loss": 4.504879414293896, + "tokens_seen": 161218560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007685937813440322, + "loss": 1.9021, + "theoretical_loss": 4.50464414868345, + "tokens_seen": 161284096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007685777331995988, + "loss": 1.886, + "theoretical_loss": 4.504409005406078, + "tokens_seen": 161349632 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007685616850551656, + "loss": 1.8253, + "theoretical_loss": 4.504173984348521, + "tokens_seen": 161415168 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007685456369107322, + "loss": 1.87, + "theoretical_loss": 4.503939085397667, + "tokens_seen": 161480704 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768529588766299, + "loss": 1.839, + "theoretical_loss": 4.5037043084405575, + "tokens_seen": 161546240 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007685135406218656, + "loss": 1.8864, + "theoretical_loss": 4.503469653364386, + "tokens_seen": 161611776 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007684974924774324, + "loss": 1.8118, + "theoretical_loss": 4.503235120056491, + "tokens_seen": 161677312 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768481444332999, + "loss": 1.8709, + "theoretical_loss": 4.503000708404365, + "tokens_seen": 161742848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007684653961885658, + "loss": 1.8024, + "theoretical_loss": 4.502766418295647, + "tokens_seen": 161808384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007684493480441324, + "loss": 1.9784, + "theoretical_loss": 4.502532249618129, + "tokens_seen": 161873920 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007684332998996992, + "loss": 1.9365, + "theoretical_loss": 4.502298202259748, + "tokens_seen": 161939456 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007684172517552658, + "loss": 1.7535, + "theoretical_loss": 4.502064276108593, + "tokens_seen": 162004992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007684012036108326, + "loss": 1.8173, + "theoretical_loss": 4.501830471052898, + "tokens_seen": 162070528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007683851554663992, + "loss": 1.7702, + "theoretical_loss": 4.501596786981047, + "tokens_seen": 162136064 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768369107321966, + "loss": 1.8446, + "theoretical_loss": 4.501363223781574, + "tokens_seen": 162201600 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007683530591775326, + "loss": 1.8767, + "theoretical_loss": 4.501129781343157, + "tokens_seen": 162267136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007683370110330994, + "loss": 1.8726, + "theoretical_loss": 4.500896459554624, + "tokens_seen": 162332672 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768320962888666, + "loss": 1.8045, + "theoretical_loss": 4.500663258304948, + "tokens_seen": 162398208 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007683049147442328, + "loss": 1.7604, + "theoretical_loss": 4.500430177483253, + "tokens_seen": 162463744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007682888665997994, + "loss": 1.8735, + "theoretical_loss": 4.500197216978804, + "tokens_seen": 162529280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007682728184553662, + "loss": 1.9388, + "theoretical_loss": 4.499964376681016, + "tokens_seen": 162594816 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007682567703109328, + "loss": 1.8884, + "theoretical_loss": 4.49973165647945, + "tokens_seen": 162660352 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007682407221664996, + "loss": 1.9694, + "theoretical_loss": 4.4994990562638115, + "tokens_seen": 162725888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007682246740220662, + "loss": 1.7291, + "theoretical_loss": 4.499266575923954, + "tokens_seen": 162791424 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768208625877633, + "loss": 2.0627, + "theoretical_loss": 4.499034215349875, + "tokens_seen": 162856960 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007681925777331996, + "loss": 1.8795, + "theoretical_loss": 4.498801974431714, + "tokens_seen": 162922496 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007681765295887664, + "loss": 1.8185, + "theoretical_loss": 4.498569853059762, + "tokens_seen": 162988032 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768160481444333, + "loss": 1.8305, + "theoretical_loss": 4.49833785112445, + "tokens_seen": 163053568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007681444332998998, + "loss": 1.8815, + "theoretical_loss": 4.498105968516354, + "tokens_seen": 163119104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007681283851554664, + "loss": 1.798, + "theoretical_loss": 4.497874205126195, + "tokens_seen": 163184640 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007681123370110332, + "loss": 1.8148, + "theoretical_loss": 4.497642560844838, + "tokens_seen": 163250176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007680962888665998, + "loss": 1.9132, + "theoretical_loss": 4.497411035563291, + "tokens_seen": 163315712 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007680802407221666, + "loss": 1.6896, + "theoretical_loss": 4.497179629172704, + "tokens_seen": 163381248 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007680641925777332, + "loss": 1.9411, + "theoretical_loss": 4.4969483415643765, + "tokens_seen": 163446784 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007680481444333, + "loss": 1.7564, + "theoretical_loss": 4.496717172629742, + "tokens_seen": 163512320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007680320962888666, + "loss": 1.8874, + "theoretical_loss": 4.496486122260382, + "tokens_seen": 163577856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007680160481444334, + "loss": 1.9471, + "theoretical_loss": 4.4962551903480215, + "tokens_seen": 163643392 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768, + "loss": 1.8679, + "theoretical_loss": 4.496024376784522, + "tokens_seen": 163708928 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007679839518555668, + "loss": 1.7973, + "theoretical_loss": 4.495793681461894, + "tokens_seen": 163774464 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.9943779706954956, + "objective/train/avg_token_score": 0.005622027907520533, + "objective/train/docs_used": 66569, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.7482017278671265, + "objective/train/original_loss": 1.7589516639709473, + "objective/train/theoretical_loss": 4.495563104272286, + "objective/train/tokens_used": 184300000, + "objective/train/unlikelihood_loss": 0.010292040184140205, + "theoretical_loss": 4.495563104272286, + "tokens_seen": 163840000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007679679037111334, + "loss": 1.6476, + "theoretical_loss": 4.495563104272286, + "tokens_seen": 163840000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007679518555667002, + "loss": 1.7729, + "theoretical_loss": 4.495332645107985, + "tokens_seen": 163905536 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007679358074222668, + "loss": 1.897, + "theoretical_loss": 4.495102303861426, + "tokens_seen": 163971072 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007679197592778336, + "loss": 1.8404, + "theoretical_loss": 4.4948720804251785, + "tokens_seen": 164036608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007679037111334002, + "loss": 1.8874, + "theoretical_loss": 4.494641974691959, + "tokens_seen": 164102144 + }, + { + "epoch": 0.05, + "learning_rate": 0.000767887662988967, + "loss": 1.9297, + "theoretical_loss": 4.494411986554619, + "tokens_seen": 164167680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007678716148445336, + "loss": 1.8687, + "theoretical_loss": 4.494182115906153, + "tokens_seen": 164233216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007678555667001004, + "loss": 1.8161, + "theoretical_loss": 4.493952362639694, + "tokens_seen": 164298752 + }, + { + "epoch": 0.05, + "learning_rate": 0.000767839518555667, + "loss": 1.8773, + "theoretical_loss": 4.493722726648519, + "tokens_seen": 164364288 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007678234704112338, + "loss": 1.778, + "theoretical_loss": 4.493493207826037, + "tokens_seen": 164429824 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007678074222668004, + "loss": 1.82, + "theoretical_loss": 4.493263806065803, + "tokens_seen": 164495360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007677913741223672, + "loss": 1.8206, + "theoretical_loss": 4.493034521261509, + "tokens_seen": 164560896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007677753259779338, + "loss": 1.813, + "theoretical_loss": 4.492805353306984, + "tokens_seen": 164626432 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007677592778335006, + "loss": 1.8068, + "theoretical_loss": 4.492576302096199, + "tokens_seen": 164691968 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007677432296890672, + "loss": 1.7009, + "theoretical_loss": 4.49234736752326, + "tokens_seen": 164757504 + }, + { + "epoch": 0.05, + "learning_rate": 0.000767727181544634, + "loss": 1.9323, + "theoretical_loss": 4.492118549482413, + "tokens_seen": 164823040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007677111334002006, + "loss": 1.7965, + "theoretical_loss": 4.491889847868041, + "tokens_seen": 164888576 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007676950852557674, + "loss": 1.6842, + "theoretical_loss": 4.491661262574665, + "tokens_seen": 164954112 + }, + { + "epoch": 0.05, + "learning_rate": 0.000767679037111334, + "loss": 1.7939, + "theoretical_loss": 4.491432793496945, + "tokens_seen": 165019648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007676629889669008, + "loss": 1.7263, + "theoretical_loss": 4.4912044405296765, + "tokens_seen": 165085184 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007676469408224674, + "loss": 1.7965, + "theoretical_loss": 4.49097620356779, + "tokens_seen": 165150720 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007676308926780342, + "loss": 1.785, + "theoretical_loss": 4.490748082506356, + "tokens_seen": 165216256 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007676148445336008, + "loss": 1.7683, + "theoretical_loss": 4.490520077240582, + "tokens_seen": 165281792 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007675987963891676, + "loss": 1.7251, + "theoretical_loss": 4.490292187665808, + "tokens_seen": 165347328 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007675827482447342, + "loss": 1.8289, + "theoretical_loss": 4.490064413677514, + "tokens_seen": 165412864 + }, + { + "epoch": 0.05, + "learning_rate": 0.000767566700100301, + "loss": 1.7697, + "theoretical_loss": 4.4898367551713125, + "tokens_seen": 165478400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007675506519558676, + "loss": 1.6726, + "theoretical_loss": 4.489609212042954, + "tokens_seen": 165543936 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007675346038114344, + "loss": 1.8622, + "theoretical_loss": 4.489381784188322, + "tokens_seen": 165609472 + }, + { + "epoch": 0.05, + "learning_rate": 0.000767518555667001, + "loss": 1.8227, + "theoretical_loss": 4.489154471503438, + "tokens_seen": 165675008 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007675025075225677, + "loss": 1.6693, + "theoretical_loss": 4.488927273884456, + "tokens_seen": 165740544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007674864593781344, + "loss": 1.5987, + "theoretical_loss": 4.488700191227665, + "tokens_seen": 165806080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007674704112337011, + "loss": 1.8471, + "theoretical_loss": 4.488473223429491, + "tokens_seen": 165871616 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007674543630892678, + "loss": 1.7731, + "theoretical_loss": 4.48824637038649, + "tokens_seen": 165937152 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007674383149448345, + "loss": 1.7656, + "theoretical_loss": 4.488019631995354, + "tokens_seen": 166002688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007674222668004012, + "loss": 1.7763, + "theoretical_loss": 4.48779300815291, + "tokens_seen": 166068224 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007674062186559679, + "loss": 1.7119, + "theoretical_loss": 4.4875664987561175, + "tokens_seen": 166133760 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007673901705115346, + "loss": 1.7646, + "theoretical_loss": 4.487340103702068, + "tokens_seen": 166199296 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007673741223671013, + "loss": 1.8606, + "theoretical_loss": 4.487113822887988, + "tokens_seen": 166264832 + }, + { + "epoch": 0.05, + "learning_rate": 0.000767358074222668, + "loss": 1.8356, + "theoretical_loss": 4.486887656211234, + "tokens_seen": 166330368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007673420260782347, + "loss": 1.709, + "theoretical_loss": 4.486661603569301, + "tokens_seen": 166395904 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007673259779338014, + "loss": 1.7561, + "theoretical_loss": 4.486435664859809, + "tokens_seen": 166461440 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007673099297893681, + "loss": 1.873, + "theoretical_loss": 4.486209839980516, + "tokens_seen": 166526976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007672938816449348, + "loss": 1.8096, + "theoretical_loss": 4.485984128829307, + "tokens_seen": 166592512 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007672778335005015, + "loss": 1.7806, + "theoretical_loss": 4.485758531304205, + "tokens_seen": 166658048 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007672617853560682, + "loss": 1.8499, + "theoretical_loss": 4.485533047303358, + "tokens_seen": 166723584 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007672457372116349, + "loss": 1.7658, + "theoretical_loss": 4.485307676725048, + "tokens_seen": 166789120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007672296890672016, + "loss": 1.7126, + "theoretical_loss": 4.485082419467691, + "tokens_seen": 166854656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007672136409227683, + "loss": 1.7308, + "theoretical_loss": 4.484857275429828, + "tokens_seen": 166920192 + }, + { + "epoch": 0.05, + "learning_rate": 0.000767197592778335, + "loss": 1.6489, + "theoretical_loss": 4.484632244510136, + "tokens_seen": 166985728 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007671815446339017, + "loss": 1.8373, + "theoretical_loss": 4.4844073266074185, + "tokens_seen": 167051264 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.9909823536872864, + "objective/train/avg_token_score": 0.009017640724778175, + "objective/train/docs_used": 67770, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.904003620147705, + "objective/train/original_loss": 1.9215614795684814, + "objective/train/theoretical_loss": 4.484182521620611, + "objective/train/tokens_used": 187576800, + "objective/train/unlikelihood_loss": 0.01863185316324234, + "theoretical_loss": 4.484182521620611, + "tokens_seen": 167116800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007671654964894685, + "loss": 1.8636, + "theoretical_loss": 4.484182521620611, + "tokens_seen": 167116800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007671494483450351, + "loss": 1.8222, + "theoretical_loss": 4.48395782944878, + "tokens_seen": 167182336 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007671334002006019, + "loss": 1.9068, + "theoretical_loss": 4.483733249991118, + "tokens_seen": 167247872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007671173520561685, + "loss": 1.7288, + "theoretical_loss": 4.483508783146952, + "tokens_seen": 167313408 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007671013039117353, + "loss": 1.7484, + "theoretical_loss": 4.483284428815735, + "tokens_seen": 167378944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007670852557673019, + "loss": 1.8868, + "theoretical_loss": 4.483060186897049, + "tokens_seen": 167444480 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007670692076228687, + "loss": 1.7858, + "theoretical_loss": 4.482836057290607, + "tokens_seen": 167510016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007670531594784353, + "loss": 1.8239, + "theoretical_loss": 4.4826120398962495, + "tokens_seen": 167575552 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007670371113340021, + "loss": 1.8734, + "theoretical_loss": 4.482388134613943, + "tokens_seen": 167641088 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007670210631895687, + "loss": 1.771, + "theoretical_loss": 4.482164341343788, + "tokens_seen": 167706624 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007670050150451355, + "loss": 1.6944, + "theoretical_loss": 4.481940659986008, + "tokens_seen": 167772160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007669889669007021, + "loss": 1.9042, + "theoretical_loss": 4.481717090440956, + "tokens_seen": 167837696 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007669729187562689, + "loss": 1.7988, + "theoretical_loss": 4.481493632609112, + "tokens_seen": 167903232 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007669568706118355, + "loss": 1.5783, + "theoretical_loss": 4.481270286391084, + "tokens_seen": 167968768 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007669408224674023, + "loss": 1.7788, + "theoretical_loss": 4.481047051687609, + "tokens_seen": 168034304 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007669247743229689, + "loss": 1.7843, + "theoretical_loss": 4.480823928399547, + "tokens_seen": 168099840 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007669087261785357, + "loss": 1.7638, + "theoretical_loss": 4.480600916427887, + "tokens_seen": 168165376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007668926780341023, + "loss": 1.6739, + "theoretical_loss": 4.480378015673745, + "tokens_seen": 168230912 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007668766298896691, + "loss": 1.9001, + "theoretical_loss": 4.480155226038362, + "tokens_seen": 168296448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007668605817452357, + "loss": 1.8137, + "theoretical_loss": 4.479932547423107, + "tokens_seen": 168361984 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007668445336008025, + "loss": 1.8602, + "theoretical_loss": 4.479709979729472, + "tokens_seen": 168427520 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007668284854563691, + "loss": 1.7575, + "theoretical_loss": 4.479487522859078, + "tokens_seen": 168493056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007668124373119359, + "loss": 1.887, + "theoretical_loss": 4.4792651767136675, + "tokens_seen": 168558592 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007667963891675025, + "loss": 1.8977, + "theoretical_loss": 4.479042941195113, + "tokens_seen": 168624128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007667803410230693, + "loss": 1.7128, + "theoretical_loss": 4.478820816205408, + "tokens_seen": 168689664 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007667642928786359, + "loss": 1.822, + "theoretical_loss": 4.4785988016466725, + "tokens_seen": 168755200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007667482447342027, + "loss": 1.8319, + "theoretical_loss": 4.4783768974211515, + "tokens_seen": 168820736 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007667321965897693, + "loss": 1.6294, + "theoretical_loss": 4.478155103431215, + "tokens_seen": 168886272 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007667161484453361, + "loss": 1.8536, + "theoretical_loss": 4.477933419579356, + "tokens_seen": 168951808 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007667001003009027, + "loss": 1.6348, + "theoretical_loss": 4.477711845768191, + "tokens_seen": 169017344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007666840521564695, + "loss": 1.6976, + "theoretical_loss": 4.477490381900461, + "tokens_seen": 169082880 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007666680040120361, + "loss": 1.9706, + "theoretical_loss": 4.477269027879032, + "tokens_seen": 169148416 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007666519558676029, + "loss": 1.7087, + "theoretical_loss": 4.477047783606892, + "tokens_seen": 169213952 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007666359077231695, + "loss": 1.8468, + "theoretical_loss": 4.476826648987151, + "tokens_seen": 169279488 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007666198595787363, + "loss": 1.8394, + "theoretical_loss": 4.476605623923044, + "tokens_seen": 169345024 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007666038114343029, + "loss": 1.7994, + "theoretical_loss": 4.47638470831793, + "tokens_seen": 169410560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007665877632898697, + "loss": 1.8442, + "theoretical_loss": 4.476163902075286, + "tokens_seen": 169476096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007665717151454363, + "loss": 1.8456, + "theoretical_loss": 4.475943205098716, + "tokens_seen": 169541632 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007665556670010031, + "loss": 1.9253, + "theoretical_loss": 4.475722617291944, + "tokens_seen": 169607168 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007665396188565697, + "loss": 1.8189, + "theoretical_loss": 4.4755021385588165, + "tokens_seen": 169672704 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007665235707121365, + "loss": 1.7603, + "theoretical_loss": 4.475281768803301, + "tokens_seen": 169738240 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007665075225677031, + "loss": 1.872, + "theoretical_loss": 4.475061507929487, + "tokens_seen": 169803776 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007664914744232699, + "loss": 1.5973, + "theoretical_loss": 4.474841355841587, + "tokens_seen": 169869312 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007664754262788365, + "loss": 1.9111, + "theoretical_loss": 4.474621312443932, + "tokens_seen": 169934848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007664593781344033, + "loss": 1.7829, + "theoretical_loss": 4.474401377640977, + "tokens_seen": 170000384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007664433299899699, + "loss": 1.8858, + "theoretical_loss": 4.474181551337294, + "tokens_seen": 170065920 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007664272818455367, + "loss": 1.701, + "theoretical_loss": 4.473961833437579, + "tokens_seen": 170131456 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007664112337011033, + "loss": 1.8138, + "theoretical_loss": 4.473742223846646, + "tokens_seen": 170196992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007663951855566701, + "loss": 1.8765, + "theoretical_loss": 4.473522722469431, + "tokens_seen": 170262528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007663791374122367, + "loss": 1.9526, + "theoretical_loss": 4.47330332921099, + "tokens_seen": 170328064 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.9946981072425842, + "objective/train/avg_token_score": 0.00530186528339982, + "objective/train/docs_used": 69022, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.9901429414749146, + "objective/train/original_loss": 2.002562999725342, + "objective/train/theoretical_loss": 4.473084043976495, + "objective/train/tokens_used": 190853600, + "objective/train/unlikelihood_loss": 0.00918820220977068, + "theoretical_loss": 4.473084043976495, + "tokens_seen": 170393600 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007663630892678035, + "loss": 1.8877, + "theoretical_loss": 4.473084043976495, + "tokens_seen": 170393600 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007663470411233702, + "loss": 1.9643, + "theoretical_loss": 4.472864866671245, + "tokens_seen": 170459136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007663309929789369, + "loss": 1.8938, + "theoretical_loss": 4.472645797200651, + "tokens_seen": 170524672 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007663149448345036, + "loss": 1.7126, + "theoretical_loss": 4.472426835470247, + "tokens_seen": 170590208 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007662988966900703, + "loss": 1.751, + "theoretical_loss": 4.472207981385686, + "tokens_seen": 170655744 + }, + { + "epoch": 0.05, + "learning_rate": 0.000766282848545637, + "loss": 1.8816, + "theoretical_loss": 4.471989234852737, + "tokens_seen": 170721280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007662668004012037, + "loss": 1.7975, + "theoretical_loss": 4.471770595777292, + "tokens_seen": 170786816 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007662507522567703, + "loss": 1.7783, + "theoretical_loss": 4.4715520640653565, + "tokens_seen": 170852352 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007662347041123371, + "loss": 1.7947, + "theoretical_loss": 4.471333639623059, + "tokens_seen": 170917888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007662186559679037, + "loss": 1.8314, + "theoretical_loss": 4.471115322356643, + "tokens_seen": 170983424 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007662026078234705, + "loss": 1.6414, + "theoretical_loss": 4.470897112172471, + "tokens_seen": 171048960 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007661865596790371, + "loss": 1.9883, + "theoretical_loss": 4.470679008977022, + "tokens_seen": 171114496 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007661705115346039, + "loss": 1.7695, + "theoretical_loss": 4.470461012676894, + "tokens_seen": 171180032 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007661544633901705, + "loss": 1.7598, + "theoretical_loss": 4.470243123178802, + "tokens_seen": 171245568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007661384152457373, + "loss": 1.8201, + "theoretical_loss": 4.4700253403895776, + "tokens_seen": 171311104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007661223671013039, + "loss": 1.8719, + "theoretical_loss": 4.469807664216168, + "tokens_seen": 171376640 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007661063189568706, + "loss": 1.7569, + "theoretical_loss": 4.469590094565638, + "tokens_seen": 171442176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007660902708124373, + "loss": 1.8008, + "theoretical_loss": 4.469372631345172, + "tokens_seen": 171507712 + }, + { + "epoch": 0.05, + "learning_rate": 0.000766074222668004, + "loss": 1.6999, + "theoretical_loss": 4.469155274462064, + "tokens_seen": 171573248 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007660581745235707, + "loss": 1.791, + "theoretical_loss": 4.468938023823732, + "tokens_seen": 171638784 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007660421263791374, + "loss": 1.7842, + "theoretical_loss": 4.468720879337704, + "tokens_seen": 171704320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007660260782347041, + "loss": 1.7225, + "theoretical_loss": 4.468503840911627, + "tokens_seen": 171769856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007660100300902708, + "loss": 1.8041, + "theoretical_loss": 4.46828690845326, + "tokens_seen": 171835392 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007659939819458375, + "loss": 1.7848, + "theoretical_loss": 4.468070081870483, + "tokens_seen": 171900928 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007659779338014042, + "loss": 1.6898, + "theoretical_loss": 4.467853361071284, + "tokens_seen": 171966464 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765961885656971, + "loss": 1.7422, + "theoretical_loss": 4.467636745963772, + "tokens_seen": 172032000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007659458375125376, + "loss": 1.9034, + "theoretical_loss": 4.467420236456169, + "tokens_seen": 172097536 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007659297893681043, + "loss": 1.8617, + "theoretical_loss": 4.46720383245681, + "tokens_seen": 172163072 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765913741223671, + "loss": 1.7378, + "theoretical_loss": 4.466987533874145, + "tokens_seen": 172228608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007658976930792377, + "loss": 1.81, + "theoretical_loss": 4.4667713406167415, + "tokens_seen": 172294144 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007658816449348044, + "loss": 1.7578, + "theoretical_loss": 4.466555252593277, + "tokens_seen": 172359680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007658655967903711, + "loss": 1.864, + "theoretical_loss": 4.4663392697125435, + "tokens_seen": 172425216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007658495486459378, + "loss": 1.7278, + "theoretical_loss": 4.466123391883448, + "tokens_seen": 172490752 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007658335005015046, + "loss": 1.9137, + "theoretical_loss": 4.4659076190150095, + "tokens_seen": 172556288 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007658174523570712, + "loss": 1.7369, + "theoretical_loss": 4.465691951016364, + "tokens_seen": 172621824 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765801404212638, + "loss": 1.9349, + "theoretical_loss": 4.465476387796755, + "tokens_seen": 172687360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007657853560682046, + "loss": 1.7911, + "theoretical_loss": 4.465260929265543, + "tokens_seen": 172752896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007657693079237714, + "loss": 1.817, + "theoretical_loss": 4.4650455753322005, + "tokens_seen": 172818432 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765753259779338, + "loss": 1.7813, + "theoretical_loss": 4.464830325906312, + "tokens_seen": 172883968 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007657372116349048, + "loss": 1.7616, + "theoretical_loss": 4.464615180897574, + "tokens_seen": 172949504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007657211634904714, + "loss": 1.727, + "theoretical_loss": 4.464400140215796, + "tokens_seen": 173015040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007657051153460382, + "loss": 1.8201, + "theoretical_loss": 4.4641852037708984, + "tokens_seen": 173080576 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007656890672016048, + "loss": 1.8146, + "theoretical_loss": 4.463970371472916, + "tokens_seen": 173146112 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007656730190571716, + "loss": 1.9589, + "theoretical_loss": 4.463755643231992, + "tokens_seen": 173211648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007656569709127382, + "loss": 1.7395, + "theoretical_loss": 4.463541018958384, + "tokens_seen": 173277184 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765640922768305, + "loss": 1.9165, + "theoretical_loss": 4.463326498562459, + "tokens_seen": 173342720 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007656248746238716, + "loss": 1.7174, + "theoretical_loss": 4.463112081954696, + "tokens_seen": 173408256 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007656088264794384, + "loss": 1.9194, + "theoretical_loss": 4.462897769045684, + "tokens_seen": 173473792 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765592778335005, + "loss": 1.8324, + "theoretical_loss": 4.462683559746123, + "tokens_seen": 173539328 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007655767301905718, + "loss": 1.781, + "theoretical_loss": 4.4624694539668255, + "tokens_seen": 173604864 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.9944310188293457, + "objective/train/avg_token_score": 0.005568972788751125, + "objective/train/docs_used": 70257, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.9294142723083496, + "objective/train/original_loss": 1.9412367343902588, + "objective/train/theoretical_loss": 4.462255451618713, + "objective/train/tokens_used": 194130400, + "objective/train/unlikelihood_loss": 0.009403076022863388, + "theoretical_loss": 4.462255451618713, + "tokens_seen": 173670400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007655606820461384, + "loss": 1.8894, + "theoretical_loss": 4.462255451618713, + "tokens_seen": 173670400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007655446339017052, + "loss": 1.8606, + "theoretical_loss": 4.462041552612815, + "tokens_seen": 173735936 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007655285857572718, + "loss": 1.8334, + "theoretical_loss": 4.461827756860275, + "tokens_seen": 173801472 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007655125376128386, + "loss": 1.7129, + "theoretical_loss": 4.461614064272345, + "tokens_seen": 173867008 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007654964894684052, + "loss": 1.7342, + "theoretical_loss": 4.461400474760383, + "tokens_seen": 173932544 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765480441323972, + "loss": 1.8069, + "theoretical_loss": 4.461186988235864, + "tokens_seen": 173998080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007654643931795386, + "loss": 1.6158, + "theoretical_loss": 4.460973604610366, + "tokens_seen": 174063616 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007654483450351054, + "loss": 1.7509, + "theoretical_loss": 4.4607603237955775, + "tokens_seen": 174129152 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765432296890672, + "loss": 1.7392, + "theoretical_loss": 4.460547145703298, + "tokens_seen": 174194688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007654162487462388, + "loss": 1.8033, + "theoretical_loss": 4.460334070245436, + "tokens_seen": 174260224 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007654002006018054, + "loss": 1.8391, + "theoretical_loss": 4.460121097334004, + "tokens_seen": 174325760 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007653841524573722, + "loss": 1.8849, + "theoretical_loss": 4.4599082268811285, + "tokens_seen": 174391296 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007653681043129388, + "loss": 1.6779, + "theoretical_loss": 4.459695458799041, + "tokens_seen": 174456832 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007653520561685056, + "loss": 1.6389, + "theoretical_loss": 4.459482793000083, + "tokens_seen": 174522368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007653360080240722, + "loss": 1.7377, + "theoretical_loss": 4.459270229396701, + "tokens_seen": 174587904 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765319959879639, + "loss": 1.8012, + "theoretical_loss": 4.459057767901454, + "tokens_seen": 174653440 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007653039117352056, + "loss": 1.6655, + "theoretical_loss": 4.458845408427004, + "tokens_seen": 174718976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007652878635907724, + "loss": 1.7475, + "theoretical_loss": 4.458633150886124, + "tokens_seen": 174784512 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765271815446339, + "loss": 1.8359, + "theoretical_loss": 4.458420995191691, + "tokens_seen": 174850048 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007652557673019058, + "loss": 1.9452, + "theoretical_loss": 4.458208941256691, + "tokens_seen": 174915584 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007652397191574724, + "loss": 1.9582, + "theoretical_loss": 4.457996988994218, + "tokens_seen": 174981120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007652236710130392, + "loss": 1.7833, + "theoretical_loss": 4.457785138317469, + "tokens_seen": 175046656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007652076228686058, + "loss": 1.7973, + "theoretical_loss": 4.4575733891397515, + "tokens_seen": 175112192 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007651915747241726, + "loss": 1.7082, + "theoretical_loss": 4.457361741374477, + "tokens_seen": 175177728 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007651755265797392, + "loss": 1.7878, + "theoretical_loss": 4.457150194935165, + "tokens_seen": 175243264 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765159478435306, + "loss": 1.8385, + "theoretical_loss": 4.456938749735439, + "tokens_seen": 175308800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007651434302908727, + "loss": 1.8698, + "theoretical_loss": 4.45672740568903, + "tokens_seen": 175374336 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007651273821464394, + "loss": 1.7732, + "theoretical_loss": 4.456516162709774, + "tokens_seen": 175439872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007651113340020061, + "loss": 1.8039, + "theoretical_loss": 4.456305020711613, + "tokens_seen": 175505408 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007650952858575728, + "loss": 1.7803, + "theoretical_loss": 4.456093979608594, + "tokens_seen": 175570944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007650792377131395, + "loss": 1.7729, + "theoretical_loss": 4.455883039314869, + "tokens_seen": 175636480 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007650631895687062, + "loss": 1.8687, + "theoretical_loss": 4.455672199744695, + "tokens_seen": 175702016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007650471414242729, + "loss": 1.7432, + "theoretical_loss": 4.455461460812435, + "tokens_seen": 175767552 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007650310932798396, + "loss": 1.8718, + "theoretical_loss": 4.455250822432556, + "tokens_seen": 175833088 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007650150451354063, + "loss": 1.8908, + "theoretical_loss": 4.4550402845196295, + "tokens_seen": 175898624 + }, + { + "epoch": 0.05, + "learning_rate": 0.000764998996990973, + "loss": 1.8422, + "theoretical_loss": 4.45482984698833, + "tokens_seen": 175964160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007649829488465397, + "loss": 1.7663, + "theoretical_loss": 4.454619509753439, + "tokens_seen": 176029696 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007649669007021064, + "loss": 1.7312, + "theoretical_loss": 4.45440927272984, + "tokens_seen": 176095232 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007649508525576731, + "loss": 1.7961, + "theoretical_loss": 4.454199135832521, + "tokens_seen": 176160768 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007649348044132398, + "loss": 1.851, + "theoretical_loss": 4.4539890989765745, + "tokens_seen": 176226304 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007649187562688065, + "loss": 1.8954, + "theoretical_loss": 4.453779162077193, + "tokens_seen": 176291840 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007649027081243732, + "loss": 1.7616, + "theoretical_loss": 4.453569325049679, + "tokens_seen": 176357376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007648866599799399, + "loss": 1.818, + "theoretical_loss": 4.453359587809431, + "tokens_seen": 176422912 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007648706118355066, + "loss": 1.7646, + "theoretical_loss": 4.453149950271956, + "tokens_seen": 176488448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007648545636910732, + "loss": 1.9652, + "theoretical_loss": 4.452940412352861, + "tokens_seen": 176553984 + }, + { + "epoch": 0.05, + "learning_rate": 0.00076483851554664, + "loss": 1.9658, + "theoretical_loss": 4.452730973967856, + "tokens_seen": 176619520 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007648224674022066, + "loss": 1.6993, + "theoretical_loss": 4.452521635032754, + "tokens_seen": 176685056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007648064192577734, + "loss": 1.8999, + "theoretical_loss": 4.452312395463473, + "tokens_seen": 176750592 + }, + { + "epoch": 0.05, + "learning_rate": 0.00076479037111334, + "loss": 1.8277, + "theoretical_loss": 4.452103255176027, + "tokens_seen": 176816128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007647743229689068, + "loss": 1.8924, + "theoretical_loss": 4.451894214086537, + "tokens_seen": 176881664 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.9935407638549805, + "objective/train/avg_token_score": 0.006459188647568226, + "objective/train/docs_used": 71338, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6354304552078247, + "objective/train/original_loss": 1.6453888416290283, + "objective/train/theoretical_loss": 4.4516852721112254, + "objective/train/tokens_used": 197407200, + "objective/train/unlikelihood_loss": 0.016938256099820137, + "theoretical_loss": 4.4516852721112254, + "tokens_seen": 176947200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007647582748244734, + "loss": 1.7371, + "theoretical_loss": 4.4516852721112254, + "tokens_seen": 176947200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007647422266800402, + "loss": 1.7488, + "theoretical_loss": 4.451476429166416, + "tokens_seen": 177012736 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007647261785356068, + "loss": 1.8475, + "theoretical_loss": 4.451267685168532, + "tokens_seen": 177078272 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007647101303911735, + "loss": 1.9216, + "theoretical_loss": 4.451059040034102, + "tokens_seen": 177143808 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007646940822467402, + "loss": 1.7831, + "theoretical_loss": 4.450850493679752, + "tokens_seen": 177209344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007646780341023069, + "loss": 1.9057, + "theoretical_loss": 4.450642046022211, + "tokens_seen": 177274880 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007646619859578736, + "loss": 1.755, + "theoretical_loss": 4.45043369697831, + "tokens_seen": 177340416 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007646459378134403, + "loss": 1.7529, + "theoretical_loss": 4.450225446464978, + "tokens_seen": 177405952 + }, + { + "epoch": 0.05, + "learning_rate": 0.000764629889669007, + "loss": 1.8573, + "theoretical_loss": 4.450017294399247, + "tokens_seen": 177471488 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007646138415245737, + "loss": 1.7997, + "theoretical_loss": 4.449809240698249, + "tokens_seen": 177537024 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007645977933801404, + "loss": 1.7281, + "theoretical_loss": 4.449601285279215, + "tokens_seen": 177602560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007645817452357071, + "loss": 1.7175, + "theoretical_loss": 4.449393428059478, + "tokens_seen": 177668096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007645656970912738, + "loss": 1.8621, + "theoretical_loss": 4.449185668956469, + "tokens_seen": 177733632 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007645496489468405, + "loss": 1.8144, + "theoretical_loss": 4.448978007887721, + "tokens_seen": 177799168 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007645336008024072, + "loss": 1.6543, + "theoretical_loss": 4.448770444770865, + "tokens_seen": 177864704 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007645175526579739, + "loss": 1.7658, + "theoretical_loss": 4.448562979523633, + "tokens_seen": 177930240 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007645015045135406, + "loss": 1.7849, + "theoretical_loss": 4.448355612063855, + "tokens_seen": 177995776 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007644854563691073, + "loss": 1.8178, + "theoretical_loss": 4.4481483423094605, + "tokens_seen": 178061312 + }, + { + "epoch": 0.05, + "learning_rate": 0.000764469408224674, + "loss": 1.8114, + "theoretical_loss": 4.447941170178478, + "tokens_seen": 178126848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007644533600802407, + "loss": 1.8069, + "theoretical_loss": 4.447734095589036, + "tokens_seen": 178192384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007644373119358075, + "loss": 1.7815, + "theoretical_loss": 4.4475271184593606, + "tokens_seen": 178257920 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007644212637913741, + "loss": 1.6862, + "theoretical_loss": 4.447320238707777, + "tokens_seen": 178323456 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007644052156469409, + "loss": 1.8957, + "theoretical_loss": 4.447113456252707, + "tokens_seen": 178388992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007643891675025075, + "loss": 1.6996, + "theoretical_loss": 4.446906771012674, + "tokens_seen": 178454528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007643731193580743, + "loss": 1.6346, + "theoretical_loss": 4.446700182906298, + "tokens_seen": 178520064 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007643570712136409, + "loss": 1.8653, + "theoretical_loss": 4.446493691852297, + "tokens_seen": 178585600 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007643410230692077, + "loss": 1.7501, + "theoretical_loss": 4.4462872977694845, + "tokens_seen": 178651136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007643249749247743, + "loss": 1.8622, + "theoretical_loss": 4.4460810005767755, + "tokens_seen": 178716672 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007643089267803411, + "loss": 1.7562, + "theoretical_loss": 4.445874800193181, + "tokens_seen": 178782208 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007642928786359077, + "loss": 1.6799, + "theoretical_loss": 4.445668696537808, + "tokens_seen": 178847744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007642768304914745, + "loss": 1.7356, + "theoretical_loss": 4.4454626895298635, + "tokens_seen": 178913280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007642607823470411, + "loss": 1.5906, + "theoretical_loss": 4.4452567790886475, + "tokens_seen": 178978816 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007642447342026079, + "loss": 1.8824, + "theoretical_loss": 4.445050965133561, + "tokens_seen": 179044352 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007642286860581745, + "loss": 1.7332, + "theoretical_loss": 4.4448452475841, + "tokens_seen": 179109888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007642126379137413, + "loss": 1.8927, + "theoretical_loss": 4.444639626359856, + "tokens_seen": 179175424 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007641965897693079, + "loss": 1.869, + "theoretical_loss": 4.444434101380519, + "tokens_seen": 179240960 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007641805416248747, + "loss": 1.9356, + "theoretical_loss": 4.444228672565874, + "tokens_seen": 179306496 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007641644934804413, + "loss": 1.9195, + "theoretical_loss": 4.444023339835803, + "tokens_seen": 179372032 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007641484453360081, + "loss": 1.966, + "theoretical_loss": 4.443818103110283, + "tokens_seen": 179437568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007641323971915747, + "loss": 1.6917, + "theoretical_loss": 4.443612962309387, + "tokens_seen": 179503104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007641163490471415, + "loss": 1.8172, + "theoretical_loss": 4.4434079173532846, + "tokens_seen": 179568640 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007641003009027081, + "loss": 1.8232, + "theoretical_loss": 4.4432029681622405, + "tokens_seen": 179634176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007640842527582749, + "loss": 1.7897, + "theoretical_loss": 4.442998114656614, + "tokens_seen": 179699712 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007640682046138415, + "loss": 1.8752, + "theoretical_loss": 4.442793356756861, + "tokens_seen": 179765248 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007640521564694083, + "loss": 1.7798, + "theoretical_loss": 4.442588694383532, + "tokens_seen": 179830784 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007640361083249749, + "loss": 1.7163, + "theoretical_loss": 4.44238412745727, + "tokens_seen": 179896320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007640200601805417, + "loss": 1.8447, + "theoretical_loss": 4.442179655898819, + "tokens_seen": 179961856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007640040120361083, + "loss": 1.81, + "theoretical_loss": 4.44197527962901, + "tokens_seen": 180027392 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007639879638916751, + "loss": 1.8709, + "theoretical_loss": 4.441770998568773, + "tokens_seen": 180092928 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007639719157472417, + "loss": 1.8827, + "theoretical_loss": 4.441566812639133, + "tokens_seen": 180158464 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.9942334294319153, + "objective/train/avg_token_score": 0.005766597576439381, + "objective/train/docs_used": 72550, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.750836730003357, + "objective/train/original_loss": 1.760735034942627, + "objective/train/theoretical_loss": 4.441362721761207, + "objective/train/tokens_used": 200684000, + "objective/train/unlikelihood_loss": 0.012741424143314362, + "theoretical_loss": 4.441362721761207, + "tokens_seen": 180224000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007639558676028085, + "loss": 1.817, + "theoretical_loss": 4.441362721761207, + "tokens_seen": 180224000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007639398194583752, + "loss": 1.9522, + "theoretical_loss": 4.441158725856207, + "tokens_seen": 180289536 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007639237713139419, + "loss": 1.9461, + "theoretical_loss": 4.440954824845438, + "tokens_seen": 180355072 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007639077231695086, + "loss": 1.9136, + "theoretical_loss": 4.4407510186503, + "tokens_seen": 180420608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007638916750250753, + "loss": 1.7778, + "theoretical_loss": 4.440547307192286, + "tokens_seen": 180486144 + }, + { + "epoch": 0.05, + "learning_rate": 0.000763875626880642, + "loss": 1.679, + "theoretical_loss": 4.440343690392984, + "tokens_seen": 180551680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007638595787362087, + "loss": 1.7731, + "theoretical_loss": 4.4401401681740715, + "tokens_seen": 180617216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007638435305917754, + "loss": 1.79, + "theoretical_loss": 4.439936740457323, + "tokens_seen": 180682752 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007638274824473421, + "loss": 1.8851, + "theoretical_loss": 4.439733407164605, + "tokens_seen": 180748288 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007638114343029088, + "loss": 1.8397, + "theoretical_loss": 4.439530168217875, + "tokens_seen": 180813824 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007637953861584755, + "loss": 1.6643, + "theoretical_loss": 4.439327023539187, + "tokens_seen": 180879360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007637793380140422, + "loss": 1.8099, + "theoretical_loss": 4.439123973050684, + "tokens_seen": 180944896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007637632898696089, + "loss": 1.868, + "theoretical_loss": 4.438921016674603, + "tokens_seen": 181010432 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007637472417251756, + "loss": 1.7284, + "theoretical_loss": 4.438718154333274, + "tokens_seen": 181075968 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007637311935807423, + "loss": 1.8348, + "theoretical_loss": 4.438515385949118, + "tokens_seen": 181141504 + }, + { + "epoch": 0.05, + "learning_rate": 0.000763715145436309, + "loss": 1.7687, + "theoretical_loss": 4.438312711444647, + "tokens_seen": 181207040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007636990972918757, + "loss": 1.8861, + "theoretical_loss": 4.438110130742469, + "tokens_seen": 181272576 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007636830491474424, + "loss": 1.8207, + "theoretical_loss": 4.437907643765281, + "tokens_seen": 181338112 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007636670010030091, + "loss": 1.8004, + "theoretical_loss": 4.43770525043587, + "tokens_seen": 181403648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007636509528585758, + "loss": 1.9886, + "theoretical_loss": 4.437502950677116, + "tokens_seen": 181469184 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007636349047141425, + "loss": 1.8199, + "theoretical_loss": 4.437300744411992, + "tokens_seen": 181534720 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007636188565697092, + "loss": 1.8323, + "theoretical_loss": 4.43709863156356, + "tokens_seen": 181600256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007636028084252759, + "loss": 1.7526, + "theoretical_loss": 4.436896612054975, + "tokens_seen": 181665792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007635867602808426, + "loss": 1.7715, + "theoretical_loss": 4.43669468580948, + "tokens_seen": 181731328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007635707121364093, + "loss": 1.8929, + "theoretical_loss": 4.43649285275041, + "tokens_seen": 181796864 + }, + { + "epoch": 0.06, + "learning_rate": 0.000763554663991976, + "loss": 1.7224, + "theoretical_loss": 4.436291112801193, + "tokens_seen": 181862400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007635386158475427, + "loss": 1.87, + "theoretical_loss": 4.436089465885345, + "tokens_seen": 181927936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007635225677031094, + "loss": 1.7974, + "theoretical_loss": 4.435887911926472, + "tokens_seen": 181993472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007635065195586761, + "loss": 1.8037, + "theoretical_loss": 4.435686450848273, + "tokens_seen": 182059008 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007634904714142428, + "loss": 1.8285, + "theoretical_loss": 4.435485082574533, + "tokens_seen": 182124544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007634744232698095, + "loss": 1.8503, + "theoretical_loss": 4.43528380702913, + "tokens_seen": 182190080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007634583751253762, + "loss": 1.7588, + "theoretical_loss": 4.435082624136031, + "tokens_seen": 182255616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007634423269809429, + "loss": 1.9123, + "theoretical_loss": 4.4348815338192935, + "tokens_seen": 182321152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007634262788365095, + "loss": 1.7426, + "theoretical_loss": 4.434680536003062, + "tokens_seen": 182386688 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007634102306920763, + "loss": 1.7688, + "theoretical_loss": 4.434479630611573, + "tokens_seen": 182452224 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007633941825476429, + "loss": 1.8957, + "theoretical_loss": 4.43427881756915, + "tokens_seen": 182517760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007633781344032097, + "loss": 1.8272, + "theoretical_loss": 4.434078096800209, + "tokens_seen": 182583296 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007633620862587763, + "loss": 1.8454, + "theoretical_loss": 4.433877468229251, + "tokens_seen": 182648832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007633460381143431, + "loss": 1.8089, + "theoretical_loss": 4.433676931780868, + "tokens_seen": 182714368 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007633299899699097, + "loss": 1.9271, + "theoretical_loss": 4.43347648737974, + "tokens_seen": 182779904 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007633139418254764, + "loss": 1.9429, + "theoretical_loss": 4.433276134950637, + "tokens_seen": 182845440 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007632978936810431, + "loss": 1.7343, + "theoretical_loss": 4.4330758744184156, + "tokens_seen": 182910976 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007632818455366098, + "loss": 1.6758, + "theoretical_loss": 4.432875705708021, + "tokens_seen": 182976512 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007632657973921765, + "loss": 1.8231, + "theoretical_loss": 4.432675628744488, + "tokens_seen": 183042048 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007632497492477432, + "loss": 1.8566, + "theoretical_loss": 4.432475643452937, + "tokens_seen": 183107584 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007632337011033099, + "loss": 1.9801, + "theoretical_loss": 4.432275749758579, + "tokens_seen": 183173120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007632176529588766, + "loss": 1.8579, + "theoretical_loss": 4.432075947586711, + "tokens_seen": 183238656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007632016048144433, + "loss": 1.7271, + "theoretical_loss": 4.431876236862717, + "tokens_seen": 183304192 + }, + { + "epoch": 0.06, + "learning_rate": 0.00076318555667001, + "loss": 1.7076, + "theoretical_loss": 4.43167661751207, + "tokens_seen": 183369728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007631695085255767, + "loss": 1.7598, + "theoretical_loss": 4.431477089460331, + "tokens_seen": 183435264 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.9887768030166626, + "objective/train/avg_token_score": 0.011223235167562962, + "objective/train/docs_used": 73723, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.867813229560852, + "objective/train/original_loss": 1.8948664665222168, + "objective/train/theoretical_loss": 4.431277652633145, + "objective/train/tokens_used": 203960800, + "objective/train/unlikelihood_loss": 0.014880944043397903, + "theoretical_loss": 4.431277652633145, + "tokens_seen": 183500800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007631534603811434, + "loss": 1.8047, + "theoretical_loss": 4.431277652633145, + "tokens_seen": 183500800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007631374122367101, + "loss": 1.8672, + "theoretical_loss": 4.431078306956248, + "tokens_seen": 183566336 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007631213640922768, + "loss": 1.8235, + "theoretical_loss": 4.43087905235546, + "tokens_seen": 183631872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007631053159478436, + "loss": 1.77, + "theoretical_loss": 4.43067988875669, + "tokens_seen": 183697408 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007630892678034102, + "loss": 1.7896, + "theoretical_loss": 4.430480816085931, + "tokens_seen": 183762944 + }, + { + "epoch": 0.06, + "learning_rate": 0.000763073219658977, + "loss": 1.8521, + "theoretical_loss": 4.430281834269265, + "tokens_seen": 183828480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007630571715145436, + "loss": 1.685, + "theoretical_loss": 4.4300829432328594, + "tokens_seen": 183894016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007630411233701104, + "loss": 1.8334, + "theoretical_loss": 4.429884142902968, + "tokens_seen": 183959552 + }, + { + "epoch": 0.06, + "learning_rate": 0.000763025075225677, + "loss": 1.7994, + "theoretical_loss": 4.429685433205931, + "tokens_seen": 184025088 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007630090270812438, + "loss": 1.8333, + "theoretical_loss": 4.429486814068174, + "tokens_seen": 184090624 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007629929789368104, + "loss": 1.7718, + "theoretical_loss": 4.4292882854162094, + "tokens_seen": 184156160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007629769307923772, + "loss": 1.8121, + "theoretical_loss": 4.429089847176634, + "tokens_seen": 184221696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007629608826479438, + "loss": 1.845, + "theoretical_loss": 4.428891499276132, + "tokens_seen": 184287232 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007629448345035106, + "loss": 1.916, + "theoretical_loss": 4.42869324164147, + "tokens_seen": 184352768 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007629287863590772, + "loss": 1.805, + "theoretical_loss": 4.428495074199505, + "tokens_seen": 184418304 + }, + { + "epoch": 0.06, + "learning_rate": 0.000762912738214644, + "loss": 1.8162, + "theoretical_loss": 4.428296996877176, + "tokens_seen": 184483840 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007628966900702106, + "loss": 1.8145, + "theoretical_loss": 4.428099009601506, + "tokens_seen": 184549376 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007628806419257774, + "loss": 1.7821, + "theoretical_loss": 4.4279011122996055, + "tokens_seen": 184614912 + }, + { + "epoch": 0.06, + "learning_rate": 0.000762864593781344, + "loss": 1.9208, + "theoretical_loss": 4.427703304898668, + "tokens_seen": 184680448 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007628485456369108, + "loss": 1.7444, + "theoretical_loss": 4.427505587325974, + "tokens_seen": 184745984 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007628324974924774, + "loss": 1.7567, + "theoretical_loss": 4.427307959508885, + "tokens_seen": 184811520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007628164493480442, + "loss": 1.7581, + "theoretical_loss": 4.4271104213748504, + "tokens_seen": 184877056 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007628004012036108, + "loss": 1.7546, + "theoretical_loss": 4.426912972851404, + "tokens_seen": 184942592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007627843530591776, + "loss": 1.7524, + "theoretical_loss": 4.426715613866159, + "tokens_seen": 185008128 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007627683049147442, + "loss": 1.9131, + "theoretical_loss": 4.426518344346819, + "tokens_seen": 185073664 + }, + { + "epoch": 0.06, + "learning_rate": 0.000762752256770311, + "loss": 1.9347, + "theoretical_loss": 4.426321164221168, + "tokens_seen": 185139200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007627362086258777, + "loss": 1.875, + "theoretical_loss": 4.426124073417074, + "tokens_seen": 185204736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007627201604814444, + "loss": 1.7577, + "theoretical_loss": 4.42592707186249, + "tokens_seen": 185270272 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007627041123370111, + "loss": 1.7702, + "theoretical_loss": 4.42573015948545, + "tokens_seen": 185335808 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007626880641925778, + "loss": 1.9289, + "theoretical_loss": 4.425533336214076, + "tokens_seen": 185401344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007626720160481445, + "loss": 1.8138, + "theoretical_loss": 4.425336601976568, + "tokens_seen": 185466880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007626559679037112, + "loss": 1.8969, + "theoretical_loss": 4.425139956701214, + "tokens_seen": 185532416 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007626399197592779, + "loss": 1.7621, + "theoretical_loss": 4.42494340031638, + "tokens_seen": 185597952 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007626238716148446, + "loss": 1.8985, + "theoretical_loss": 4.424746932750521, + "tokens_seen": 185663488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007626078234704113, + "loss": 1.7934, + "theoretical_loss": 4.4245505539321694, + "tokens_seen": 185729024 + }, + { + "epoch": 0.06, + "learning_rate": 0.000762591775325978, + "loss": 1.8054, + "theoretical_loss": 4.424354263789943, + "tokens_seen": 185794560 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007625757271815447, + "loss": 1.7986, + "theoretical_loss": 4.424158062252541, + "tokens_seen": 185860096 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007625596790371114, + "loss": 1.8248, + "theoretical_loss": 4.423961949248748, + "tokens_seen": 185925632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007625436308926781, + "loss": 1.7429, + "theoretical_loss": 4.423765924707427, + "tokens_seen": 185991168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007625275827482448, + "loss": 1.7813, + "theoretical_loss": 4.4235699885575235, + "tokens_seen": 186056704 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007625115346038115, + "loss": 1.7665, + "theoretical_loss": 4.423374140728068, + "tokens_seen": 186122240 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007624954864593782, + "loss": 1.7068, + "theoretical_loss": 4.423178381148172, + "tokens_seen": 186187776 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007624794383149449, + "loss": 1.7859, + "theoretical_loss": 4.422982709747026, + "tokens_seen": 186253312 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007624633901705116, + "loss": 1.7412, + "theoretical_loss": 4.4227871264539065, + "tokens_seen": 186318848 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007624473420260783, + "loss": 1.7398, + "theoretical_loss": 4.422591631198167, + "tokens_seen": 186384384 + }, + { + "epoch": 0.06, + "learning_rate": 0.000762431293881645, + "loss": 1.7081, + "theoretical_loss": 4.422396223909248, + "tokens_seen": 186449920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007624152457372117, + "loss": 1.7199, + "theoretical_loss": 4.422200904516666, + "tokens_seen": 186515456 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007623991975927784, + "loss": 1.7317, + "theoretical_loss": 4.4220056729500214, + "tokens_seen": 186580992 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007623831494483451, + "loss": 1.8218, + "theoretical_loss": 4.4218105291389955, + "tokens_seen": 186646528 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007623671013039118, + "loss": 1.8548, + "theoretical_loss": 4.4216154730133495, + "tokens_seen": 186712064 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.9930493831634521, + "objective/train/avg_token_score": 0.006950636859983206, + "objective/train/docs_used": 74904, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.786165475845337, + "objective/train/original_loss": 1.8000397682189941, + "objective/train/theoretical_loss": 4.421420504502928, + "objective/train/tokens_used": 207237600, + "objective/train/unlikelihood_loss": 0.014263229444622993, + "theoretical_loss": 4.421420504502928, + "tokens_seen": 186777600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007623510531594785, + "loss": 1.763, + "theoretical_loss": 4.421420504502928, + "tokens_seen": 186777600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007623350050150452, + "loss": 1.9021, + "theoretical_loss": 4.421225623537652, + "tokens_seen": 186843136 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007623189568706119, + "loss": 1.8862, + "theoretical_loss": 4.421030830047528, + "tokens_seen": 186908672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007623029087261786, + "loss": 1.9296, + "theoretical_loss": 4.420836123962639, + "tokens_seen": 186974208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007622868605817453, + "loss": 1.7276, + "theoretical_loss": 4.420641505213151, + "tokens_seen": 187039744 + }, + { + "epoch": 0.06, + "learning_rate": 0.000762270812437312, + "loss": 1.7449, + "theoretical_loss": 4.42044697372931, + "tokens_seen": 187105280 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007622547642928787, + "loss": 1.7714, + "theoretical_loss": 4.420252529441439, + "tokens_seen": 187170816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007622387161484454, + "loss": 1.8943, + "theoretical_loss": 4.420058172279945, + "tokens_seen": 187236352 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007622226680040121, + "loss": 1.8347, + "theoretical_loss": 4.419863902175313, + "tokens_seen": 187301888 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007622066198595788, + "loss": 1.8171, + "theoretical_loss": 4.419669719058108, + "tokens_seen": 187367424 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007621905717151455, + "loss": 1.8921, + "theoretical_loss": 4.419475622858974, + "tokens_seen": 187432960 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007621745235707122, + "loss": 1.8484, + "theoretical_loss": 4.419281613508637, + "tokens_seen": 187498496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007621584754262789, + "loss": 1.9184, + "theoretical_loss": 4.419087690937898, + "tokens_seen": 187564032 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007621424272818456, + "loss": 1.8848, + "theoretical_loss": 4.418893855077641, + "tokens_seen": 187629568 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007621263791374123, + "loss": 1.795, + "theoretical_loss": 4.4187001058588296, + "tokens_seen": 187695104 + }, + { + "epoch": 0.06, + "learning_rate": 0.000762110330992979, + "loss": 1.8575, + "theoretical_loss": 4.418506443212503, + "tokens_seen": 187760640 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007620942828485457, + "loss": 1.8445, + "theoretical_loss": 4.4183128670697815, + "tokens_seen": 187826176 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007620782347041124, + "loss": 1.92, + "theoretical_loss": 4.418119377361863, + "tokens_seen": 187891712 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007620621865596791, + "loss": 1.8095, + "theoretical_loss": 4.417925974020028, + "tokens_seen": 187957248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007620461384152458, + "loss": 1.8159, + "theoretical_loss": 4.4177326569756294, + "tokens_seen": 188022784 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007620300902708124, + "loss": 1.856, + "theoretical_loss": 4.417539426160104, + "tokens_seen": 188088320 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007620140421263792, + "loss": 1.7934, + "theoretical_loss": 4.417346281504963, + "tokens_seen": 188153856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007619979939819458, + "loss": 1.8367, + "theoretical_loss": 4.417153222941798, + "tokens_seen": 188219392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007619819458375126, + "loss": 1.87, + "theoretical_loss": 4.416960250402279, + "tokens_seen": 188284928 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007619658976930792, + "loss": 1.8527, + "theoretical_loss": 4.416767363818153, + "tokens_seen": 188350464 + }, + { + "epoch": 0.06, + "learning_rate": 0.000761949849548646, + "loss": 1.711, + "theoretical_loss": 4.4165745631212445, + "tokens_seen": 188416000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007619338014042126, + "loss": 1.8081, + "theoretical_loss": 4.416381848243456, + "tokens_seen": 188481536 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007619177532597794, + "loss": 1.7475, + "theoretical_loss": 4.41618921911677, + "tokens_seen": 188547072 + }, + { + "epoch": 0.06, + "learning_rate": 0.000761901705115346, + "loss": 1.9302, + "theoretical_loss": 4.415996675673242, + "tokens_seen": 188612608 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007618856569709127, + "loss": 1.9279, + "theoretical_loss": 4.415804217845009, + "tokens_seen": 188678144 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007618696088264794, + "loss": 1.9807, + "theoretical_loss": 4.415611845564284, + "tokens_seen": 188743680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007618535606820461, + "loss": 1.8004, + "theoretical_loss": 4.415419558763356, + "tokens_seen": 188809216 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007618375125376128, + "loss": 1.8989, + "theoretical_loss": 4.415227357374592, + "tokens_seen": 188874752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007618214643931795, + "loss": 1.8131, + "theoretical_loss": 4.415035241330436, + "tokens_seen": 188940288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007618054162487462, + "loss": 1.8214, + "theoretical_loss": 4.41484321056341, + "tokens_seen": 189005824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007617893681043129, + "loss": 1.8488, + "theoretical_loss": 4.4146512650061105, + "tokens_seen": 189071360 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007617733199598796, + "loss": 1.8534, + "theoretical_loss": 4.414459404591211, + "tokens_seen": 189136896 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007617572718154463, + "loss": 1.6961, + "theoretical_loss": 4.414267629251463, + "tokens_seen": 189202432 + }, + { + "epoch": 0.06, + "learning_rate": 0.000761741223671013, + "loss": 1.7408, + "theoretical_loss": 4.414075938919694, + "tokens_seen": 189267968 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007617251755265797, + "loss": 1.8545, + "theoretical_loss": 4.413884333528807, + "tokens_seen": 189333504 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007617091273821465, + "loss": 1.7706, + "theoretical_loss": 4.413692813011781, + "tokens_seen": 189399040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007616930792377131, + "loss": 1.7566, + "theoretical_loss": 4.413501377301671, + "tokens_seen": 189464576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007616770310932799, + "loss": 1.7333, + "theoretical_loss": 4.41331002633161, + "tokens_seen": 189530112 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007616609829488465, + "loss": 1.8923, + "theoretical_loss": 4.413118760034804, + "tokens_seen": 189595648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007616449348044133, + "loss": 1.8218, + "theoretical_loss": 4.4129275783445365, + "tokens_seen": 189661184 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007616288866599799, + "loss": 1.7808, + "theoretical_loss": 4.412736481194166, + "tokens_seen": 189726720 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007616128385155467, + "loss": 1.8102, + "theoretical_loss": 4.412545468517127, + "tokens_seen": 189792256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007615967903711133, + "loss": 1.8356, + "theoretical_loss": 4.412354540246929, + "tokens_seen": 189857792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007615807422266801, + "loss": 1.6864, + "theoretical_loss": 4.4121636963171555, + "tokens_seen": 189923328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007615646940822467, + "loss": 1.8967, + "theoretical_loss": 4.411972936661467, + "tokens_seen": 189988864 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.9934732913970947, + "objective/train/avg_token_score": 0.006526707671582699, + "objective/train/docs_used": 76077, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.798323392868042, + "objective/train/original_loss": 1.8105287551879883, + "objective/train/theoretical_loss": 4.411782261213599, + "objective/train/tokens_used": 210514400, + "objective/train/unlikelihood_loss": 0.012906456366181374, + "theoretical_loss": 4.411782261213599, + "tokens_seen": 190054400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007615486459378135, + "loss": 1.8631, + "theoretical_loss": 4.411782261213599, + "tokens_seen": 190054400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007615325977933802, + "loss": 1.8066, + "theoretical_loss": 4.41159166990736, + "tokens_seen": 190119936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007615165496489469, + "loss": 1.8524, + "theoretical_loss": 4.411401162676636, + "tokens_seen": 190185472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007615005015045136, + "loss": 1.9581, + "theoretical_loss": 4.411210739455386, + "tokens_seen": 190251008 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007614844533600803, + "loss": 1.7575, + "theoretical_loss": 4.4110204001776445, + "tokens_seen": 190316544 + }, + { + "epoch": 0.06, + "learning_rate": 0.000761468405215647, + "loss": 1.6456, + "theoretical_loss": 4.410830144777519, + "tokens_seen": 190382080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007614523570712137, + "loss": 1.694, + "theoretical_loss": 4.410639973189192, + "tokens_seen": 190447616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007614363089267804, + "loss": 1.7971, + "theoretical_loss": 4.410449885346922, + "tokens_seen": 190513152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007614202607823471, + "loss": 1.9162, + "theoretical_loss": 4.4102598811850395, + "tokens_seen": 190578688 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007614042126379138, + "loss": 1.8671, + "theoretical_loss": 4.410069960637949, + "tokens_seen": 190644224 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007613881644934805, + "loss": 1.8626, + "theoretical_loss": 4.409880123640131, + "tokens_seen": 190709760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007613721163490472, + "loss": 1.8039, + "theoretical_loss": 4.409690370126139, + "tokens_seen": 190775296 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007613560682046139, + "loss": 1.8787, + "theoretical_loss": 4.409500700030598, + "tokens_seen": 190840832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007613400200601806, + "loss": 1.9032, + "theoretical_loss": 4.409311113288211, + "tokens_seen": 190906368 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007613239719157473, + "loss": 1.8354, + "theoretical_loss": 4.409121609833749, + "tokens_seen": 190971904 + }, + { + "epoch": 0.06, + "learning_rate": 0.000761307923771314, + "loss": 1.7094, + "theoretical_loss": 4.408932189602061, + "tokens_seen": 191037440 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007612918756268807, + "loss": 1.8854, + "theoretical_loss": 4.408742852528068, + "tokens_seen": 191102976 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007612758274824474, + "loss": 1.7192, + "theoretical_loss": 4.408553598546764, + "tokens_seen": 191168512 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007612597793380141, + "loss": 1.7811, + "theoretical_loss": 4.408364427593215, + "tokens_seen": 191234048 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007612437311935808, + "loss": 1.8821, + "theoretical_loss": 4.40817533960256, + "tokens_seen": 191299584 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007612276830491475, + "loss": 1.7331, + "theoretical_loss": 4.407986334510014, + "tokens_seen": 191365120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007612116349047142, + "loss": 1.9862, + "theoretical_loss": 4.407797412250861, + "tokens_seen": 191430656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007611955867602809, + "loss": 1.8037, + "theoretical_loss": 4.40760857276046, + "tokens_seen": 191496192 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007611795386158476, + "loss": 1.6942, + "theoretical_loss": 4.407419815974242, + "tokens_seen": 191561728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007611634904714143, + "loss": 1.8165, + "theoretical_loss": 4.407231141827708, + "tokens_seen": 191627264 + }, + { + "epoch": 0.06, + "learning_rate": 0.000761147442326981, + "loss": 1.8573, + "theoretical_loss": 4.407042550256436, + "tokens_seen": 191692800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007611313941825477, + "loss": 1.8497, + "theoretical_loss": 4.406854041196072, + "tokens_seen": 191758336 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007611153460381144, + "loss": 1.7271, + "theoretical_loss": 4.406665614582338, + "tokens_seen": 191823872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007610992978936811, + "loss": 1.7927, + "theoretical_loss": 4.406477270351024, + "tokens_seen": 191889408 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007610832497492478, + "loss": 1.7864, + "theoretical_loss": 4.4062890084379935, + "tokens_seen": 191954944 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007610672016048145, + "loss": 1.6902, + "theoretical_loss": 4.406100828779183, + "tokens_seen": 192020480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007610511534603812, + "loss": 1.6773, + "theoretical_loss": 4.4059127313106, + "tokens_seen": 192086016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007610351053159479, + "loss": 1.7898, + "theoretical_loss": 4.405724715968323, + "tokens_seen": 192151552 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007610190571715146, + "loss": 1.8132, + "theoretical_loss": 4.405536782688502, + "tokens_seen": 192217088 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007610030090270813, + "loss": 1.8654, + "theoretical_loss": 4.405348931407359, + "tokens_seen": 192282624 + }, + { + "epoch": 0.06, + "learning_rate": 0.000760986960882648, + "loss": 1.8136, + "theoretical_loss": 4.405161162061187, + "tokens_seen": 192348160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007609709127382147, + "loss": 1.742, + "theoretical_loss": 4.404973474586351, + "tokens_seen": 192413696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007609548645937814, + "loss": 1.8915, + "theoretical_loss": 4.404785868919285, + "tokens_seen": 192479232 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007609388164493481, + "loss": 1.6682, + "theoretical_loss": 4.404598344996497, + "tokens_seen": 192544768 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007609227683049148, + "loss": 1.6898, + "theoretical_loss": 4.404410902754562, + "tokens_seen": 192610304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007609067201604815, + "loss": 1.7817, + "theoretical_loss": 4.404223542130129, + "tokens_seen": 192675840 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007608906720160482, + "loss": 1.7535, + "theoretical_loss": 4.404036263059917, + "tokens_seen": 192741376 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007608746238716149, + "loss": 1.7785, + "theoretical_loss": 4.403849065480714, + "tokens_seen": 192806912 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007608585757271816, + "loss": 1.8921, + "theoretical_loss": 4.403661949329381, + "tokens_seen": 192872448 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007608425275827483, + "loss": 1.8855, + "theoretical_loss": 4.403474914542848, + "tokens_seen": 192937984 + }, + { + "epoch": 0.06, + "learning_rate": 0.000760826479438315, + "loss": 1.7981, + "theoretical_loss": 4.403287961058112, + "tokens_seen": 193003520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007608104312938817, + "loss": 1.6811, + "theoretical_loss": 4.403101088812248, + "tokens_seen": 193069056 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007607943831494484, + "loss": 1.8354, + "theoretical_loss": 4.4029142977423925, + "tokens_seen": 193134592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007607783350050151, + "loss": 1.7412, + "theoretical_loss": 4.402727587785758, + "tokens_seen": 193200128 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007607622868605818, + "loss": 1.8035, + "theoretical_loss": 4.402540958879625, + "tokens_seen": 193265664 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.993314802646637, + "objective/train/avg_token_score": 0.006685143802314997, + "objective/train/docs_used": 77266, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.8666588068008423, + "objective/train/original_loss": 1.8811280727386475, + "objective/train/theoretical_loss": 4.402354410961341, + "objective/train/tokens_used": 213791200, + "objective/train/unlikelihood_loss": 0.010974266566336155, + "theoretical_loss": 4.402354410961341, + "tokens_seen": 193331200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007607462387161485, + "loss": 1.7911, + "theoretical_loss": 4.402354410961341, + "tokens_seen": 193331200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007607301905717152, + "loss": 1.7189, + "theoretical_loss": 4.402167943968328, + "tokens_seen": 193396736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007607141424272819, + "loss": 1.7096, + "theoretical_loss": 4.4019815578380745, + "tokens_seen": 193462272 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007606980942828486, + "loss": 1.7312, + "theoretical_loss": 4.401795252508139, + "tokens_seen": 193527808 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007606820461384153, + "loss": 1.699, + "theoretical_loss": 4.401609027916149, + "tokens_seen": 193593344 + }, + { + "epoch": 0.06, + "learning_rate": 0.000760665997993982, + "loss": 1.6363, + "theoretical_loss": 4.4014228839998015, + "tokens_seen": 193658880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007606499498495487, + "loss": 1.7868, + "theoretical_loss": 4.401236820696863, + "tokens_seen": 193724416 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007606339017051153, + "loss": 1.7412, + "theoretical_loss": 4.40105083794517, + "tokens_seen": 193789952 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007606178535606821, + "loss": 1.7256, + "theoretical_loss": 4.400864935682625, + "tokens_seen": 193855488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007606018054162487, + "loss": 1.7487, + "theoretical_loss": 4.400679113847202, + "tokens_seen": 193921024 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007605857572718155, + "loss": 1.9484, + "theoretical_loss": 4.400493372376943, + "tokens_seen": 193986560 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007605697091273821, + "loss": 1.7401, + "theoretical_loss": 4.400307711209957, + "tokens_seen": 194052096 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007605536609829489, + "loss": 1.8065, + "theoretical_loss": 4.400122130284426, + "tokens_seen": 194117632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007605376128385155, + "loss": 1.7246, + "theoretical_loss": 4.399936629538595, + "tokens_seen": 194183168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007605215646940823, + "loss": 1.6109, + "theoretical_loss": 4.3997512089107795, + "tokens_seen": 194248704 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007605055165496489, + "loss": 1.7196, + "theoretical_loss": 4.399565868339366, + "tokens_seen": 194314240 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007604894684052156, + "loss": 1.8048, + "theoretical_loss": 4.399380607762804, + "tokens_seen": 194379776 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007604734202607823, + "loss": 1.7681, + "theoretical_loss": 4.399195427119616, + "tokens_seen": 194445312 + }, + { + "epoch": 0.06, + "learning_rate": 0.000760457372116349, + "loss": 1.8348, + "theoretical_loss": 4.399010326348388, + "tokens_seen": 194510848 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007604413239719157, + "loss": 1.7913, + "theoretical_loss": 4.398825305387778, + "tokens_seen": 194576384 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007604252758274824, + "loss": 1.8542, + "theoretical_loss": 4.39864036417651, + "tokens_seen": 194641920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007604092276830491, + "loss": 1.754, + "theoretical_loss": 4.398455502653373, + "tokens_seen": 194707456 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007603931795386158, + "loss": 1.8536, + "theoretical_loss": 4.398270720757228, + "tokens_seen": 194772992 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007603771313941825, + "loss": 1.9014, + "theoretical_loss": 4.398086018427001, + "tokens_seen": 194838528 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007603610832497492, + "loss": 1.7964, + "theoretical_loss": 4.397901395601687, + "tokens_seen": 194904064 + }, + { + "epoch": 0.06, + "learning_rate": 0.000760345035105316, + "loss": 1.8661, + "theoretical_loss": 4.397716852220345, + "tokens_seen": 194969600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007603289869608827, + "loss": 1.8565, + "theoretical_loss": 4.397532388222105, + "tokens_seen": 195035136 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007603129388164494, + "loss": 1.8363, + "theoretical_loss": 4.397348003546161, + "tokens_seen": 195100672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007602968906720161, + "loss": 1.7331, + "theoretical_loss": 4.397163698131776, + "tokens_seen": 195166208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007602808425275828, + "loss": 1.6883, + "theoretical_loss": 4.39697947191828, + "tokens_seen": 195231744 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007602647943831495, + "loss": 1.814, + "theoretical_loss": 4.396795324845069, + "tokens_seen": 195297280 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007602487462387162, + "loss": 1.7562, + "theoretical_loss": 4.396611256851604, + "tokens_seen": 195362816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007602326980942829, + "loss": 1.7164, + "theoretical_loss": 4.3964272678774154, + "tokens_seen": 195428352 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007602166499498496, + "loss": 1.7153, + "theoretical_loss": 4.396243357862099, + "tokens_seen": 195493888 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007602006018054163, + "loss": 1.7891, + "theoretical_loss": 4.396059526745318, + "tokens_seen": 195559424 + }, + { + "epoch": 0.06, + "learning_rate": 0.000760184553660983, + "loss": 1.754, + "theoretical_loss": 4.395875774466799, + "tokens_seen": 195624960 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007601685055165497, + "loss": 1.7688, + "theoretical_loss": 4.395692100966338, + "tokens_seen": 195690496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007601524573721164, + "loss": 1.6594, + "theoretical_loss": 4.395508506183796, + "tokens_seen": 195756032 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007601364092276831, + "loss": 1.9462, + "theoretical_loss": 4.395324990059098, + "tokens_seen": 195821568 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007601203610832498, + "loss": 1.7497, + "theoretical_loss": 4.395141552532239, + "tokens_seen": 195887104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007601043129388165, + "loss": 1.8605, + "theoretical_loss": 4.394958193543277, + "tokens_seen": 195952640 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007600882647943832, + "loss": 1.8013, + "theoretical_loss": 4.394774913032337, + "tokens_seen": 196018176 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007600722166499499, + "loss": 1.7137, + "theoretical_loss": 4.394591710939608, + "tokens_seen": 196083712 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007600561685055166, + "loss": 1.6889, + "theoretical_loss": 4.394408587205344, + "tokens_seen": 196149248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007600401203610833, + "loss": 1.7887, + "theoretical_loss": 4.39422554176987, + "tokens_seen": 196214784 + }, + { + "epoch": 0.06, + "learning_rate": 0.00076002407221665, + "loss": 1.6679, + "theoretical_loss": 4.39404257457357, + "tokens_seen": 196280320 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007600080240722167, + "loss": 1.6537, + "theoretical_loss": 4.393859685556895, + "tokens_seen": 196345856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007599919759277834, + "loss": 1.8375, + "theoretical_loss": 4.393676874660364, + "tokens_seen": 196411392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007599759277833501, + "loss": 1.7172, + "theoretical_loss": 4.393494141824557, + "tokens_seen": 196476928 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007599598796389168, + "loss": 1.7485, + "theoretical_loss": 4.393311486990122, + "tokens_seen": 196542464 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.9933859705924988, + "objective/train/avg_token_score": 0.006614040117710829, + "objective/train/docs_used": 78313, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5652415752410889, + "objective/train/original_loss": 1.5773851871490479, + "objective/train/theoretical_loss": 4.39312891009777, + "objective/train/tokens_used": 217068000, + "objective/train/unlikelihood_loss": 0.012543030083179474, + "theoretical_loss": 4.39312891009777, + "tokens_seen": 196608000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007599438314944835, + "loss": 1.6603, + "theoretical_loss": 4.39312891009777, + "tokens_seen": 196608000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007599277833500502, + "loss": 1.757, + "theoretical_loss": 4.392946411088279, + "tokens_seen": 196673536 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007599117352056169, + "loss": 1.8165, + "theoretical_loss": 4.392763989902489, + "tokens_seen": 196739072 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007598956870611836, + "loss": 1.7385, + "theoretical_loss": 4.3925816464813074, + "tokens_seen": 196804608 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007598796389167503, + "loss": 1.7646, + "theoretical_loss": 4.392399380765704, + "tokens_seen": 196870144 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759863590772317, + "loss": 1.7494, + "theoretical_loss": 4.392217192696712, + "tokens_seen": 196935680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007598475426278837, + "loss": 1.8743, + "theoretical_loss": 4.392035082215434, + "tokens_seen": 197001216 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007598314944834504, + "loss": 1.7299, + "theoretical_loss": 4.391853049263032, + "tokens_seen": 197066752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007598154463390171, + "loss": 1.75, + "theoretical_loss": 4.391671093780733, + "tokens_seen": 197132288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007597993981945838, + "loss": 1.651, + "theoretical_loss": 4.39148921570983, + "tokens_seen": 197197824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007597833500501505, + "loss": 1.5816, + "theoretical_loss": 4.391307414991678, + "tokens_seen": 197263360 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007597673019057172, + "loss": 1.8284, + "theoretical_loss": 4.391125691567697, + "tokens_seen": 197328896 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007597512537612839, + "loss": 1.8216, + "theoretical_loss": 4.390944045379371, + "tokens_seen": 197394432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007597352056168506, + "loss": 1.7825, + "theoretical_loss": 4.390762476368245, + "tokens_seen": 197459968 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007597191574724173, + "loss": 1.759, + "theoretical_loss": 4.390580984475932, + "tokens_seen": 197525504 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759703109327984, + "loss": 1.825, + "theoretical_loss": 4.3903995696441065, + "tokens_seen": 197591040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007596870611835507, + "loss": 1.8749, + "theoretical_loss": 4.390218231814504, + "tokens_seen": 197656576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007596710130391174, + "loss": 1.6858, + "theoretical_loss": 4.390036970928929, + "tokens_seen": 197722112 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007596549648946841, + "loss": 1.6813, + "theoretical_loss": 4.3898557869292425, + "tokens_seen": 197787648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007596389167502508, + "loss": 1.7493, + "theoretical_loss": 4.389674679757374, + "tokens_seen": 197853184 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007596228686058175, + "loss": 1.6932, + "theoretical_loss": 4.389493649355314, + "tokens_seen": 197918720 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007596068204613842, + "loss": 1.7031, + "theoretical_loss": 4.389312695665115, + "tokens_seen": 197984256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007595907723169509, + "loss": 1.8176, + "theoretical_loss": 4.389131818628895, + "tokens_seen": 198049792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007595747241725176, + "loss": 1.7857, + "theoretical_loss": 4.388951018188832, + "tokens_seen": 198115328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007595586760280843, + "loss": 1.7777, + "theoretical_loss": 4.388770294287169, + "tokens_seen": 198180864 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759542627883651, + "loss": 1.8292, + "theoretical_loss": 4.38858964686621, + "tokens_seen": 198246400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007595265797392177, + "loss": 1.828, + "theoretical_loss": 4.388409075868323, + "tokens_seen": 198311936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007595105315947844, + "loss": 1.8577, + "theoretical_loss": 4.388228581235937, + "tokens_seen": 198377472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007594944834503511, + "loss": 1.8075, + "theoretical_loss": 4.388048162911545, + "tokens_seen": 198443008 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007594784353059178, + "loss": 1.7316, + "theoretical_loss": 4.3878678208377, + "tokens_seen": 198508544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007594623871614845, + "loss": 1.8025, + "theoretical_loss": 4.38768755495702, + "tokens_seen": 198574080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007594463390170512, + "loss": 1.7651, + "theoretical_loss": 4.387507365212182, + "tokens_seen": 198639616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007594302908726179, + "loss": 1.729, + "theoretical_loss": 4.387327251545927, + "tokens_seen": 198705152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007594142427281846, + "loss": 1.7251, + "theoretical_loss": 4.387147213901059, + "tokens_seen": 198770688 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007593981945837513, + "loss": 1.7039, + "theoretical_loss": 4.386967252220441, + "tokens_seen": 198836224 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759382146439318, + "loss": 1.8385, + "theoretical_loss": 4.386787366447, + "tokens_seen": 198901760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007593660982948847, + "loss": 1.8467, + "theoretical_loss": 4.386607556523723, + "tokens_seen": 198967296 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007593500501504514, + "loss": 1.8394, + "theoretical_loss": 4.386427822393661, + "tokens_seen": 199032832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007593340020060181, + "loss": 1.7751, + "theoretical_loss": 4.386248163999923, + "tokens_seen": 199098368 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007593179538615848, + "loss": 1.8344, + "theoretical_loss": 4.3860685812856826, + "tokens_seen": 199163904 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007593019057171515, + "loss": 1.858, + "theoretical_loss": 4.385889074194173, + "tokens_seen": 199229440 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007592858575727182, + "loss": 1.7245, + "theoretical_loss": 4.385709642668688, + "tokens_seen": 199294976 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759269809428285, + "loss": 1.6607, + "theoretical_loss": 4.385530286652586, + "tokens_seen": 199360512 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007592537612838516, + "loss": 1.7094, + "theoretical_loss": 4.385351006089283, + "tokens_seen": 199426048 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007592377131394182, + "loss": 1.7187, + "theoretical_loss": 4.385171800922257, + "tokens_seen": 199491584 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759221664994985, + "loss": 1.7698, + "theoretical_loss": 4.384992671095047, + "tokens_seen": 199557120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007592056168505516, + "loss": 1.7154, + "theoretical_loss": 4.384813616551252, + "tokens_seen": 199622656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007591895687061184, + "loss": 1.6795, + "theoretical_loss": 4.384634637234534, + "tokens_seen": 199688192 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759173520561685, + "loss": 1.7635, + "theoretical_loss": 4.384455733088612, + "tokens_seen": 199753728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007591574724172518, + "loss": 1.771, + "theoretical_loss": 4.38427690405727, + "tokens_seen": 199819264 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.9942682981491089, + "objective/train/avg_token_score": 0.005731688812375069, + "objective/train/docs_used": 79488, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6610815525054932, + "objective/train/original_loss": 1.668832540512085, + "objective/train/theoretical_loss": 4.384098150084349, + "objective/train/tokens_used": 220344800, + "objective/train/unlikelihood_loss": 0.01913726143538952, + "theoretical_loss": 4.384098150084349, + "tokens_seen": 199884800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007591414242728184, + "loss": 1.7197, + "theoretical_loss": 4.384098150084349, + "tokens_seen": 199884800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007591253761283852, + "loss": 1.8096, + "theoretical_loss": 4.383919471113751, + "tokens_seen": 199950336 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007591093279839518, + "loss": 1.8548, + "theoretical_loss": 4.383740867089441, + "tokens_seen": 200015872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007590932798395186, + "loss": 1.7602, + "theoretical_loss": 4.383562337955439, + "tokens_seen": 200081408 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007590772316950852, + "loss": 1.6317, + "theoretical_loss": 4.3833838836558305, + "tokens_seen": 200146944 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759061183550652, + "loss": 1.7867, + "theoretical_loss": 4.383205504134758, + "tokens_seen": 200212480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007590451354062186, + "loss": 1.8332, + "theoretical_loss": 4.383027199336424, + "tokens_seen": 200278016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007590290872617854, + "loss": 1.7941, + "theoretical_loss": 4.382848969205092, + "tokens_seen": 200343552 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759013039117352, + "loss": 1.8805, + "theoretical_loss": 4.3826708136850865, + "tokens_seen": 200409088 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007589969909729188, + "loss": 1.8173, + "theoretical_loss": 4.382492732720786, + "tokens_seen": 200474624 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007589809428284855, + "loss": 1.8867, + "theoretical_loss": 4.382314726256637, + "tokens_seen": 200540160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007589648946840522, + "loss": 1.7143, + "theoretical_loss": 4.382136794237138, + "tokens_seen": 200605696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007589488465396189, + "loss": 1.8014, + "theoretical_loss": 4.3819589366068525, + "tokens_seen": 200671232 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007589327983951856, + "loss": 1.714, + "theoretical_loss": 4.381781153310399, + "tokens_seen": 200736768 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007589167502507523, + "loss": 1.7549, + "theoretical_loss": 4.381603444292459, + "tokens_seen": 200802304 + }, + { + "epoch": 0.06, + "learning_rate": 0.000758900702106319, + "loss": 1.6745, + "theoretical_loss": 4.38142580949777, + "tokens_seen": 200867840 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007588846539618857, + "loss": 1.7348, + "theoretical_loss": 4.381248248871131, + "tokens_seen": 200933376 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007588686058174524, + "loss": 1.7389, + "theoretical_loss": 4.3810707623574, + "tokens_seen": 200998912 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007588525576730191, + "loss": 1.7598, + "theoretical_loss": 4.380893349901491, + "tokens_seen": 201064448 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007588365095285858, + "loss": 1.6704, + "theoretical_loss": 4.380716011448381, + "tokens_seen": 201129984 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007588204613841525, + "loss": 1.7883, + "theoretical_loss": 4.380538746943103, + "tokens_seen": 201195520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007588044132397192, + "loss": 1.6518, + "theoretical_loss": 4.380361556330749, + "tokens_seen": 201261056 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007587883650952859, + "loss": 1.6166, + "theoretical_loss": 4.380184439556472, + "tokens_seen": 201326592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007587723169508526, + "loss": 1.7479, + "theoretical_loss": 4.38000739656548, + "tokens_seen": 201392128 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007587562688064193, + "loss": 1.7554, + "theoretical_loss": 4.379830427303041, + "tokens_seen": 201457664 + }, + { + "epoch": 0.06, + "learning_rate": 0.000758740220661986, + "loss": 1.7234, + "theoretical_loss": 4.379653531714483, + "tokens_seen": 201523200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007587241725175527, + "loss": 1.8675, + "theoretical_loss": 4.37947670974519, + "tokens_seen": 201588736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007587081243731194, + "loss": 1.6707, + "theoretical_loss": 4.379299961340605, + "tokens_seen": 201654272 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007586920762286861, + "loss": 1.8227, + "theoretical_loss": 4.37912328644623, + "tokens_seen": 201719808 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007586760280842528, + "loss": 1.796, + "theoretical_loss": 4.378946685007623, + "tokens_seen": 201785344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007586599799398195, + "loss": 1.8331, + "theoretical_loss": 4.378770156970403, + "tokens_seen": 201850880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007586439317953862, + "loss": 1.7457, + "theoretical_loss": 4.378593702280243, + "tokens_seen": 201916416 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007586278836509529, + "loss": 1.718, + "theoretical_loss": 4.378417320882879, + "tokens_seen": 201981952 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007586118355065196, + "loss": 1.6547, + "theoretical_loss": 4.378241012724099, + "tokens_seen": 202047488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007585957873620863, + "loss": 1.8034, + "theoretical_loss": 4.378064777749753, + "tokens_seen": 202113024 + }, + { + "epoch": 0.06, + "learning_rate": 0.000758579739217653, + "loss": 1.7008, + "theoretical_loss": 4.377888615905746, + "tokens_seen": 202178560 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007585636910732197, + "loss": 1.7677, + "theoretical_loss": 4.377712527138042, + "tokens_seen": 202244096 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007585476429287864, + "loss": 1.8097, + "theoretical_loss": 4.3775365113926625, + "tokens_seen": 202309632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007585315947843531, + "loss": 1.6586, + "theoretical_loss": 4.377360568615685, + "tokens_seen": 202375168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007585155466399198, + "loss": 1.7483, + "theoretical_loss": 4.377184698753244, + "tokens_seen": 202440704 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007584994984954865, + "loss": 1.6592, + "theoretical_loss": 4.377008901751536, + "tokens_seen": 202506240 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007584834503510532, + "loss": 1.7582, + "theoretical_loss": 4.376833177556806, + "tokens_seen": 202571776 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007584674022066199, + "loss": 1.7947, + "theoretical_loss": 4.3766575261153635, + "tokens_seen": 202637312 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007584513540621866, + "loss": 1.8119, + "theoretical_loss": 4.3764819473735725, + "tokens_seen": 202702848 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007584353059177533, + "loss": 1.7078, + "theoretical_loss": 4.376306441277852, + "tokens_seen": 202768384 + }, + { + "epoch": 0.06, + "learning_rate": 0.00075841925777332, + "loss": 1.7157, + "theoretical_loss": 4.376131007774681, + "tokens_seen": 202833920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007584032096288867, + "loss": 1.7593, + "theoretical_loss": 4.375955646810594, + "tokens_seen": 202899456 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007583871614844534, + "loss": 1.6621, + "theoretical_loss": 4.37578035833218, + "tokens_seen": 202964992 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007583711133400201, + "loss": 1.6953, + "theoretical_loss": 4.375605142286087, + "tokens_seen": 203030528 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007583550651955868, + "loss": 1.7572, + "theoretical_loss": 4.37542999861902, + "tokens_seen": 203096064 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.9904599189758301, + "objective/train/avg_token_score": 0.009540031664073467, + "objective/train/docs_used": 80605, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.8388686180114746, + "objective/train/original_loss": 1.855396032333374, + "objective/train/theoretical_loss": 4.375254927277737, + "objective/train/tokens_used": 223621600, + "objective/train/unlikelihood_loss": 0.01867806538939476, + "theoretical_loss": 4.375254927277737, + "tokens_seen": 203161600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007583390170511535, + "loss": 1.8452, + "theoretical_loss": 4.375254927277737, + "tokens_seen": 203161600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007583229689067202, + "loss": 1.84, + "theoretical_loss": 4.375079928209058, + "tokens_seen": 203227136 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007583069207622869, + "loss": 1.6646, + "theoretical_loss": 4.374905001359851, + "tokens_seen": 203292672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007582908726178536, + "loss": 1.7615, + "theoretical_loss": 4.374730146677049, + "tokens_seen": 203358208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007582748244734203, + "loss": 1.7963, + "theoretical_loss": 4.374555364107635, + "tokens_seen": 203423744 + }, + { + "epoch": 0.06, + "learning_rate": 0.000758258776328987, + "loss": 1.7445, + "theoretical_loss": 4.37438065359865, + "tokens_seen": 203489280 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007582427281845537, + "loss": 1.8021, + "theoretical_loss": 4.374206015097193, + "tokens_seen": 203554816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007582266800401204, + "loss": 1.6555, + "theoretical_loss": 4.374031448550414, + "tokens_seen": 203620352 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007582106318956871, + "loss": 1.7419, + "theoretical_loss": 4.373856953905523, + "tokens_seen": 203685888 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007581945837512538, + "loss": 1.8524, + "theoretical_loss": 4.373682531109785, + "tokens_seen": 203751424 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007581785356068205, + "loss": 1.7556, + "theoretical_loss": 4.37350818011052, + "tokens_seen": 203816960 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007581624874623872, + "loss": 1.8314, + "theoretical_loss": 4.3733339008551, + "tokens_seen": 203882496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007581464393179539, + "loss": 1.7717, + "theoretical_loss": 4.373159693290961, + "tokens_seen": 203948032 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007581303911735206, + "loss": 1.7378, + "theoretical_loss": 4.372985557365586, + "tokens_seen": 204013568 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007581143430290873, + "loss": 1.7574, + "theoretical_loss": 4.372811493026518, + "tokens_seen": 204079104 + }, + { + "epoch": 0.06, + "learning_rate": 0.000758098294884654, + "loss": 1.7176, + "theoretical_loss": 4.3726375002213524, + "tokens_seen": 204144640 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007580822467402207, + "loss": 1.6532, + "theoretical_loss": 4.372463578897743, + "tokens_seen": 204210176 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007580661985957874, + "loss": 1.7816, + "theoretical_loss": 4.372289729003397, + "tokens_seen": 204275712 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007580501504513541, + "loss": 1.6903, + "theoretical_loss": 4.372115950486075, + "tokens_seen": 204341248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007580341023069208, + "loss": 1.8494, + "theoretical_loss": 4.371942243293595, + "tokens_seen": 204406784 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007580180541624875, + "loss": 1.7826, + "theoretical_loss": 4.371768607373829, + "tokens_seen": 204472320 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007580020060180542, + "loss": 1.8092, + "theoretical_loss": 4.371595042674704, + "tokens_seen": 204537856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007579859578736209, + "loss": 1.6333, + "theoretical_loss": 4.371421549144202, + "tokens_seen": 204603392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007579699097291876, + "loss": 1.7658, + "theoretical_loss": 4.371248126730357, + "tokens_seen": 204668928 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007579538615847543, + "loss": 1.912, + "theoretical_loss": 4.3710747753812615, + "tokens_seen": 204734464 + }, + { + "epoch": 0.06, + "learning_rate": 0.000757937813440321, + "loss": 1.6706, + "theoretical_loss": 4.370901495045061, + "tokens_seen": 204800000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007579217652958877, + "loss": 1.859, + "theoretical_loss": 4.370728285669954, + "tokens_seen": 204865536 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007579057171514544, + "loss": 1.8376, + "theoretical_loss": 4.370555147204195, + "tokens_seen": 204931072 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007578896690070211, + "loss": 1.7519, + "theoretical_loss": 4.370382079596092, + "tokens_seen": 204996608 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007578736208625878, + "loss": 1.8746, + "theoretical_loss": 4.370209082794006, + "tokens_seen": 205062144 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007578575727181545, + "loss": 1.7587, + "theoretical_loss": 4.370036156746357, + "tokens_seen": 205127680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007578415245737211, + "loss": 1.6571, + "theoretical_loss": 4.369863301401612, + "tokens_seen": 205193216 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007578254764292879, + "loss": 1.918, + "theoretical_loss": 4.369690516708298, + "tokens_seen": 205258752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007578094282848545, + "loss": 1.7532, + "theoretical_loss": 4.3695178026149915, + "tokens_seen": 205324288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007577933801404213, + "loss": 1.7141, + "theoretical_loss": 4.369345159070327, + "tokens_seen": 205389824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007577773319959879, + "loss": 1.6605, + "theoretical_loss": 4.369172586022989, + "tokens_seen": 205455360 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007577612838515547, + "loss": 1.7915, + "theoretical_loss": 4.369000083421716, + "tokens_seen": 205520896 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007577452357071213, + "loss": 1.719, + "theoretical_loss": 4.368827651215304, + "tokens_seen": 205586432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007577291875626881, + "loss": 1.7059, + "theoretical_loss": 4.368655289352598, + "tokens_seen": 205651968 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007577131394182547, + "loss": 1.7597, + "theoretical_loss": 4.3684829977824995, + "tokens_seen": 205717504 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007576970912738215, + "loss": 1.6569, + "theoretical_loss": 4.368310776453961, + "tokens_seen": 205783040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007576810431293881, + "loss": 1.7009, + "theoretical_loss": 4.368138625315989, + "tokens_seen": 205848576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007576649949849549, + "loss": 1.6754, + "theoretical_loss": 4.367966544317646, + "tokens_seen": 205914112 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007576489468405215, + "loss": 1.7726, + "theoretical_loss": 4.367794533408043, + "tokens_seen": 205979648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007576328986960883, + "loss": 1.6297, + "theoretical_loss": 4.367622592536348, + "tokens_seen": 206045184 + }, + { + "epoch": 0.06, + "learning_rate": 0.000757616850551655, + "loss": 1.7266, + "theoretical_loss": 4.36745072165178, + "tokens_seen": 206110720 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007576008024072217, + "loss": 1.7622, + "theoretical_loss": 4.36727892070361, + "tokens_seen": 206176256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007575847542627884, + "loss": 1.7344, + "theoretical_loss": 4.367107189641167, + "tokens_seen": 206241792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007575687061183551, + "loss": 1.7342, + "theoretical_loss": 4.3669355284138245, + "tokens_seen": 206307328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007575526579739218, + "loss": 1.7906, + "theoretical_loss": 4.366763936971017, + "tokens_seen": 206372864 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.9905458092689514, + "objective/train/avg_token_score": 0.009454199112951756, + "objective/train/docs_used": 81724, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.724741816520691, + "objective/train/original_loss": 1.7386188507080078, + "objective/train/theoretical_loss": 4.366592415262226, + "objective/train/tokens_used": 226898400, + "objective/train/unlikelihood_loss": 0.02593975141644478, + "theoretical_loss": 4.366592415262226, + "tokens_seen": 206438400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007575366098294885, + "loss": 1.7222, + "theoretical_loss": 4.366592415262226, + "tokens_seen": 206438400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007575205616850552, + "loss": 1.8016, + "theoretical_loss": 4.366420963236989, + "tokens_seen": 206503936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007575045135406219, + "loss": 1.8836, + "theoretical_loss": 4.366249580844894, + "tokens_seen": 206569472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007574884653961886, + "loss": 1.783, + "theoretical_loss": 4.366078268035581, + "tokens_seen": 206635008 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007574724172517553, + "loss": 1.7338, + "theoretical_loss": 4.365907024758744, + "tokens_seen": 206700544 + }, + { + "epoch": 0.06, + "learning_rate": 0.000757456369107322, + "loss": 1.8539, + "theoretical_loss": 4.365735850964131, + "tokens_seen": 206766080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007574403209628887, + "loss": 1.6896, + "theoretical_loss": 4.365564746601537, + "tokens_seen": 206831616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007574242728184554, + "loss": 1.8819, + "theoretical_loss": 4.365393711620814, + "tokens_seen": 206897152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007574082246740221, + "loss": 1.7455, + "theoretical_loss": 4.365222745971863, + "tokens_seen": 206962688 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007573921765295888, + "loss": 1.5057, + "theoretical_loss": 4.365051849604638, + "tokens_seen": 207028224 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007573761283851555, + "loss": 1.7737, + "theoretical_loss": 4.364881022469148, + "tokens_seen": 207093760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007573600802407222, + "loss": 1.8228, + "theoretical_loss": 4.364710264515448, + "tokens_seen": 207159296 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007573440320962889, + "loss": 1.6674, + "theoretical_loss": 4.36453957569365, + "tokens_seen": 207224832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007573279839518556, + "loss": 1.7271, + "theoretical_loss": 4.364368955953913, + "tokens_seen": 207290368 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007573119358074223, + "loss": 1.6785, + "theoretical_loss": 4.364198405246453, + "tokens_seen": 207355904 + }, + { + "epoch": 0.06, + "learning_rate": 0.000757295887662989, + "loss": 1.7583, + "theoretical_loss": 4.364027923521535, + "tokens_seen": 207421440 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007572798395185557, + "loss": 1.6698, + "theoretical_loss": 4.363857510729473, + "tokens_seen": 207486976 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007572637913741224, + "loss": 1.7983, + "theoretical_loss": 4.363687166820637, + "tokens_seen": 207552512 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007572477432296891, + "loss": 1.8567, + "theoretical_loss": 4.363516891745445, + "tokens_seen": 207618048 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007572316950852558, + "loss": 1.8839, + "theoretical_loss": 4.36334668545437, + "tokens_seen": 207683584 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007572156469408225, + "loss": 1.8074, + "theoretical_loss": 4.363176547897931, + "tokens_seen": 207749120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007571995987963892, + "loss": 1.7782, + "theoretical_loss": 4.363006479026702, + "tokens_seen": 207814656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007571835506519559, + "loss": 1.7803, + "theoretical_loss": 4.3628364787913085, + "tokens_seen": 207880192 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007571675025075226, + "loss": 1.6615, + "theoretical_loss": 4.362666547142424, + "tokens_seen": 207945728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007571514543630893, + "loss": 1.5731, + "theoretical_loss": 4.362496684030775, + "tokens_seen": 208011264 + }, + { + "epoch": 0.06, + "learning_rate": 0.000757135406218656, + "loss": 1.7008, + "theoretical_loss": 4.36232688940714, + "tokens_seen": 208076800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007571193580742227, + "loss": 1.8844, + "theoretical_loss": 4.362157163222346, + "tokens_seen": 208142336 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007571033099297894, + "loss": 1.7922, + "theoretical_loss": 4.361987505427272, + "tokens_seen": 208207872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007570872617853561, + "loss": 1.8327, + "theoretical_loss": 4.361817915972846, + "tokens_seen": 208273408 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007570712136409228, + "loss": 1.778, + "theoretical_loss": 4.36164839481005, + "tokens_seen": 208338944 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007570551654964895, + "loss": 1.908, + "theoretical_loss": 4.361478941889915, + "tokens_seen": 208404480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007570391173520562, + "loss": 1.7921, + "theoretical_loss": 4.36130955716352, + "tokens_seen": 208470016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007570230692076229, + "loss": 1.7252, + "theoretical_loss": 4.361140240581998, + "tokens_seen": 208535552 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007570070210631896, + "loss": 1.7745, + "theoretical_loss": 4.36097099209653, + "tokens_seen": 208601088 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007569909729187563, + "loss": 1.7632, + "theoretical_loss": 4.36080181165835, + "tokens_seen": 208666624 + }, + { + "epoch": 0.06, + "learning_rate": 0.000756974924774323, + "loss": 1.6888, + "theoretical_loss": 4.360632699218739, + "tokens_seen": 208732160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007569588766298897, + "loss": 1.6976, + "theoretical_loss": 4.360463654729029, + "tokens_seen": 208797696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007569428284854564, + "loss": 1.7605, + "theoretical_loss": 4.360294678140605, + "tokens_seen": 208863232 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007569267803410231, + "loss": 1.77, + "theoretical_loss": 4.360125769404897, + "tokens_seen": 208928768 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007569107321965898, + "loss": 1.8416, + "theoretical_loss": 4.35995692847339, + "tokens_seen": 208994304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007568946840521565, + "loss": 1.6641, + "theoretical_loss": 4.359788155297615, + "tokens_seen": 209059840 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007568786359077232, + "loss": 1.6892, + "theoretical_loss": 4.359619449829156, + "tokens_seen": 209125376 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007568625877632899, + "loss": 1.8215, + "theoretical_loss": 4.359450812019643, + "tokens_seen": 209190912 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007568465396188566, + "loss": 1.796, + "theoretical_loss": 4.359282241820761, + "tokens_seen": 209256448 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007568304914744233, + "loss": 1.6351, + "theoretical_loss": 4.359113739184238, + "tokens_seen": 209321984 + }, + { + "epoch": 0.06, + "learning_rate": 0.00075681444332999, + "loss": 1.826, + "theoretical_loss": 4.358945304061858, + "tokens_seen": 209387520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007567983951855567, + "loss": 1.7385, + "theoretical_loss": 4.35877693640545, + "tokens_seen": 209453056 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007567823470411234, + "loss": 1.858, + "theoretical_loss": 4.358608636166894, + "tokens_seen": 209518592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007567662988966901, + "loss": 1.7849, + "theoretical_loss": 4.358440403298119, + "tokens_seen": 209584128 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007567502507522569, + "loss": 1.6816, + "theoretical_loss": 4.3582722377511045, + "tokens_seen": 209649664 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.9917749166488647, + "objective/train/avg_token_score": 0.008225037716329098, + "objective/train/docs_used": 82917, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6863547563552856, + "objective/train/original_loss": 1.7006776332855225, + "objective/train/theoretical_loss": 4.358104139477878, + "objective/train/tokens_used": 230175200, + "objective/train/unlikelihood_loss": 0.016744226217269897, + "theoretical_loss": 4.358104139477878, + "tokens_seen": 209715200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007567342026078235, + "loss": 1.7456, + "theoretical_loss": 4.358104139477878, + "tokens_seen": 209715200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007567181544633903, + "loss": 1.7748, + "theoretical_loss": 4.357936108430517, + "tokens_seen": 209780736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007567021063189569, + "loss": 1.7624, + "theoretical_loss": 4.357768144561146, + "tokens_seen": 209846272 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007566860581745237, + "loss": 1.8052, + "theoretical_loss": 4.357600247821942, + "tokens_seen": 209911808 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007566700100300903, + "loss": 1.724, + "theoretical_loss": 4.3574324181651285, + "tokens_seen": 209977344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007566539618856571, + "loss": 1.7175, + "theoretical_loss": 4.357264655542978, + "tokens_seen": 210042880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007566379137412237, + "loss": 1.7056, + "theoretical_loss": 4.357096959907813, + "tokens_seen": 210108416 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007566218655967905, + "loss": 1.6765, + "theoretical_loss": 4.356929331212004, + "tokens_seen": 210173952 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007566058174523571, + "loss": 1.6844, + "theoretical_loss": 4.356761769407969, + "tokens_seen": 210239488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007565897693079239, + "loss": 1.872, + "theoretical_loss": 4.356594274448178, + "tokens_seen": 210305024 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007565737211634905, + "loss": 1.8926, + "theoretical_loss": 4.356426846285147, + "tokens_seen": 210370560 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007565576730190572, + "loss": 1.7802, + "theoretical_loss": 4.3562594848714395, + "tokens_seen": 210436096 + }, + { + "epoch": 0.06, + "learning_rate": 0.000756541624874624, + "loss": 1.8121, + "theoretical_loss": 4.3560921901596705, + "tokens_seen": 210501632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007565255767301906, + "loss": 1.8559, + "theoretical_loss": 4.355924962102502, + "tokens_seen": 210567168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007565095285857573, + "loss": 1.8336, + "theoretical_loss": 4.3557578006526425, + "tokens_seen": 210632704 + }, + { + "epoch": 0.06, + "learning_rate": 0.000756493480441324, + "loss": 1.7014, + "theoretical_loss": 4.355590705762852, + "tokens_seen": 210698240 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007564774322968907, + "loss": 1.7451, + "theoretical_loss": 4.355423677385937, + "tokens_seen": 210763776 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007564613841524574, + "loss": 1.7152, + "theoretical_loss": 4.355256715474751, + "tokens_seen": 210829312 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007564453360080242, + "loss": 1.7928, + "theoretical_loss": 4.355089819982198, + "tokens_seen": 210894848 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007564292878635908, + "loss": 1.8042, + "theoretical_loss": 4.354922990861229, + "tokens_seen": 210960384 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007564132397191574, + "loss": 1.7776, + "theoretical_loss": 4.354756228064842, + "tokens_seen": 211025920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007563971915747242, + "loss": 1.8368, + "theoretical_loss": 4.354589531546083, + "tokens_seen": 211091456 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007563811434302908, + "loss": 1.7442, + "theoretical_loss": 4.354422901258048, + "tokens_seen": 211156992 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007563650952858576, + "loss": 1.6273, + "theoretical_loss": 4.354256337153878, + "tokens_seen": 211222528 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007563490471414242, + "loss": 1.7197, + "theoretical_loss": 4.354089839186763, + "tokens_seen": 211288064 + }, + { + "epoch": 0.06, + "learning_rate": 0.000756332998996991, + "loss": 1.7478, + "theoretical_loss": 4.35392340730994, + "tokens_seen": 211353600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007563169508525576, + "loss": 1.7181, + "theoretical_loss": 4.353757041476695, + "tokens_seen": 211419136 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007563009027081244, + "loss": 1.7808, + "theoretical_loss": 4.35359074164036, + "tokens_seen": 211484672 + }, + { + "epoch": 0.06, + "learning_rate": 0.000756284854563691, + "loss": 1.741, + "theoretical_loss": 4.3534245077543146, + "tokens_seen": 211550208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007562688064192578, + "loss": 1.8229, + "theoretical_loss": 4.353258339771986, + "tokens_seen": 211615744 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007562527582748245, + "loss": 1.7026, + "theoretical_loss": 4.353092237646849, + "tokens_seen": 211681280 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007562367101303912, + "loss": 1.7302, + "theoretical_loss": 4.3529262013324255, + "tokens_seen": 211746816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007562206619859579, + "loss": 1.6862, + "theoretical_loss": 4.352760230782284, + "tokens_seen": 211812352 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007562046138415246, + "loss": 1.7168, + "theoretical_loss": 4.352594325950041, + "tokens_seen": 211877888 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007561885656970913, + "loss": 1.7483, + "theoretical_loss": 4.35242848678936, + "tokens_seen": 211943424 + }, + { + "epoch": 0.06, + "learning_rate": 0.000756172517552658, + "loss": 1.7896, + "theoretical_loss": 4.35226271325395, + "tokens_seen": 212008960 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007561564694082247, + "loss": 1.8604, + "theoretical_loss": 4.352097005297569, + "tokens_seen": 212074496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007561404212637914, + "loss": 1.6971, + "theoretical_loss": 4.351931362874019, + "tokens_seen": 212140032 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007561243731193581, + "loss": 1.7748, + "theoretical_loss": 4.351765785937154, + "tokens_seen": 212205568 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007561083249749248, + "loss": 1.8017, + "theoretical_loss": 4.3516002744408695, + "tokens_seen": 212271104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007560922768304915, + "loss": 1.7644, + "theoretical_loss": 4.351434828339109, + "tokens_seen": 212336640 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007560762286860582, + "loss": 1.8144, + "theoretical_loss": 4.351269447585864, + "tokens_seen": 212402176 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007560601805416249, + "loss": 1.8285, + "theoretical_loss": 4.3511041321351716, + "tokens_seen": 212467712 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007560441323971916, + "loss": 1.7598, + "theoretical_loss": 4.350938881941116, + "tokens_seen": 212533248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007560280842527583, + "loss": 1.7416, + "theoretical_loss": 4.350773696957827, + "tokens_seen": 212598784 + }, + { + "epoch": 0.06, + "learning_rate": 0.000756012036108325, + "loss": 1.8417, + "theoretical_loss": 4.350608577139481, + "tokens_seen": 212664320 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007559959879638917, + "loss": 1.5603, + "theoretical_loss": 4.350443522440301, + "tokens_seen": 212729856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007559799398194584, + "loss": 1.807, + "theoretical_loss": 4.350278532814556, + "tokens_seen": 212795392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007559638916750251, + "loss": 1.6939, + "theoretical_loss": 4.350113608216563, + "tokens_seen": 212860928 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007559478435305918, + "loss": 1.788, + "theoretical_loss": 4.3499487486006805, + "tokens_seen": 212926464 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.9927161335945129, + "objective/train/avg_token_score": 0.007283922750502825, + "objective/train/docs_used": 84140, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6139389276504517, + "objective/train/original_loss": 1.6242115497589111, + "objective/train/theoretical_loss": 4.349783953921318, + "objective/train/tokens_used": 233452000, + "objective/train/unlikelihood_loss": 0.01892167702317238, + "theoretical_loss": 4.349783953921318, + "tokens_seen": 212992000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007559317953861585, + "loss": 1.6306, + "theoretical_loss": 4.349783953921318, + "tokens_seen": 212992000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007559157472417252, + "loss": 1.677, + "theoretical_loss": 4.349619224132929, + "tokens_seen": 213057536 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007558996990972919, + "loss": 1.6926, + "theoretical_loss": 4.349454559190011, + "tokens_seen": 213123072 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007558836509528586, + "loss": 1.7302, + "theoretical_loss": 4.349289959047112, + "tokens_seen": 213188608 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007558676028084253, + "loss": 1.6885, + "theoretical_loss": 4.349125423658821, + "tokens_seen": 213254144 + }, + { + "epoch": 0.06, + "learning_rate": 0.000755851554663992, + "loss": 1.8117, + "theoretical_loss": 4.348960952979776, + "tokens_seen": 213319680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007558355065195587, + "loss": 1.7405, + "theoretical_loss": 4.3487965469646594, + "tokens_seen": 213385216 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007558194583751254, + "loss": 1.7578, + "theoretical_loss": 4.3486322055682, + "tokens_seen": 213450752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007558034102306921, + "loss": 1.7673, + "theoretical_loss": 4.348467928745171, + "tokens_seen": 213516288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007557873620862588, + "loss": 1.6602, + "theoretical_loss": 4.348303716450391, + "tokens_seen": 213581824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007557713139418255, + "loss": 1.8099, + "theoretical_loss": 4.348139568638727, + "tokens_seen": 213647360 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007557552657973922, + "loss": 1.7084, + "theoretical_loss": 4.347975485265087, + "tokens_seen": 213712896 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007557392176529589, + "loss": 1.753, + "theoretical_loss": 4.347811466284428, + "tokens_seen": 213778432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007557231695085256, + "loss": 1.7068, + "theoretical_loss": 4.34764751165175, + "tokens_seen": 213843968 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007557071213640923, + "loss": 1.6997, + "theoretical_loss": 4.347483621322099, + "tokens_seen": 213909504 + }, + { + "epoch": 0.06, + "learning_rate": 0.000755691073219659, + "loss": 1.7472, + "theoretical_loss": 4.347319795250567, + "tokens_seen": 213975040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007556750250752257, + "loss": 1.7262, + "theoretical_loss": 4.347156033392291, + "tokens_seen": 214040576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007556589769307924, + "loss": 1.7895, + "theoretical_loss": 4.346992335702451, + "tokens_seen": 214106112 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007556429287863591, + "loss": 1.7322, + "theoretical_loss": 4.346828702136273, + "tokens_seen": 214171648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007556268806419258, + "loss": 1.7545, + "theoretical_loss": 4.34666513264903, + "tokens_seen": 214237184 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007556108324974925, + "loss": 1.7947, + "theoretical_loss": 4.346501627196037, + "tokens_seen": 214302720 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007555947843530592, + "loss": 1.7894, + "theoretical_loss": 4.346338185732655, + "tokens_seen": 214368256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007555787362086259, + "loss": 1.7775, + "theoretical_loss": 4.3461748082142915, + "tokens_seen": 214433792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007555626880641926, + "loss": 1.8673, + "theoretical_loss": 4.346011494596395, + "tokens_seen": 214499328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007555466399197594, + "loss": 1.6456, + "theoretical_loss": 4.34584824483446, + "tokens_seen": 214564864 + }, + { + "epoch": 0.07, + "learning_rate": 0.000755530591775326, + "loss": 1.7897, + "theoretical_loss": 4.345685058884029, + "tokens_seen": 214630400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007555145436308928, + "loss": 1.6858, + "theoretical_loss": 4.345521936700684, + "tokens_seen": 214695936 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007554984954864594, + "loss": 1.7889, + "theoretical_loss": 4.345358878240053, + "tokens_seen": 214761472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007554824473420262, + "loss": 1.8857, + "theoretical_loss": 4.345195883457811, + "tokens_seen": 214827008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007554663991975928, + "loss": 1.8102, + "theoretical_loss": 4.3450329523096745, + "tokens_seen": 214892544 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007554503510531596, + "loss": 1.853, + "theoretical_loss": 4.344870084751406, + "tokens_seen": 214958080 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007554343029087262, + "loss": 1.8598, + "theoretical_loss": 4.34470728073881, + "tokens_seen": 215023616 + }, + { + "epoch": 0.07, + "learning_rate": 0.000755418254764293, + "loss": 1.7561, + "theoretical_loss": 4.344544540227737, + "tokens_seen": 215089152 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007554022066198596, + "loss": 1.7389, + "theoretical_loss": 4.344381863174082, + "tokens_seen": 215154688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007553861584754264, + "loss": 1.7767, + "theoretical_loss": 4.344219249533783, + "tokens_seen": 215220224 + }, + { + "epoch": 0.07, + "learning_rate": 0.000755370110330993, + "loss": 1.8955, + "theoretical_loss": 4.34405669926282, + "tokens_seen": 215285760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007553540621865598, + "loss": 1.7615, + "theoretical_loss": 4.343894212317224, + "tokens_seen": 215351296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007553380140421264, + "loss": 1.7592, + "theoretical_loss": 4.34373178865306, + "tokens_seen": 215416832 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007553219658976932, + "loss": 1.5976, + "theoretical_loss": 4.343569428226445, + "tokens_seen": 215482368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007553059177532598, + "loss": 1.6366, + "theoretical_loss": 4.343407130993536, + "tokens_seen": 215547904 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007552898696088266, + "loss": 1.8051, + "theoretical_loss": 4.343244896910534, + "tokens_seen": 215613440 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007552738214643932, + "loss": 1.8985, + "theoretical_loss": 4.343082725933684, + "tokens_seen": 215678976 + }, + { + "epoch": 0.07, + "learning_rate": 0.00075525777331996, + "loss": 1.8126, + "theoretical_loss": 4.342920618019275, + "tokens_seen": 215744512 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007552417251755266, + "loss": 1.6406, + "theoretical_loss": 4.342758573123639, + "tokens_seen": 215810048 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007552256770310934, + "loss": 1.8039, + "theoretical_loss": 4.342596591203153, + "tokens_seen": 215875584 + }, + { + "epoch": 0.07, + "learning_rate": 0.00075520962888666, + "loss": 1.6398, + "theoretical_loss": 4.3424346722142335, + "tokens_seen": 215941120 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007551935807422268, + "loss": 1.6926, + "theoretical_loss": 4.342272816113344, + "tokens_seen": 216006656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007551775325977934, + "loss": 1.7971, + "theoretical_loss": 4.342111022856992, + "tokens_seen": 216072192 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007551614844533601, + "loss": 1.8252, + "theoretical_loss": 4.341949292401725, + "tokens_seen": 216137728 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007551454363089268, + "loss": 1.733, + "theoretical_loss": 4.341787624704135, + "tokens_seen": 216203264 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.9911940097808838, + "objective/train/avg_token_score": 0.008805959485471249, + "objective/train/docs_used": 85191, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.790683388710022, + "objective/train/original_loss": 1.8078062534332275, + "objective/train/theoretical_loss": 4.341626019720858, + "objective/train/tokens_used": 236728800, + "objective/train/unlikelihood_loss": 0.01679060235619545, + "theoretical_loss": 4.341626019720858, + "tokens_seen": 216268800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007551293881644935, + "loss": 1.7184, + "theoretical_loss": 4.341626019720858, + "tokens_seen": 216268800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007551133400200602, + "loss": 1.7532, + "theoretical_loss": 4.341464477408572, + "tokens_seen": 216334336 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007550972918756269, + "loss": 1.7467, + "theoretical_loss": 4.341302997724, + "tokens_seen": 216399872 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007550812437311937, + "loss": 1.8881, + "theoretical_loss": 4.341141580623905, + "tokens_seen": 216465408 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007550651955867603, + "loss": 1.739, + "theoretical_loss": 4.340980226065096, + "tokens_seen": 216530944 + }, + { + "epoch": 0.07, + "learning_rate": 0.000755049147442327, + "loss": 1.8005, + "theoretical_loss": 4.340818934004421, + "tokens_seen": 216596480 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007550330992978937, + "loss": 1.6896, + "theoretical_loss": 4.340657704398774, + "tokens_seen": 216662016 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007550170511534603, + "loss": 1.8001, + "theoretical_loss": 4.340496537205093, + "tokens_seen": 216727552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007550010030090271, + "loss": 1.7232, + "theoretical_loss": 4.340335432380353, + "tokens_seen": 216793088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007549849548645937, + "loss": 1.8136, + "theoretical_loss": 4.340174389881579, + "tokens_seen": 216858624 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007549689067201605, + "loss": 1.8182, + "theoretical_loss": 4.340013409665833, + "tokens_seen": 216924160 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007549528585757271, + "loss": 1.6698, + "theoretical_loss": 4.339852491690221, + "tokens_seen": 216989696 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007549368104312939, + "loss": 1.7595, + "theoretical_loss": 4.339691635911892, + "tokens_seen": 217055232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007549207622868605, + "loss": 1.7444, + "theoretical_loss": 4.339530842288037, + "tokens_seen": 217120768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007549047141424273, + "loss": 1.7109, + "theoretical_loss": 4.339370110775892, + "tokens_seen": 217186304 + }, + { + "epoch": 0.07, + "learning_rate": 0.000754888665997994, + "loss": 1.7147, + "theoretical_loss": 4.33920944133273, + "tokens_seen": 217251840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007548726178535607, + "loss": 1.7927, + "theoretical_loss": 4.3390488339158715, + "tokens_seen": 217317376 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007548565697091274, + "loss": 1.7832, + "theoretical_loss": 4.338888288482675, + "tokens_seen": 217382912 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007548405215646941, + "loss": 1.7352, + "theoretical_loss": 4.338727804990544, + "tokens_seen": 217448448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007548244734202608, + "loss": 1.7036, + "theoretical_loss": 4.338567383396924, + "tokens_seen": 217513984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007548084252758275, + "loss": 1.7637, + "theoretical_loss": 4.338407023659302, + "tokens_seen": 217579520 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007547923771313942, + "loss": 1.5937, + "theoretical_loss": 4.338246725735205, + "tokens_seen": 217645056 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007547763289869609, + "loss": 1.7426, + "theoretical_loss": 4.338086489582205, + "tokens_seen": 217710592 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007547602808425276, + "loss": 1.6871, + "theoretical_loss": 4.3379263151579135, + "tokens_seen": 217776128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007547442326980943, + "loss": 1.6333, + "theoretical_loss": 4.337766202419986, + "tokens_seen": 217841664 + }, + { + "epoch": 0.07, + "learning_rate": 0.000754728184553661, + "loss": 1.7883, + "theoretical_loss": 4.337606151326119, + "tokens_seen": 217907200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007547121364092277, + "loss": 1.7621, + "theoretical_loss": 4.337446161834048, + "tokens_seen": 217972736 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007546960882647944, + "loss": 1.822, + "theoretical_loss": 4.3372862339015565, + "tokens_seen": 218038272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007546800401203611, + "loss": 1.6667, + "theoretical_loss": 4.337126367486462, + "tokens_seen": 218103808 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007546639919759278, + "loss": 1.7581, + "theoretical_loss": 4.33696656254663, + "tokens_seen": 218169344 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007546479438314945, + "loss": 1.7665, + "theoretical_loss": 4.336806819039962, + "tokens_seen": 218234880 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007546318956870612, + "loss": 1.5401, + "theoretical_loss": 4.336647136924407, + "tokens_seen": 218300416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007546158475426279, + "loss": 1.675, + "theoretical_loss": 4.33648751615795, + "tokens_seen": 218365952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007545997993981946, + "loss": 1.7475, + "theoretical_loss": 4.33632795669862, + "tokens_seen": 218431488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007545837512537613, + "loss": 1.7691, + "theoretical_loss": 4.336168458504486, + "tokens_seen": 218497024 + }, + { + "epoch": 0.07, + "learning_rate": 0.000754567703109328, + "loss": 1.7672, + "theoretical_loss": 4.336009021533661, + "tokens_seen": 218562560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007545516549648947, + "loss": 1.8836, + "theoretical_loss": 4.335849645744295, + "tokens_seen": 218628096 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007545356068204614, + "loss": 1.7271, + "theoretical_loss": 4.3356903310945825, + "tokens_seen": 218693632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007545195586760281, + "loss": 1.7295, + "theoretical_loss": 4.335531077542758, + "tokens_seen": 218759168 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007545035105315948, + "loss": 1.7406, + "theoretical_loss": 4.335371885047096, + "tokens_seen": 218824704 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007544874623871615, + "loss": 1.6907, + "theoretical_loss": 4.335212753565914, + "tokens_seen": 218890240 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007544714142427282, + "loss": 1.7933, + "theoretical_loss": 4.33505368305757, + "tokens_seen": 218955776 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007544553660982949, + "loss": 1.7705, + "theoretical_loss": 4.33489467348046, + "tokens_seen": 219021312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007544393179538616, + "loss": 1.6352, + "theoretical_loss": 4.334735724793026, + "tokens_seen": 219086848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007544232698094283, + "loss": 1.8139, + "theoretical_loss": 4.334576836953744, + "tokens_seen": 219152384 + }, + { + "epoch": 0.07, + "learning_rate": 0.000754407221664995, + "loss": 1.759, + "theoretical_loss": 4.334418009921139, + "tokens_seen": 219217920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007543911735205617, + "loss": 1.7173, + "theoretical_loss": 4.334259243653769, + "tokens_seen": 219283456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007543751253761284, + "loss": 1.8456, + "theoretical_loss": 4.334100538110237, + "tokens_seen": 219348992 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007543590772316951, + "loss": 1.8069, + "theoretical_loss": 4.333941893249184, + "tokens_seen": 219414528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007543430290872619, + "loss": 1.7318, + "theoretical_loss": 4.333783309029296, + "tokens_seen": 219480064 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.9948246479034424, + "objective/train/avg_token_score": 0.005175345577299595, + "objective/train/docs_used": 86440, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6789898872375488, + "objective/train/original_loss": 1.6892293691635132, + "objective/train/theoretical_loss": 4.3336247854092935, + "objective/train/tokens_used": 240005600, + "objective/train/unlikelihood_loss": 0.010693446733057499, + "theoretical_loss": 4.3336247854092935, + "tokens_seen": 219545600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007543269809428285, + "loss": 1.6797, + "theoretical_loss": 4.3336247854092935, + "tokens_seen": 219545600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007543109327983953, + "loss": 1.6723, + "theoretical_loss": 4.333466322347942, + "tokens_seen": 219611136 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007542948846539619, + "loss": 1.8259, + "theoretical_loss": 4.333307919804045, + "tokens_seen": 219676672 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007542788365095287, + "loss": 1.8594, + "theoretical_loss": 4.333149577736446, + "tokens_seen": 219742208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007542627883650953, + "loss": 1.8179, + "theoretical_loss": 4.332991296104032, + "tokens_seen": 219807744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007542467402206621, + "loss": 1.7695, + "theoretical_loss": 4.332833074865725, + "tokens_seen": 219873280 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007542306920762287, + "loss": 1.7515, + "theoretical_loss": 4.332674913980492, + "tokens_seen": 219938816 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007542146439317955, + "loss": 1.7879, + "theoretical_loss": 4.332516813407337, + "tokens_seen": 220004352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007541985957873621, + "loss": 1.6841, + "theoretical_loss": 4.332358773105307, + "tokens_seen": 220069888 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007541825476429289, + "loss": 1.7171, + "theoretical_loss": 4.332200793033485, + "tokens_seen": 220135424 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007541664994984955, + "loss": 1.6521, + "theoretical_loss": 4.332042873150997, + "tokens_seen": 220200960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007541504513540623, + "loss": 1.7784, + "theoretical_loss": 4.3318850134170095, + "tokens_seen": 220266496 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007541344032096289, + "loss": 1.7616, + "theoretical_loss": 4.331727213790725, + "tokens_seen": 220332032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007541183550651957, + "loss": 1.7967, + "theoretical_loss": 4.331569474231388, + "tokens_seen": 220397568 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007541023069207623, + "loss": 1.8003, + "theoretical_loss": 4.331411794698286, + "tokens_seen": 220463104 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007540862587763291, + "loss": 1.695, + "theoretical_loss": 4.33125417515074, + "tokens_seen": 220528640 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007540702106318957, + "loss": 1.7611, + "theoretical_loss": 4.331096615548115, + "tokens_seen": 220594176 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007540541624874625, + "loss": 1.7969, + "theoretical_loss": 4.330939115849814, + "tokens_seen": 220659712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007540381143430291, + "loss": 1.7288, + "theoretical_loss": 4.33078167601528, + "tokens_seen": 220725248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007540220661985959, + "loss": 1.8087, + "theoretical_loss": 4.330624296003996, + "tokens_seen": 220790784 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007540060180541625, + "loss": 1.7396, + "theoretical_loss": 4.330466975775483, + "tokens_seen": 220856320 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007539899699097293, + "loss": 1.8256, + "theoretical_loss": 4.330309715289301, + "tokens_seen": 220921856 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007539739217652959, + "loss": 1.7061, + "theoretical_loss": 4.330152514505053, + "tokens_seen": 220987392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007539578736208627, + "loss": 1.6286, + "theoretical_loss": 4.329995373382378, + "tokens_seen": 221052928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007539418254764293, + "loss": 1.7145, + "theoretical_loss": 4.329838291880954, + "tokens_seen": 221118464 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007539257773319961, + "loss": 1.8465, + "theoretical_loss": 4.329681269960499, + "tokens_seen": 221184000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007539097291875627, + "loss": 1.7533, + "theoretical_loss": 4.329524307580773, + "tokens_seen": 221249536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007538936810431295, + "loss": 1.5678, + "theoretical_loss": 4.3293674047015696, + "tokens_seen": 221315072 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007538776328986961, + "loss": 1.7075, + "theoretical_loss": 4.329210561282726, + "tokens_seen": 221380608 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007538615847542629, + "loss": 1.6401, + "theoretical_loss": 4.329053777284115, + "tokens_seen": 221446144 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007538455366098295, + "loss": 1.701, + "theoretical_loss": 4.328897052665653, + "tokens_seen": 221511680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007538294884653963, + "loss": 1.831, + "theoretical_loss": 4.328740387387289, + "tokens_seen": 221577216 + }, + { + "epoch": 0.07, + "learning_rate": 0.000753813440320963, + "loss": 1.6713, + "theoretical_loss": 4.328583781409016, + "tokens_seen": 221642752 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007537973921765297, + "loss": 1.7304, + "theoretical_loss": 4.328427234690864, + "tokens_seen": 221708288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007537813440320963, + "loss": 1.7916, + "theoretical_loss": 4.328270747192901, + "tokens_seen": 221773824 + }, + { + "epoch": 0.07, + "learning_rate": 0.000753765295887663, + "loss": 1.7503, + "theoretical_loss": 4.328114318875236, + "tokens_seen": 221839360 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007537492477432297, + "loss": 1.8717, + "theoretical_loss": 4.327957949698012, + "tokens_seen": 221904896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007537331995987964, + "loss": 1.7797, + "theoretical_loss": 4.327801639621417, + "tokens_seen": 221970432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007537171514543632, + "loss": 1.7827, + "theoretical_loss": 4.3276453886056725, + "tokens_seen": 222035968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007537011033099298, + "loss": 1.643, + "theoretical_loss": 4.32748919661104, + "tokens_seen": 222101504 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007536850551654966, + "loss": 1.7542, + "theoretical_loss": 4.327333063597821, + "tokens_seen": 222167040 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007536690070210632, + "loss": 1.6897, + "theoretical_loss": 4.327176989526352, + "tokens_seen": 222232576 + }, + { + "epoch": 0.07, + "learning_rate": 0.00075365295887663, + "loss": 1.7106, + "theoretical_loss": 4.327020974357012, + "tokens_seen": 222298112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007536369107321966, + "loss": 1.7792, + "theoretical_loss": 4.326865018050215, + "tokens_seen": 222363648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007536208625877632, + "loss": 1.6784, + "theoretical_loss": 4.326709120566415, + "tokens_seen": 222429184 + }, + { + "epoch": 0.07, + "learning_rate": 0.00075360481444333, + "loss": 1.6871, + "theoretical_loss": 4.326553281866103, + "tokens_seen": 222494720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007535887662988966, + "loss": 1.7076, + "theoretical_loss": 4.326397501909811, + "tokens_seen": 222560256 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007535727181544634, + "loss": 1.6428, + "theoretical_loss": 4.326241780658104, + "tokens_seen": 222625792 + }, + { + "epoch": 0.07, + "learning_rate": 0.00075355667001003, + "loss": 1.7427, + "theoretical_loss": 4.32608611807159, + "tokens_seen": 222691328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007535406218655968, + "loss": 1.6274, + "theoretical_loss": 4.325930514110912, + "tokens_seen": 222756864 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.9933840036392212, + "objective/train/avg_token_score": 0.006615967024117708, + "objective/train/docs_used": 87601, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5695569515228271, + "objective/train/original_loss": 1.5787562131881714, + "objective/train/theoretical_loss": 4.325774968736753, + "objective/train/tokens_used": 243282400, + "objective/train/unlikelihood_loss": 0.019730329513549805, + "theoretical_loss": 4.325774968736753, + "tokens_seen": 222822400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007535245737211634, + "loss": 1.6113, + "theoretical_loss": 4.325774968736753, + "tokens_seen": 222822400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007535085255767302, + "loss": 1.8128, + "theoretical_loss": 4.325619481909833, + "tokens_seen": 222887936 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007534924774322969, + "loss": 1.6172, + "theoretical_loss": 4.325464053590908, + "tokens_seen": 222953472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007534764292878636, + "loss": 1.7371, + "theoretical_loss": 4.325308683740776, + "tokens_seen": 223019008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007534603811434303, + "loss": 1.7735, + "theoretical_loss": 4.325153372320269, + "tokens_seen": 223084544 + }, + { + "epoch": 0.07, + "learning_rate": 0.000753444332998997, + "loss": 1.7465, + "theoretical_loss": 4.32499811929026, + "tokens_seen": 223150080 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007534282848545637, + "loss": 1.7897, + "theoretical_loss": 4.324842924611655, + "tokens_seen": 223215616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007534122367101304, + "loss": 1.7394, + "theoretical_loss": 4.324687788245403, + "tokens_seen": 223281152 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007533961885656971, + "loss": 1.7238, + "theoretical_loss": 4.3245327101524875, + "tokens_seen": 223346688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007533801404212638, + "loss": 1.7391, + "theoretical_loss": 4.324377690293929, + "tokens_seen": 223412224 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007533640922768305, + "loss": 1.7606, + "theoretical_loss": 4.324222728630787, + "tokens_seen": 223477760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007533480441323972, + "loss": 1.7484, + "theoretical_loss": 4.324067825124159, + "tokens_seen": 223543296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007533319959879639, + "loss": 1.7011, + "theoretical_loss": 4.323912979735178, + "tokens_seen": 223608832 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007533159478435306, + "loss": 1.8036, + "theoretical_loss": 4.323758192425016, + "tokens_seen": 223674368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007532998996990973, + "loss": 1.7024, + "theoretical_loss": 4.323603463154882, + "tokens_seen": 223739904 + }, + { + "epoch": 0.07, + "learning_rate": 0.000753283851554664, + "loss": 1.7483, + "theoretical_loss": 4.323448791886022, + "tokens_seen": 223805440 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007532678034102307, + "loss": 1.8757, + "theoretical_loss": 4.323294178579717, + "tokens_seen": 223870976 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007532517552657974, + "loss": 1.6807, + "theoretical_loss": 4.323139623197289, + "tokens_seen": 223936512 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007532357071213641, + "loss": 1.7978, + "theoretical_loss": 4.322985125700096, + "tokens_seen": 224002048 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007532196589769308, + "loss": 1.6731, + "theoretical_loss": 4.322830686049532, + "tokens_seen": 224067584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007532036108324975, + "loss": 1.8172, + "theoretical_loss": 4.322676304207027, + "tokens_seen": 224133120 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007531875626880642, + "loss": 1.7876, + "theoretical_loss": 4.32252198013405, + "tokens_seen": 224198656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007531715145436309, + "loss": 1.716, + "theoretical_loss": 4.32236771379211, + "tokens_seen": 224264192 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007531554663991976, + "loss": 1.726, + "theoretical_loss": 4.322213505142744, + "tokens_seen": 224329728 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007531394182547644, + "loss": 1.7717, + "theoretical_loss": 4.322059354147535, + "tokens_seen": 224395264 + }, + { + "epoch": 0.07, + "learning_rate": 0.000753123370110331, + "loss": 1.5834, + "theoretical_loss": 4.321905260768097, + "tokens_seen": 224460800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007531073219658978, + "loss": 1.7279, + "theoretical_loss": 4.321751224966083, + "tokens_seen": 224526336 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007530912738214644, + "loss": 1.6361, + "theoretical_loss": 4.321597246703183, + "tokens_seen": 224591872 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007530752256770312, + "loss": 1.7756, + "theoretical_loss": 4.321443325941123, + "tokens_seen": 224657408 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007530591775325978, + "loss": 1.7239, + "theoretical_loss": 4.3212894626416665, + "tokens_seen": 224722944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007530431293881646, + "loss": 1.644, + "theoretical_loss": 4.321135656766611, + "tokens_seen": 224788480 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007530270812437312, + "loss": 1.7055, + "theoretical_loss": 4.320981908277793, + "tokens_seen": 224854016 + }, + { + "epoch": 0.07, + "learning_rate": 0.000753011033099298, + "loss": 1.8167, + "theoretical_loss": 4.320828217137086, + "tokens_seen": 224919552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007529949849548646, + "loss": 1.8082, + "theoretical_loss": 4.320674583306396, + "tokens_seen": 224985088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007529789368104314, + "loss": 1.8435, + "theoretical_loss": 4.32052100674767, + "tokens_seen": 225050624 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752962888665998, + "loss": 1.8954, + "theoretical_loss": 4.320367487422891, + "tokens_seen": 225116160 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007529468405215648, + "loss": 1.7751, + "theoretical_loss": 4.3202140252940735, + "tokens_seen": 225181696 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007529307923771314, + "loss": 1.7178, + "theoretical_loss": 4.3200606203232725, + "tokens_seen": 225247232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007529147442326982, + "loss": 1.7585, + "theoretical_loss": 4.319907272472579, + "tokens_seen": 225312768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007528986960882648, + "loss": 1.6418, + "theoretical_loss": 4.319753981704119, + "tokens_seen": 225378304 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007528826479438316, + "loss": 1.7013, + "theoretical_loss": 4.319600747980056, + "tokens_seen": 225443840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007528665997993982, + "loss": 1.7337, + "theoretical_loss": 4.319447571262586, + "tokens_seen": 225509376 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752850551654965, + "loss": 1.8979, + "theoretical_loss": 4.3192944515139455, + "tokens_seen": 225574912 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007528345035105316, + "loss": 1.8278, + "theoretical_loss": 4.319141388696404, + "tokens_seen": 225640448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007528184553660984, + "loss": 1.6762, + "theoretical_loss": 4.318988382772269, + "tokens_seen": 225705984 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752802407221665, + "loss": 1.7764, + "theoretical_loss": 4.318835433703883, + "tokens_seen": 225771520 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007527863590772318, + "loss": 1.8413, + "theoretical_loss": 4.318682541453622, + "tokens_seen": 225837056 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007527703109327984, + "loss": 1.6459, + "theoretical_loss": 4.318529705983903, + "tokens_seen": 225902592 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007527542627883652, + "loss": 1.7662, + "theoretical_loss": 4.318376927257174, + "tokens_seen": 225968128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007527382146439318, + "loss": 1.7147, + "theoretical_loss": 4.3182242052359205, + "tokens_seen": 226033664 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.9942114949226379, + "objective/train/avg_token_score": 0.005788503214716911, + "objective/train/docs_used": 88830, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.7128173112869263, + "objective/train/original_loss": 1.724166750907898, + "objective/train/theoretical_loss": 4.318071539882665, + "objective/train/tokens_used": 246559200, + "objective/train/unlikelihood_loss": 0.009751290082931519, + "theoretical_loss": 4.318071539882665, + "tokens_seen": 226099200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007527221664994986, + "loss": 1.7026, + "theoretical_loss": 4.318071539882665, + "tokens_seen": 226099200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007527061183550652, + "loss": 1.8389, + "theoretical_loss": 4.317918931159963, + "tokens_seen": 226164736 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752690070210632, + "loss": 1.6389, + "theoretical_loss": 4.317766379030407, + "tokens_seen": 226230272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007526740220661986, + "loss": 1.7262, + "theoretical_loss": 4.317613883456625, + "tokens_seen": 226295808 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007526579739217654, + "loss": 1.6699, + "theoretical_loss": 4.317461444401281, + "tokens_seen": 226361344 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752641925777332, + "loss": 1.7639, + "theoretical_loss": 4.317309061827073, + "tokens_seen": 226426880 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007526258776328988, + "loss": 1.6108, + "theoretical_loss": 4.317156735696736, + "tokens_seen": 226492416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007526098294884654, + "loss": 1.6102, + "theoretical_loss": 4.3170044659730396, + "tokens_seen": 226557952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007525937813440322, + "loss": 1.6968, + "theoretical_loss": 4.316852252618788, + "tokens_seen": 226623488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007525777331995988, + "loss": 1.7735, + "theoretical_loss": 4.316700095596822, + "tokens_seen": 226689024 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007525616850551656, + "loss": 1.7035, + "theoretical_loss": 4.3165479948700165, + "tokens_seen": 226754560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007525456369107322, + "loss": 1.6332, + "theoretical_loss": 4.316395950401283, + "tokens_seen": 226820096 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752529588766299, + "loss": 1.7182, + "theoretical_loss": 4.316243962153566, + "tokens_seen": 226885632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007525135406218656, + "loss": 1.7007, + "theoretical_loss": 4.316092030089847, + "tokens_seen": 226951168 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007524974924774324, + "loss": 1.7474, + "theoretical_loss": 4.315940154173143, + "tokens_seen": 227016704 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752481444332999, + "loss": 1.6837, + "theoretical_loss": 4.315788334366504, + "tokens_seen": 227082240 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007524653961885658, + "loss": 1.6621, + "theoretical_loss": 4.315636570633015, + "tokens_seen": 227147776 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007524493480441324, + "loss": 1.6811, + "theoretical_loss": 4.315484862935799, + "tokens_seen": 227213312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007524332998996992, + "loss": 1.7849, + "theoretical_loss": 4.315333211238009, + "tokens_seen": 227278848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007524172517552658, + "loss": 1.6203, + "theoretical_loss": 4.315181615502838, + "tokens_seen": 227344384 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007524012036108326, + "loss": 1.6514, + "theoretical_loss": 4.31503007569351, + "tokens_seen": 227409920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007523851554663992, + "loss": 1.7391, + "theoretical_loss": 4.314878591773287, + "tokens_seen": 227475456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007523691073219659, + "loss": 1.7436, + "theoretical_loss": 4.31472716370546, + "tokens_seen": 227540992 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007523530591775327, + "loss": 1.6263, + "theoretical_loss": 4.314575791453362, + "tokens_seen": 227606528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007523370110330993, + "loss": 1.6902, + "theoretical_loss": 4.314424474980355, + "tokens_seen": 227672064 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752320962888666, + "loss": 1.7012, + "theoretical_loss": 4.31427321424984, + "tokens_seen": 227737600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007523049147442327, + "loss": 1.759, + "theoretical_loss": 4.314122009225248, + "tokens_seen": 227803136 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007522888665997995, + "loss": 1.7436, + "theoretical_loss": 4.313970859870048, + "tokens_seen": 227868672 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007522728184553661, + "loss": 1.6735, + "theoretical_loss": 4.313819766147741, + "tokens_seen": 227934208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007522567703109329, + "loss": 1.7425, + "theoretical_loss": 4.313668728021866, + "tokens_seen": 227999744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007522407221664995, + "loss": 1.8713, + "theoretical_loss": 4.313517745455991, + "tokens_seen": 228065280 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007522246740220661, + "loss": 1.6687, + "theoretical_loss": 4.313366818413724, + "tokens_seen": 228130816 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007522086258776329, + "loss": 1.6733, + "theoretical_loss": 4.313215946858702, + "tokens_seen": 228196352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007521925777331995, + "loss": 1.7776, + "theoretical_loss": 4.313065130754601, + "tokens_seen": 228261888 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007521765295887663, + "loss": 1.7892, + "theoretical_loss": 4.312914370065128, + "tokens_seen": 228327424 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752160481444333, + "loss": 1.7546, + "theoretical_loss": 4.312763664754025, + "tokens_seen": 228392960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007521444332998997, + "loss": 1.7166, + "theoretical_loss": 4.312613014785069, + "tokens_seen": 228458496 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007521283851554664, + "loss": 1.787, + "theoretical_loss": 4.3124624201220705, + "tokens_seen": 228524032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007521123370110331, + "loss": 1.7368, + "theoretical_loss": 4.312311880728873, + "tokens_seen": 228589568 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007520962888665998, + "loss": 1.756, + "theoretical_loss": 4.312161396569356, + "tokens_seen": 228655104 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007520802407221665, + "loss": 1.7602, + "theoretical_loss": 4.312010967607431, + "tokens_seen": 228720640 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007520641925777332, + "loss": 1.7707, + "theoretical_loss": 4.311860593807045, + "tokens_seen": 228786176 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007520481444332999, + "loss": 1.7336, + "theoretical_loss": 4.311710275132178, + "tokens_seen": 228851712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007520320962888666, + "loss": 1.7399, + "theoretical_loss": 4.3115600115468435, + "tokens_seen": 228917248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007520160481444333, + "loss": 1.6965, + "theoretical_loss": 4.311409803015089, + "tokens_seen": 228982784 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752, + "loss": 1.8458, + "theoretical_loss": 4.3112596495009985, + "tokens_seen": 229048320 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007519839518555667, + "loss": 1.7267, + "theoretical_loss": 4.311109550968684, + "tokens_seen": 229113856 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007519679037111334, + "loss": 1.7882, + "theoretical_loss": 4.310959507382296, + "tokens_seen": 229179392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007519518555667001, + "loss": 1.7128, + "theoretical_loss": 4.310809518706017, + "tokens_seen": 229244928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007519358074222669, + "loss": 1.757, + "theoretical_loss": 4.310659584904062, + "tokens_seen": 229310464 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.9928604364395142, + "objective/train/avg_token_score": 0.007139581721276045, + "objective/train/docs_used": 90061, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.85369074344635, + "objective/train/original_loss": 1.8677916526794434, + "objective/train/theoretical_loss": 4.310509705940682, + "objective/train/tokens_used": 249836000, + "objective/train/unlikelihood_loss": 0.013347210362553596, + "theoretical_loss": 4.310509705940682, + "tokens_seen": 229376000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007519197592778335, + "loss": 1.7877, + "theoretical_loss": 4.310509705940682, + "tokens_seen": 229376000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007519037111334003, + "loss": 1.6793, + "theoretical_loss": 4.310359881780159, + "tokens_seen": 229441536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007518876629889669, + "loss": 1.8271, + "theoretical_loss": 4.310210112386811, + "tokens_seen": 229507072 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007518716148445337, + "loss": 1.8358, + "theoretical_loss": 4.310060397724987, + "tokens_seen": 229572608 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007518555667001003, + "loss": 1.7794, + "theoretical_loss": 4.30991073775907, + "tokens_seen": 229638144 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007518395185556671, + "loss": 1.6946, + "theoretical_loss": 4.3097611324534775, + "tokens_seen": 229703680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007518234704112337, + "loss": 1.8004, + "theoretical_loss": 4.309611581772659, + "tokens_seen": 229769216 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007518074222668005, + "loss": 1.7677, + "theoretical_loss": 4.309462085681098, + "tokens_seen": 229834752 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007517913741223671, + "loss": 1.8432, + "theoretical_loss": 4.3093126441433105, + "tokens_seen": 229900288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007517753259779339, + "loss": 1.7459, + "theoretical_loss": 4.3091632571238465, + "tokens_seen": 229965824 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007517592778335005, + "loss": 1.7809, + "theoretical_loss": 4.30901392458729, + "tokens_seen": 230031360 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007517432296890673, + "loss": 1.7355, + "theoretical_loss": 4.308864646498254, + "tokens_seen": 230096896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007517271815446339, + "loss": 1.6792, + "theoretical_loss": 4.30871542282139, + "tokens_seen": 230162432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007517111334002007, + "loss": 1.5761, + "theoretical_loss": 4.308566253521378, + "tokens_seen": 230227968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007516950852557673, + "loss": 1.7085, + "theoretical_loss": 4.308417138562934, + "tokens_seen": 230293504 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007516790371113341, + "loss": 1.6167, + "theoretical_loss": 4.308268077910806, + "tokens_seen": 230359040 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007516629889669007, + "loss": 1.5833, + "theoretical_loss": 4.308119071529774, + "tokens_seen": 230424576 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007516469408224675, + "loss": 1.652, + "theoretical_loss": 4.3079701193846525, + "tokens_seen": 230490112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007516308926780341, + "loss": 1.9099, + "theoretical_loss": 4.307821221440286, + "tokens_seen": 230555648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007516148445336009, + "loss": 1.6988, + "theoretical_loss": 4.3076723776615555, + "tokens_seen": 230621184 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007515987963891675, + "loss": 1.7588, + "theoretical_loss": 4.307523588013371, + "tokens_seen": 230686720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007515827482447343, + "loss": 1.6925, + "theoretical_loss": 4.307374852460679, + "tokens_seen": 230752256 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007515667001003009, + "loss": 1.6605, + "theoretical_loss": 4.307226170968456, + "tokens_seen": 230817792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007515506519558677, + "loss": 1.6296, + "theoretical_loss": 4.307077543501713, + "tokens_seen": 230883328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007515346038114343, + "loss": 1.7545, + "theoretical_loss": 4.3069289700254885, + "tokens_seen": 230948864 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007515185556670011, + "loss": 1.6597, + "theoretical_loss": 4.306780450504861, + "tokens_seen": 231014400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007515025075225677, + "loss": 1.7247, + "theoretical_loss": 4.3066319849049375, + "tokens_seen": 231079936 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007514864593781345, + "loss": 1.8286, + "theoretical_loss": 4.306483573190857, + "tokens_seen": 231145472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007514704112337011, + "loss": 1.7296, + "theoretical_loss": 4.306335215327791, + "tokens_seen": 231211008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007514543630892679, + "loss": 1.7409, + "theoretical_loss": 4.306186911280945, + "tokens_seen": 231276544 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007514383149448345, + "loss": 1.6059, + "theoretical_loss": 4.306038661015557, + "tokens_seen": 231342080 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007514222668004013, + "loss": 1.6326, + "theoretical_loss": 4.305890464496895, + "tokens_seen": 231407616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007514062186559679, + "loss": 1.8804, + "theoretical_loss": 4.305742321690261, + "tokens_seen": 231473152 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007513901705115347, + "loss": 1.708, + "theoretical_loss": 4.305594232560988, + "tokens_seen": 231538688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007513741223671013, + "loss": 1.8354, + "theoretical_loss": 4.305446197074443, + "tokens_seen": 231604224 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007513580742226681, + "loss": 1.6706, + "theoretical_loss": 4.305298215196023, + "tokens_seen": 231669760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007513420260782347, + "loss": 1.7417, + "theoretical_loss": 4.305150286891159, + "tokens_seen": 231735296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007513259779338015, + "loss": 1.8655, + "theoretical_loss": 4.3050024121253125, + "tokens_seen": 231800832 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007513099297893681, + "loss": 1.6181, + "theoretical_loss": 4.304854590863979, + "tokens_seen": 231866368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007512938816449349, + "loss": 1.6356, + "theoretical_loss": 4.304706823072683, + "tokens_seen": 231931904 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007512778335005015, + "loss": 1.8619, + "theoretical_loss": 4.304559108716984, + "tokens_seen": 231997440 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007512617853560683, + "loss": 1.8153, + "theoretical_loss": 4.304411447762472, + "tokens_seen": 232062976 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007512457372116349, + "loss": 1.7185, + "theoretical_loss": 4.304263840174768, + "tokens_seen": 232128512 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007512296890672017, + "loss": 1.5679, + "theoretical_loss": 4.304116285919527, + "tokens_seen": 232194048 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007512136409227683, + "loss": 1.7335, + "theoretical_loss": 4.303968784962433, + "tokens_seen": 232259584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007511975927783351, + "loss": 1.6944, + "theoretical_loss": 4.303821337269205, + "tokens_seen": 232325120 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007511815446339017, + "loss": 1.7505, + "theoretical_loss": 4.303673942805591, + "tokens_seen": 232390656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007511654964894685, + "loss": 1.7191, + "theoretical_loss": 4.303526601537371, + "tokens_seen": 232456192 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007511494483450351, + "loss": 1.8195, + "theoretical_loss": 4.303379313430359, + "tokens_seen": 232521728 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007511334002006019, + "loss": 1.6355, + "theoretical_loss": 4.303232078450399, + "tokens_seen": 232587264 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.9940062761306763, + "objective/train/avg_token_score": 0.00599373085424304, + "objective/train/docs_used": 91258, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6383087635040283, + "objective/train/original_loss": 1.650618076324463, + "objective/train/theoretical_loss": 4.303084896563364, + "objective/train/tokens_used": 253112800, + "objective/train/unlikelihood_loss": 0.0103054940700531, + "theoretical_loss": 4.303084896563364, + "tokens_seen": 232652800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007511173520561685, + "loss": 1.7303, + "theoretical_loss": 4.303084896563364, + "tokens_seen": 232652800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007511013039117353, + "loss": 1.6137, + "theoretical_loss": 4.302937767735164, + "tokens_seen": 232718336 + }, + { + "epoch": 0.07, + "learning_rate": 0.000751085255767302, + "loss": 1.6888, + "theoretical_loss": 4.302790691931736, + "tokens_seen": 232783872 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007510692076228687, + "loss": 1.72, + "theoretical_loss": 4.30264366911905, + "tokens_seen": 232849408 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007510531594784353, + "loss": 1.7627, + "theoretical_loss": 4.3024966992631075, + "tokens_seen": 232914944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007510371113340021, + "loss": 1.6882, + "theoretical_loss": 4.302349782329942, + "tokens_seen": 232980480 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007510210631895687, + "loss": 1.7402, + "theoretical_loss": 4.302202918285616, + "tokens_seen": 233046016 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007510050150451355, + "loss": 1.7583, + "theoretical_loss": 4.302056107096226, + "tokens_seen": 233111552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007509889669007021, + "loss": 1.6443, + "theoretical_loss": 4.301909348727899, + "tokens_seen": 233177088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007509729187562688, + "loss": 1.6921, + "theoretical_loss": 4.301762643146792, + "tokens_seen": 233242624 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007509568706118356, + "loss": 1.8017, + "theoretical_loss": 4.301615990319094, + "tokens_seen": 233308160 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007509408224674022, + "loss": 1.7146, + "theoretical_loss": 4.301469390211025, + "tokens_seen": 233373696 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750924774322969, + "loss": 1.8349, + "theoretical_loss": 4.301322842788839, + "tokens_seen": 233439232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007509087261785356, + "loss": 1.6542, + "theoretical_loss": 4.301176348018814, + "tokens_seen": 233504768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007508926780341024, + "loss": 1.6747, + "theoretical_loss": 4.301029905867265, + "tokens_seen": 233570304 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750876629889669, + "loss": 1.6044, + "theoretical_loss": 4.300883516300539, + "tokens_seen": 233635840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007508605817452358, + "loss": 1.6492, + "theoretical_loss": 4.300737179285008, + "tokens_seen": 233701376 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007508445336008024, + "loss": 1.6872, + "theoretical_loss": 4.30059089478708, + "tokens_seen": 233766912 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750828485456369, + "loss": 1.7389, + "theoretical_loss": 4.300444662773192, + "tokens_seen": 233832448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007508124373119358, + "loss": 1.6557, + "theoretical_loss": 4.30029848320981, + "tokens_seen": 233897984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007507963891675024, + "loss": 1.7728, + "theoretical_loss": 4.300152356063435, + "tokens_seen": 233963520 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007507803410230692, + "loss": 1.7809, + "theoretical_loss": 4.300006281300598, + "tokens_seen": 234029056 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007507642928786359, + "loss": 1.7065, + "theoretical_loss": 4.299860258887856, + "tokens_seen": 234094592 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007507482447342026, + "loss": 1.8622, + "theoretical_loss": 4.299714288791802, + "tokens_seen": 234160128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007507321965897694, + "loss": 1.5631, + "theoretical_loss": 4.299568370979056, + "tokens_seen": 234225664 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750716148445336, + "loss": 1.7869, + "theoretical_loss": 4.299422505416272, + "tokens_seen": 234291200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007507001003009028, + "loss": 1.7633, + "theoretical_loss": 4.2992766920701335, + "tokens_seen": 234356736 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007506840521564694, + "loss": 1.7351, + "theoretical_loss": 4.299130930907352, + "tokens_seen": 234422272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007506680040120362, + "loss": 1.7236, + "theoretical_loss": 4.2989852218946725, + "tokens_seen": 234487808 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007506519558676028, + "loss": 1.7454, + "theoretical_loss": 4.298839564998869, + "tokens_seen": 234553344 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007506359077231696, + "loss": 1.8414, + "theoretical_loss": 4.298693960186747, + "tokens_seen": 234618880 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007506198595787362, + "loss": 1.8043, + "theoretical_loss": 4.298548407425141, + "tokens_seen": 234684416 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750603811434303, + "loss": 1.6453, + "theoretical_loss": 4.298402906680918, + "tokens_seen": 234749952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007505877632898696, + "loss": 1.7229, + "theoretical_loss": 4.298257457920973, + "tokens_seen": 234815488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007505717151454364, + "loss": 1.7428, + "theoretical_loss": 4.298112061112232, + "tokens_seen": 234881024 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750555667001003, + "loss": 1.795, + "theoretical_loss": 4.297966716221653, + "tokens_seen": 234946560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007505396188565698, + "loss": 1.7629, + "theoretical_loss": 4.297821423216222, + "tokens_seen": 235012096 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007505235707121364, + "loss": 1.7119, + "theoretical_loss": 4.297676182062955, + "tokens_seen": 235077632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007505075225677032, + "loss": 1.7148, + "theoretical_loss": 4.297530992728901, + "tokens_seen": 235143168 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007504914744232698, + "loss": 1.727, + "theoretical_loss": 4.297385855181136, + "tokens_seen": 235208704 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007504754262788366, + "loss": 1.7116, + "theoretical_loss": 4.297240769386768, + "tokens_seen": 235274240 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007504593781344032, + "loss": 1.7696, + "theoretical_loss": 4.297095735312934, + "tokens_seen": 235339776 + }, + { + "epoch": 0.07, + "learning_rate": 0.00075044332998997, + "loss": 1.6578, + "theoretical_loss": 4.296950752926801, + "tokens_seen": 235405312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007504272818455366, + "loss": 1.6748, + "theoretical_loss": 4.296805822195568, + "tokens_seen": 235470848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007504112337011034, + "loss": 1.711, + "theoretical_loss": 4.29666094308646, + "tokens_seen": 235536384 + }, + { + "epoch": 0.07, + "learning_rate": 0.00075039518555667, + "loss": 1.7409, + "theoretical_loss": 4.296516115566736, + "tokens_seen": 235601920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007503791374122368, + "loss": 1.841, + "theoretical_loss": 4.296371339603684, + "tokens_seen": 235667456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007503630892678034, + "loss": 1.674, + "theoretical_loss": 4.296226615164618, + "tokens_seen": 235732992 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007503470411233702, + "loss": 1.7563, + "theoretical_loss": 4.296081942216887, + "tokens_seen": 235798528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007503309929789368, + "loss": 1.6316, + "theoretical_loss": 4.295937320727866, + "tokens_seen": 235864064 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.9898257851600647, + "objective/train/avg_token_score": 0.010174235329031944, + "objective/train/docs_used": 92471, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.8199104070663452, + "objective/train/original_loss": 1.8397592306137085, + "objective/train/theoretical_loss": 4.295792750664964, + "objective/train/tokens_used": 256389600, + "objective/train/unlikelihood_loss": 0.017965320497751236, + "theoretical_loss": 4.295792750664964, + "tokens_seen": 235929600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007503149448345036, + "loss": 1.755, + "theoretical_loss": 4.295792750664964, + "tokens_seen": 235929600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007502988966900702, + "loss": 1.738, + "theoretical_loss": 4.295648231995613, + "tokens_seen": 235995136 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750282848545637, + "loss": 1.8396, + "theoretical_loss": 4.295503764687281, + "tokens_seen": 236060672 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007502668004012036, + "loss": 1.6633, + "theoretical_loss": 4.295359348707462, + "tokens_seen": 236126208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007502507522567704, + "loss": 1.6608, + "theoretical_loss": 4.2952149840236835, + "tokens_seen": 236191744 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750234704112337, + "loss": 1.7722, + "theoretical_loss": 4.295070670603496, + "tokens_seen": 236257280 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007502186559679038, + "loss": 1.6482, + "theoretical_loss": 4.294926408414486, + "tokens_seen": 236322816 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007502026078234704, + "loss": 1.6873, + "theoretical_loss": 4.294782197424265, + "tokens_seen": 236388352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007501865596790372, + "loss": 1.696, + "theoretical_loss": 4.294638037600478, + "tokens_seen": 236453888 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007501705115346038, + "loss": 1.7904, + "theoretical_loss": 4.2944939289107955, + "tokens_seen": 236519424 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007501544633901706, + "loss": 1.7962, + "theoretical_loss": 4.29434987132292, + "tokens_seen": 236584960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007501384152457372, + "loss": 1.8945, + "theoretical_loss": 4.294205864804581, + "tokens_seen": 236650496 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750122367101304, + "loss": 1.6622, + "theoretical_loss": 4.294061909323539, + "tokens_seen": 236716032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007501063189568706, + "loss": 1.7588, + "theoretical_loss": 4.293918004847585, + "tokens_seen": 236781568 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007500902708124374, + "loss": 1.7233, + "theoretical_loss": 4.293774151344537, + "tokens_seen": 236847104 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750074222668004, + "loss": 1.8122, + "theoretical_loss": 4.2936303487822425, + "tokens_seen": 236912640 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007500581745235708, + "loss": 1.7105, + "theoretical_loss": 4.293486597128578, + "tokens_seen": 236978176 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007500421263791374, + "loss": 1.6604, + "theoretical_loss": 4.293342896351452, + "tokens_seen": 237043712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007500260782347042, + "loss": 1.6901, + "theoretical_loss": 4.293199246418796, + "tokens_seen": 237109248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007500100300902708, + "loss": 1.7044, + "theoretical_loss": 4.293055647298578, + "tokens_seen": 237174784 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007499939819458376, + "loss": 1.7679, + "theoretical_loss": 4.292912098958791, + "tokens_seen": 237240320 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007499779338014042, + "loss": 1.6798, + "theoretical_loss": 4.292768601367455, + "tokens_seen": 237305856 + }, + { + "epoch": 0.07, + "learning_rate": 0.000749961885656971, + "loss": 1.6351, + "theoretical_loss": 4.292625154492623, + "tokens_seen": 237371392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007499458375125376, + "loss": 1.65, + "theoretical_loss": 4.292481758302375, + "tokens_seen": 237436928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007499297893681044, + "loss": 1.6347, + "theoretical_loss": 4.2923384127648205, + "tokens_seen": 237502464 + }, + { + "epoch": 0.07, + "learning_rate": 0.000749913741223671, + "loss": 1.6105, + "theoretical_loss": 4.292195117848098, + "tokens_seen": 237568000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007498976930792378, + "loss": 1.6757, + "theoretical_loss": 4.292051873520373, + "tokens_seen": 237633536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007498816449348044, + "loss": 1.6893, + "theoretical_loss": 4.291908679749842, + "tokens_seen": 237699072 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007498655967903712, + "loss": 1.8486, + "theoretical_loss": 4.2917655365047285, + "tokens_seen": 237764608 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007498495486459378, + "loss": 1.6268, + "theoretical_loss": 4.291622443753287, + "tokens_seen": 237830144 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007498335005015046, + "loss": 1.7773, + "theoretical_loss": 4.291479401463799, + "tokens_seen": 237895680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007498174523570712, + "loss": 1.7077, + "theoretical_loss": 4.291336409604574, + "tokens_seen": 237961216 + }, + { + "epoch": 0.07, + "learning_rate": 0.000749801404212638, + "loss": 1.8134, + "theoretical_loss": 4.291193468143952, + "tokens_seen": 238026752 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007497853560682046, + "loss": 1.768, + "theoretical_loss": 4.2910505770503, + "tokens_seen": 238092288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007497693079237714, + "loss": 1.6878, + "theoretical_loss": 4.290907736292016, + "tokens_seen": 238157824 + }, + { + "epoch": 0.07, + "learning_rate": 0.000749753259779338, + "loss": 1.6542, + "theoretical_loss": 4.290764945837522, + "tokens_seen": 238223360 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007497372116349048, + "loss": 1.7403, + "theoretical_loss": 4.290622205655273, + "tokens_seen": 238288896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007497211634904714, + "loss": 1.6285, + "theoretical_loss": 4.290479515713751, + "tokens_seen": 238354432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007497051153460382, + "loss": 1.6441, + "theoretical_loss": 4.290336875981465, + "tokens_seen": 238419968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007496890672016048, + "loss": 1.6407, + "theoretical_loss": 4.290194286426955, + "tokens_seen": 238485504 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007496730190571716, + "loss": 1.7978, + "theoretical_loss": 4.290051747018785, + "tokens_seen": 238551040 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007496569709127382, + "loss": 1.8148, + "theoretical_loss": 4.289909257725553, + "tokens_seen": 238616576 + }, + { + "epoch": 0.07, + "learning_rate": 0.000749640922768305, + "loss": 1.7288, + "theoretical_loss": 4.289766818515882, + "tokens_seen": 238682112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007496248746238716, + "loss": 1.838, + "theoretical_loss": 4.2896244293584225, + "tokens_seen": 238747648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007496088264794384, + "loss": 1.7274, + "theoretical_loss": 4.2894820902218544, + "tokens_seen": 238813184 + }, + { + "epoch": 0.07, + "learning_rate": 0.000749592778335005, + "loss": 1.6755, + "theoretical_loss": 4.289339801074888, + "tokens_seen": 238878720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007495767301905718, + "loss": 1.6731, + "theoretical_loss": 4.289197561886258, + "tokens_seen": 238944256 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007495606820461385, + "loss": 1.8164, + "theoretical_loss": 4.289055372624729, + "tokens_seen": 239009792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007495446339017051, + "loss": 1.7606, + "theoretical_loss": 4.288913233259092, + "tokens_seen": 239075328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007495285857572719, + "loss": 1.709, + "theoretical_loss": 4.28877114375817, + "tokens_seen": 239140864 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.9938614964485168, + "objective/train/avg_token_score": 0.0061384630389511585, + "objective/train/docs_used": 93570, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.8018792867660522, + "objective/train/original_loss": 1.813736915588379, + "objective/train/theoretical_loss": 4.288629104090811, + "objective/train/tokens_used": 259666400, + "objective/train/unlikelihood_loss": 0.01245147455483675, + "theoretical_loss": 4.288629104090811, + "tokens_seen": 239206400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007495125376128385, + "loss": 1.7767, + "theoretical_loss": 4.288629104090811, + "tokens_seen": 239206400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007494964894684053, + "loss": 1.7014, + "theoretical_loss": 4.28848711422589, + "tokens_seen": 239271936 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007494804413239719, + "loss": 1.7014, + "theoretical_loss": 4.288345174132313, + "tokens_seen": 239337472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007494643931795387, + "loss": 1.8444, + "theoretical_loss": 4.288203283779012, + "tokens_seen": 239403008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007494483450351053, + "loss": 1.7746, + "theoretical_loss": 4.288061443134947, + "tokens_seen": 239468544 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007494322968906721, + "loss": 1.7769, + "theoretical_loss": 4.287919652169107, + "tokens_seen": 239534080 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007494162487462387, + "loss": 1.6229, + "theoretical_loss": 4.287777910850507, + "tokens_seen": 239599616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007494002006018055, + "loss": 1.7149, + "theoretical_loss": 4.287636219148191, + "tokens_seen": 239665152 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007493841524573721, + "loss": 1.7742, + "theoretical_loss": 4.287494577031231, + "tokens_seen": 239730688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007493681043129389, + "loss": 1.7312, + "theoretical_loss": 4.287352984468727, + "tokens_seen": 239796224 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007493520561685055, + "loss": 1.7882, + "theoretical_loss": 4.287211441429805, + "tokens_seen": 239861760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007493360080240723, + "loss": 1.794, + "theoretical_loss": 4.2870699478836185, + "tokens_seen": 239927296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007493199598796389, + "loss": 1.7121, + "theoretical_loss": 4.286928503799352, + "tokens_seen": 239992832 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007493039117352057, + "loss": 1.6252, + "theoretical_loss": 4.286787109146213, + "tokens_seen": 240058368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007492878635907723, + "loss": 1.7422, + "theoretical_loss": 4.286645763893442, + "tokens_seen": 240123904 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007492718154463391, + "loss": 1.7673, + "theoretical_loss": 4.2865044680103015, + "tokens_seen": 240189440 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007492557673019057, + "loss": 1.8174, + "theoretical_loss": 4.286363221466084, + "tokens_seen": 240254976 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007492397191574725, + "loss": 1.7362, + "theoretical_loss": 4.286222024230111, + "tokens_seen": 240320512 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007492236710130391, + "loss": 1.6315, + "theoretical_loss": 4.286080876271729, + "tokens_seen": 240386048 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007492076228686059, + "loss": 1.6934, + "theoretical_loss": 4.285939777560312, + "tokens_seen": 240451584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007491915747241725, + "loss": 1.8109, + "theoretical_loss": 4.285798728065264, + "tokens_seen": 240517120 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007491755265797393, + "loss": 1.7968, + "theoretical_loss": 4.285657727756012, + "tokens_seen": 240582656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007491594784353059, + "loss": 1.814, + "theoretical_loss": 4.285516776602016, + "tokens_seen": 240648192 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007491434302908727, + "loss": 1.6675, + "theoretical_loss": 4.2853758745727575, + "tokens_seen": 240713728 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007491273821464393, + "loss": 1.5819, + "theoretical_loss": 4.285235021637748, + "tokens_seen": 240779264 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007491113340020061, + "loss": 1.6374, + "theoretical_loss": 4.285094217766527, + "tokens_seen": 240844800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007490952858575727, + "loss": 1.6466, + "theoretical_loss": 4.2849534629286605, + "tokens_seen": 240910336 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007490792377131395, + "loss": 1.6899, + "theoretical_loss": 4.284812757093741, + "tokens_seen": 240975872 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007490631895687061, + "loss": 1.6095, + "theoretical_loss": 4.284672100231388, + "tokens_seen": 241041408 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007490471414242729, + "loss": 1.7609, + "theoretical_loss": 4.284531492311248, + "tokens_seen": 241106944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007490310932798395, + "loss": 1.6355, + "theoretical_loss": 4.284390933302997, + "tokens_seen": 241172480 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007490150451354063, + "loss": 1.6593, + "theoretical_loss": 4.284250423176335, + "tokens_seen": 241238016 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007489989969909729, + "loss": 1.6879, + "theoretical_loss": 4.28410996190099, + "tokens_seen": 241303552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007489829488465397, + "loss": 1.7041, + "theoretical_loss": 4.283969549446717, + "tokens_seen": 241369088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007489669007021063, + "loss": 1.7684, + "theoretical_loss": 4.2838291857833, + "tokens_seen": 241434624 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007489508525576731, + "loss": 1.7195, + "theoretical_loss": 4.283688870880545, + "tokens_seen": 241500160 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007489348044132397, + "loss": 1.6173, + "theoretical_loss": 4.28354860470829, + "tokens_seen": 241565696 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007489187562688065, + "loss": 1.7035, + "theoretical_loss": 4.283408387236397, + "tokens_seen": 241631232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007489027081243731, + "loss": 1.6983, + "theoretical_loss": 4.283268218434755, + "tokens_seen": 241696768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007488866599799399, + "loss": 1.6614, + "theoretical_loss": 4.283128098273281, + "tokens_seen": 241762304 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007488706118355065, + "loss": 1.6902, + "theoretical_loss": 4.282988026721917, + "tokens_seen": 241827840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007488545636910733, + "loss": 1.5715, + "theoretical_loss": 4.282848003750634, + "tokens_seen": 241893376 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007488385155466399, + "loss": 1.678, + "theoretical_loss": 4.282708029329427, + "tokens_seen": 241958912 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007488224674022067, + "loss": 1.7728, + "theoretical_loss": 4.2825681034283205, + "tokens_seen": 242024448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007488064192577733, + "loss": 1.788, + "theoretical_loss": 4.2824282260173625, + "tokens_seen": 242089984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007487903711133401, + "loss": 1.8079, + "theoretical_loss": 4.282288397066631, + "tokens_seen": 242155520 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007487743229689067, + "loss": 1.7057, + "theoretical_loss": 4.282148616546227, + "tokens_seen": 242221056 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007487582748244735, + "loss": 1.6762, + "theoretical_loss": 4.282008884426281, + "tokens_seen": 242286592 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007487422266800401, + "loss": 1.5425, + "theoretical_loss": 4.281869200676948, + "tokens_seen": 242352128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007487261785356069, + "loss": 1.7597, + "theoretical_loss": 4.281729565268412, + "tokens_seen": 242417664 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.9916869401931763, + "objective/train/avg_token_score": 0.008313081227242947, + "objective/train/docs_used": 94878, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.7383466958999634, + "objective/train/original_loss": 1.7558294534683228, + "objective/train/theoretical_loss": 4.28158997817088, + "objective/train/tokens_used": 262943200, + "objective/train/unlikelihood_loss": 0.01621995121240616, + "theoretical_loss": 4.28158997817088, + "tokens_seen": 242483200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007487101303911735, + "loss": 1.6975, + "theoretical_loss": 4.28158997817088, + "tokens_seen": 242483200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007486940822467403, + "loss": 1.7364, + "theoretical_loss": 4.281450439354587, + "tokens_seen": 242548736 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007486780341023069, + "loss": 1.7068, + "theoretical_loss": 4.281310948789796, + "tokens_seen": 242614272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007486619859578737, + "loss": 1.6335, + "theoretical_loss": 4.2811715064467935, + "tokens_seen": 242679808 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007486459378134403, + "loss": 1.746, + "theoretical_loss": 4.281032112295894, + "tokens_seen": 242745344 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007486298896690071, + "loss": 1.7147, + "theoretical_loss": 4.2808927663074385, + "tokens_seen": 242810880 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007486138415245737, + "loss": 1.7, + "theoretical_loss": 4.280753468451793, + "tokens_seen": 242876416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007485977933801405, + "loss": 1.6658, + "theoretical_loss": 4.28061421869935, + "tokens_seen": 242941952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007485817452357071, + "loss": 1.7196, + "theoretical_loss": 4.28047501702053, + "tokens_seen": 243007488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007485656970912739, + "loss": 1.7172, + "theoretical_loss": 4.280335863385778, + "tokens_seen": 243073024 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007485496489468405, + "loss": 1.6317, + "theoretical_loss": 4.280196757765564, + "tokens_seen": 243138560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007485336008024073, + "loss": 1.6503, + "theoretical_loss": 4.280057700130388, + "tokens_seen": 243204096 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007485175526579739, + "loss": 1.7092, + "theoretical_loss": 4.279918690450771, + "tokens_seen": 243269632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007485015045135407, + "loss": 1.7281, + "theoretical_loss": 4.279779728697263, + "tokens_seen": 243335168 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007484854563691073, + "loss": 1.714, + "theoretical_loss": 4.279640814840441, + "tokens_seen": 243400704 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007484694082246741, + "loss": 1.6902, + "theoretical_loss": 4.279501948850905, + "tokens_seen": 243466240 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007484533600802407, + "loss": 1.6545, + "theoretical_loss": 4.279363130699284, + "tokens_seen": 243531776 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007484373119358075, + "loss": 1.7983, + "theoretical_loss": 4.279224360356231, + "tokens_seen": 243597312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007484212637913741, + "loss": 1.7695, + "theoretical_loss": 4.279085637792425, + "tokens_seen": 243662848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007484052156469409, + "loss": 1.7933, + "theoretical_loss": 4.278946962978573, + "tokens_seen": 243728384 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007483891675025075, + "loss": 1.717, + "theoretical_loss": 4.278808335885403, + "tokens_seen": 243793920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007483731193580743, + "loss": 1.6698, + "theoretical_loss": 4.278669756483674, + "tokens_seen": 243859456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007483570712136409, + "loss": 1.7323, + "theoretical_loss": 4.278531224744167, + "tokens_seen": 243924992 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007483410230692077, + "loss": 1.6768, + "theoretical_loss": 4.278392740637694, + "tokens_seen": 243990528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007483249749247745, + "loss": 1.6997, + "theoretical_loss": 4.278254304135085, + "tokens_seen": 244056064 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007483089267803411, + "loss": 1.8116, + "theoretical_loss": 4.2781159152072, + "tokens_seen": 244121600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007482928786359077, + "loss": 1.7016, + "theoretical_loss": 4.277977573824928, + "tokens_seen": 244187136 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007482768304914745, + "loss": 1.7291, + "theoretical_loss": 4.277839279959177, + "tokens_seen": 244252672 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007482607823470411, + "loss": 1.6236, + "theoretical_loss": 4.277701033580884, + "tokens_seen": 244318208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007482447342026079, + "loss": 1.5278, + "theoretical_loss": 4.277562834661013, + "tokens_seen": 244383744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007482286860581746, + "loss": 1.6706, + "theoretical_loss": 4.277424683170549, + "tokens_seen": 244449280 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007482126379137413, + "loss": 1.6369, + "theoretical_loss": 4.277286579080508, + "tokens_seen": 244514816 + }, + { + "epoch": 0.07, + "learning_rate": 0.000748196589769308, + "loss": 1.7, + "theoretical_loss": 4.277148522361927, + "tokens_seen": 244580352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007481805416248747, + "loss": 1.669, + "theoretical_loss": 4.277010512985871, + "tokens_seen": 244645888 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007481644934804414, + "loss": 1.7837, + "theoretical_loss": 4.2768725509234296, + "tokens_seen": 244711424 + }, + { + "epoch": 0.07, + "learning_rate": 0.000748148445336008, + "loss": 1.6421, + "theoretical_loss": 4.276734636145717, + "tokens_seen": 244776960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007481323971915748, + "loss": 1.8009, + "theoretical_loss": 4.276596768623875, + "tokens_seen": 244842496 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007481163490471414, + "loss": 1.7011, + "theoretical_loss": 4.276458948329068, + "tokens_seen": 244908032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007481003009027082, + "loss": 1.7277, + "theoretical_loss": 4.276321175232487, + "tokens_seen": 244973568 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007480842527582748, + "loss": 1.6389, + "theoretical_loss": 4.276183449305348, + "tokens_seen": 245039104 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007480682046138416, + "loss": 1.7892, + "theoretical_loss": 4.276045770518895, + "tokens_seen": 245104640 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007480521564694082, + "loss": 1.8932, + "theoretical_loss": 4.275908138844392, + "tokens_seen": 245170176 + }, + { + "epoch": 0.07, + "learning_rate": 0.000748036108324975, + "loss": 1.6764, + "theoretical_loss": 4.275770554253132, + "tokens_seen": 245235712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007480200601805416, + "loss": 1.6491, + "theoretical_loss": 4.275633016716432, + "tokens_seen": 245301248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007480040120361084, + "loss": 1.7388, + "theoretical_loss": 4.275495526205635, + "tokens_seen": 245366784 + }, + { + "epoch": 0.07, + "learning_rate": 0.000747987963891675, + "loss": 1.6936, + "theoretical_loss": 4.275358082692106, + "tokens_seen": 245432320 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007479719157472418, + "loss": 1.7425, + "theoretical_loss": 4.275220686147241, + "tokens_seen": 245497856 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007479558676028084, + "loss": 1.5805, + "theoretical_loss": 4.275083336542455, + "tokens_seen": 245563392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007479398194583752, + "loss": 1.6967, + "theoretical_loss": 4.274946033849189, + "tokens_seen": 245628928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007479237713139418, + "loss": 1.6975, + "theoretical_loss": 4.2748087780389135, + "tokens_seen": 245694464 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.9900068640708923, + "objective/train/avg_token_score": 0.009993130341172218, + "objective/train/docs_used": 96014, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.7929363250732422, + "objective/train/original_loss": 1.8143107891082764, + "objective/train/theoretical_loss": 4.27467156908312, + "objective/train/tokens_used": 266220000, + "objective/train/unlikelihood_loss": 0.01521708071231842, + "theoretical_loss": 4.27467156908312, + "tokens_seen": 245760000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007479077231695086, + "loss": 1.6756, + "theoretical_loss": 4.27467156908312, + "tokens_seen": 245760000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007478916750250752, + "loss": 1.6776, + "theoretical_loss": 4.274534406953325, + "tokens_seen": 245825536 + }, + { + "epoch": 0.07, + "learning_rate": 0.000747875626880642, + "loss": 1.6873, + "theoretical_loss": 4.274397291621073, + "tokens_seen": 245891072 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007478595787362086, + "loss": 1.6542, + "theoretical_loss": 4.274260223057928, + "tokens_seen": 245956608 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007478435305917754, + "loss": 1.7483, + "theoretical_loss": 4.274123201235483, + "tokens_seen": 246022144 + }, + { + "epoch": 0.07, + "learning_rate": 0.000747827482447342, + "loss": 1.7062, + "theoretical_loss": 4.273986226125356, + "tokens_seen": 246087680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007478114343029088, + "loss": 1.5037, + "theoretical_loss": 4.273849297699186, + "tokens_seen": 246153216 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007477953861584754, + "loss": 1.7528, + "theoretical_loss": 4.273712415928643, + "tokens_seen": 246218752 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007477793380140422, + "loss": 1.7415, + "theoretical_loss": 4.273575580785415, + "tokens_seen": 246284288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007477632898696088, + "loss": 1.7411, + "theoretical_loss": 4.27343879224122, + "tokens_seen": 246349824 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007477472417251756, + "loss": 1.7331, + "theoretical_loss": 4.273302050267796, + "tokens_seen": 246415360 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007477311935807422, + "loss": 1.7566, + "theoretical_loss": 4.27316535483691, + "tokens_seen": 246480896 + }, + { + "epoch": 0.07, + "learning_rate": 0.000747715145436309, + "loss": 1.7239, + "theoretical_loss": 4.273028705920351, + "tokens_seen": 246546432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007476990972918756, + "loss": 1.6561, + "theoretical_loss": 4.272892103489934, + "tokens_seen": 246611968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007476830491474424, + "loss": 1.7051, + "theoretical_loss": 4.272755547517495, + "tokens_seen": 246677504 + }, + { + "epoch": 0.07, + "learning_rate": 0.000747667001003009, + "loss": 1.5944, + "theoretical_loss": 4.272619037974902, + "tokens_seen": 246743040 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007476509528585758, + "loss": 1.6016, + "theoretical_loss": 4.272482574834039, + "tokens_seen": 246808576 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007476349047141424, + "loss": 1.6198, + "theoretical_loss": 4.272346158066821, + "tokens_seen": 246874112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007476188565697092, + "loss": 1.7841, + "theoretical_loss": 4.272209787645183, + "tokens_seen": 246939648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007476028084252758, + "loss": 1.7073, + "theoretical_loss": 4.272073463541087, + "tokens_seen": 247005184 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007475867602808426, + "loss": 1.6546, + "theoretical_loss": 4.271937185726519, + "tokens_seen": 247070720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007475707121364092, + "loss": 1.6923, + "theoretical_loss": 4.2718009541734885, + "tokens_seen": 247136256 + }, + { + "epoch": 0.07, + "learning_rate": 0.000747554663991976, + "loss": 1.6616, + "theoretical_loss": 4.27166476885403, + "tokens_seen": 247201792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007475386158475426, + "loss": 1.8023, + "theoretical_loss": 4.271528629740201, + "tokens_seen": 247267328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007475225677031094, + "loss": 1.7244, + "theoretical_loss": 4.271392536804088, + "tokens_seen": 247332864 + }, + { + "epoch": 0.07, + "learning_rate": 0.000747506519558676, + "loss": 1.7125, + "theoretical_loss": 4.2712564900177945, + "tokens_seen": 247398400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007474904714142428, + "loss": 1.6857, + "theoretical_loss": 4.271120489353454, + "tokens_seen": 247463936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007474744232698094, + "loss": 1.6274, + "theoretical_loss": 4.270984534783221, + "tokens_seen": 247529472 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007474583751253762, + "loss": 1.7201, + "theoretical_loss": 4.270848626279275, + "tokens_seen": 247595008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007474423269809428, + "loss": 1.7196, + "theoretical_loss": 4.270712763813822, + "tokens_seen": 247660544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007474262788365096, + "loss": 1.7257, + "theoretical_loss": 4.270576947359088, + "tokens_seen": 247726080 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007474102306920762, + "loss": 1.746, + "theoretical_loss": 4.270441176887327, + "tokens_seen": 247791616 + }, + { + "epoch": 0.08, + "learning_rate": 0.000747394182547643, + "loss": 1.6805, + "theoretical_loss": 4.270305452370814, + "tokens_seen": 247857152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007473781344032096, + "loss": 1.6324, + "theoretical_loss": 4.27016977378185, + "tokens_seen": 247922688 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007473620862587764, + "loss": 1.7829, + "theoretical_loss": 4.270034141092759, + "tokens_seen": 247988224 + }, + { + "epoch": 0.08, + "learning_rate": 0.000747346038114343, + "loss": 1.7015, + "theoretical_loss": 4.269898554275889, + "tokens_seen": 248053760 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007473299899699098, + "loss": 1.674, + "theoretical_loss": 4.269763013303613, + "tokens_seen": 248119296 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007473139418254764, + "loss": 1.6498, + "theoretical_loss": 4.269627518148326, + "tokens_seen": 248184832 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007472978936810432, + "loss": 1.6915, + "theoretical_loss": 4.26949206878245, + "tokens_seen": 248250368 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007472818455366098, + "loss": 1.7937, + "theoretical_loss": 4.269356665178426, + "tokens_seen": 248315904 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007472657973921766, + "loss": 1.8696, + "theoretical_loss": 4.269221307308725, + "tokens_seen": 248381440 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007472497492477432, + "loss": 1.7577, + "theoretical_loss": 4.269085995145837, + "tokens_seen": 248446976 + }, + { + "epoch": 0.08, + "learning_rate": 0.00074723370110331, + "loss": 1.7144, + "theoretical_loss": 4.268950728662278, + "tokens_seen": 248512512 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007472176529588766, + "loss": 2.0588, + "theoretical_loss": 4.268815507830587, + "tokens_seen": 248578048 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007472016048144434, + "loss": 1.627, + "theoretical_loss": 4.268680332623328, + "tokens_seen": 248643584 + }, + { + "epoch": 0.08, + "learning_rate": 0.00074718555667001, + "loss": 1.6396, + "theoretical_loss": 4.268545203013085, + "tokens_seen": 248709120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007471695085255768, + "loss": 1.7075, + "theoretical_loss": 4.26841011897247, + "tokens_seen": 248774656 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007471534603811434, + "loss": 1.8011, + "theoretical_loss": 4.268275080474118, + "tokens_seen": 248840192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007471374122367102, + "loss": 1.6753, + "theoretical_loss": 4.268140087490686, + "tokens_seen": 248905728 + }, + { + "epoch": 0.08, + "learning_rate": 0.000747121364092277, + "loss": 1.7615, + "theoretical_loss": 4.2680051399948535, + "tokens_seen": 248971264 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.9903026223182678, + "objective/train/avg_token_score": 0.00969735998660326, + "objective/train/docs_used": 97104, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6956777572631836, + "objective/train/original_loss": 1.7133991718292236, + "objective/train/theoretical_loss": 4.267870237959327, + "objective/train/tokens_used": 269496800, + "objective/train/unlikelihood_loss": 0.018521277233958244, + "theoretical_loss": 4.267870237959327, + "tokens_seen": 249036800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007471053159478436, + "loss": 1.6765, + "theoretical_loss": 4.267870237959327, + "tokens_seen": 249036800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007470892678034103, + "loss": 1.6808, + "theoretical_loss": 4.267735381356835, + "tokens_seen": 249102336 + }, + { + "epoch": 0.08, + "learning_rate": 0.000747073219658977, + "loss": 1.7438, + "theoretical_loss": 4.26760057016013, + "tokens_seen": 249167872 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007470571715145438, + "loss": 1.8044, + "theoretical_loss": 4.2674658043419855, + "tokens_seen": 249233408 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007470411233701104, + "loss": 1.6556, + "theoretical_loss": 4.267331083875202, + "tokens_seen": 249298944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007470250752256772, + "loss": 1.4385, + "theoretical_loss": 4.267196408732601, + "tokens_seen": 249364480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007470090270812438, + "loss": 1.6197, + "theoretical_loss": 4.267061778887029, + "tokens_seen": 249430016 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007469929789368106, + "loss": 1.7026, + "theoretical_loss": 4.2669271943113545, + "tokens_seen": 249495552 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007469769307923772, + "loss": 1.6234, + "theoretical_loss": 4.26679265497847, + "tokens_seen": 249561088 + }, + { + "epoch": 0.08, + "learning_rate": 0.000746960882647944, + "loss": 1.6846, + "theoretical_loss": 4.266658160861293, + "tokens_seen": 249626624 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007469448345035106, + "loss": 1.8012, + "theoretical_loss": 4.266523711932761, + "tokens_seen": 249692160 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007469287863590774, + "loss": 1.7979, + "theoretical_loss": 4.266389308165837, + "tokens_seen": 249757696 + }, + { + "epoch": 0.08, + "learning_rate": 0.000746912738214644, + "loss": 1.683, + "theoretical_loss": 4.266254949533507, + "tokens_seen": 249823232 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007468966900702106, + "loss": 1.7171, + "theoretical_loss": 4.26612063600878, + "tokens_seen": 249888768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007468806419257774, + "loss": 1.8436, + "theoretical_loss": 4.265986367564688, + "tokens_seen": 249954304 + }, + { + "epoch": 0.08, + "learning_rate": 0.000746864593781344, + "loss": 1.7409, + "theoretical_loss": 4.265852144174286, + "tokens_seen": 250019840 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007468485456369108, + "loss": 1.5715, + "theoretical_loss": 4.265717965810653, + "tokens_seen": 250085376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007468324974924775, + "loss": 1.6205, + "theoretical_loss": 4.265583832446892, + "tokens_seen": 250150912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007468164493480442, + "loss": 1.7238, + "theoretical_loss": 4.265449744056126, + "tokens_seen": 250216448 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007468004012036109, + "loss": 1.7216, + "theoretical_loss": 4.265315700611504, + "tokens_seen": 250281984 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007467843530591776, + "loss": 1.8136, + "theoretical_loss": 4.2651817020861955, + "tokens_seen": 250347520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007467683049147443, + "loss": 1.7391, + "theoretical_loss": 4.265047748453396, + "tokens_seen": 250413056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007467522567703109, + "loss": 1.7536, + "theoretical_loss": 4.264913839686323, + "tokens_seen": 250478592 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007467362086258777, + "loss": 1.6522, + "theoretical_loss": 4.264779975758215, + "tokens_seen": 250544128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007467201604814443, + "loss": 1.6051, + "theoretical_loss": 4.2646461566423355, + "tokens_seen": 250609664 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007467041123370111, + "loss": 1.6657, + "theoretical_loss": 4.26451238231197, + "tokens_seen": 250675200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007466880641925777, + "loss": 1.8157, + "theoretical_loss": 4.2643786527404295, + "tokens_seen": 250740736 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007466720160481445, + "loss": 1.7305, + "theoretical_loss": 4.264244967901043, + "tokens_seen": 250806272 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007466559679037111, + "loss": 1.6456, + "theoretical_loss": 4.264111327767164, + "tokens_seen": 250871808 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007466399197592779, + "loss": 1.8448, + "theoretical_loss": 4.263977732312174, + "tokens_seen": 250937344 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007466238716148445, + "loss": 1.6408, + "theoretical_loss": 4.26384418150947, + "tokens_seen": 251002880 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007466078234704113, + "loss": 1.7625, + "theoretical_loss": 4.263710675332476, + "tokens_seen": 251068416 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007465917753259779, + "loss": 1.7981, + "theoretical_loss": 4.263577213754638, + "tokens_seen": 251133952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007465757271815447, + "loss": 1.7959, + "theoretical_loss": 4.2634437967494225, + "tokens_seen": 251199488 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007465596790371113, + "loss": 1.7183, + "theoretical_loss": 4.2633104242903235, + "tokens_seen": 251265024 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007465436308926781, + "loss": 1.7429, + "theoretical_loss": 4.263177096350852, + "tokens_seen": 251330560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007465275827482447, + "loss": 1.6706, + "theoretical_loss": 4.263043812904547, + "tokens_seen": 251396096 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007465115346038115, + "loss": 1.8128, + "theoretical_loss": 4.262910573924965, + "tokens_seen": 251461632 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007464954864593781, + "loss": 1.5664, + "theoretical_loss": 4.26277737938569, + "tokens_seen": 251527168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007464794383149449, + "loss": 1.7401, + "theoretical_loss": 4.262644229260324, + "tokens_seen": 251592704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007464633901705115, + "loss": 1.7829, + "theoretical_loss": 4.262511123522495, + "tokens_seen": 251658240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007464473420260783, + "loss": 1.6073, + "theoretical_loss": 4.262378062145853, + "tokens_seen": 251723776 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007464312938816449, + "loss": 1.7025, + "theoretical_loss": 4.262245045104068, + "tokens_seen": 251789312 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007464152457372117, + "loss": 1.6244, + "theoretical_loss": 4.262112072370837, + "tokens_seen": 251854848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007463991975927783, + "loss": 1.7421, + "theoretical_loss": 4.261979143919873, + "tokens_seen": 251920384 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007463831494483451, + "loss": 1.7253, + "theoretical_loss": 4.261846259724917, + "tokens_seen": 251985920 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007463671013039117, + "loss": 1.6926, + "theoretical_loss": 4.261713419759731, + "tokens_seen": 252051456 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007463510531594785, + "loss": 1.6681, + "theoretical_loss": 4.2615806239981, + "tokens_seen": 252116992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007463350050150451, + "loss": 1.7254, + "theoretical_loss": 4.261447872413827, + "tokens_seen": 252182528 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007463189568706119, + "loss": 1.6286, + "theoretical_loss": 4.261315164980743, + "tokens_seen": 252248064 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.9949523210525513, + "objective/train/avg_token_score": 0.0050476547330617905, + "objective/train/docs_used": 98257, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.7054072618484497, + "objective/train/original_loss": 1.7170658111572266, + "objective/train/theoretical_loss": 4.261182501672698, + "objective/train/tokens_used": 272773600, + "objective/train/unlikelihood_loss": 0.0072880275547504425, + "theoretical_loss": 4.261182501672698, + "tokens_seen": 252313600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007463029087261785, + "loss": 1.6338, + "theoretical_loss": 4.261182501672698, + "tokens_seen": 252313600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007462868605817453, + "loss": 1.7413, + "theoretical_loss": 4.261049882463565, + "tokens_seen": 252379136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007462708124373119, + "loss": 1.7306, + "theoretical_loss": 4.26091730732724, + "tokens_seen": 252444672 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007462547642928787, + "loss": 1.593, + "theoretical_loss": 4.2607847762376405, + "tokens_seen": 252510208 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007462387161484453, + "loss": 1.6815, + "theoretical_loss": 4.260652289168706, + "tokens_seen": 252575744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007462226680040121, + "loss": 1.7139, + "theoretical_loss": 4.260519846094399, + "tokens_seen": 252641280 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007462066198595787, + "loss": 1.7339, + "theoretical_loss": 4.2603874469887035, + "tokens_seen": 252706816 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007461905717151455, + "loss": 1.7167, + "theoretical_loss": 4.260255091825625, + "tokens_seen": 252772352 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007461745235707121, + "loss": 1.7418, + "theoretical_loss": 4.260122780579193, + "tokens_seen": 252837888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007461584754262789, + "loss": 1.6296, + "theoretical_loss": 4.259990513223458, + "tokens_seen": 252903424 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007461424272818455, + "loss": 1.78, + "theoretical_loss": 4.259858289732491, + "tokens_seen": 252968960 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007461263791374123, + "loss": 1.6315, + "theoretical_loss": 4.259726110080388, + "tokens_seen": 253034496 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007461103309929789, + "loss": 1.6163, + "theoretical_loss": 4.259593974241264, + "tokens_seen": 253100032 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007460942828485457, + "loss": 1.6621, + "theoretical_loss": 4.25946188218926, + "tokens_seen": 253165568 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007460782347041123, + "loss": 1.6205, + "theoretical_loss": 4.259329833898534, + "tokens_seen": 253231104 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007460621865596791, + "loss": 1.6286, + "theoretical_loss": 4.25919782934327, + "tokens_seen": 253296640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007460461384152457, + "loss": 1.749, + "theoretical_loss": 4.259065868497672, + "tokens_seen": 253362176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007460300902708125, + "loss": 1.6726, + "theoretical_loss": 4.258933951335965, + "tokens_seen": 253427712 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007460140421263791, + "loss": 1.8205, + "theoretical_loss": 4.258802077832399, + "tokens_seen": 253493248 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007459979939819459, + "loss": 1.6805, + "theoretical_loss": 4.258670247961243, + "tokens_seen": 253558784 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007459819458375125, + "loss": 1.6863, + "theoretical_loss": 4.258538461696787, + "tokens_seen": 253624320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007459658976930793, + "loss": 1.6463, + "theoretical_loss": 4.258406719013347, + "tokens_seen": 253689856 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007459498495486459, + "loss": 1.7695, + "theoretical_loss": 4.258275019885257, + "tokens_seen": 253755392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007459338014042127, + "loss": 1.7285, + "theoretical_loss": 4.258143364286875, + "tokens_seen": 253820928 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007459177532597794, + "loss": 1.6054, + "theoretical_loss": 4.258011752192579, + "tokens_seen": 253886464 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007459017051153461, + "loss": 1.6681, + "theoretical_loss": 4.257880183576768, + "tokens_seen": 253952000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007458856569709128, + "loss": 1.6762, + "theoretical_loss": 4.257748658413867, + "tokens_seen": 254017536 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007458696088264795, + "loss": 1.6552, + "theoretical_loss": 4.257617176678317, + "tokens_seen": 254083072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007458535606820462, + "loss": 1.6599, + "theoretical_loss": 4.2574857383445845, + "tokens_seen": 254148608 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007458375125376129, + "loss": 1.7385, + "theoretical_loss": 4.257354343387156, + "tokens_seen": 254214144 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007458214643931796, + "loss": 1.7034, + "theoretical_loss": 4.257222991780541, + "tokens_seen": 254279680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007458054162487463, + "loss": 1.808, + "theoretical_loss": 4.257091683499268, + "tokens_seen": 254345216 + }, + { + "epoch": 0.08, + "learning_rate": 0.000745789368104313, + "loss": 1.8676, + "theoretical_loss": 4.25696041851789, + "tokens_seen": 254410752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007457733199598797, + "loss": 1.7501, + "theoretical_loss": 4.256829196810978, + "tokens_seen": 254476288 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007457572718154464, + "loss": 1.6174, + "theoretical_loss": 4.256698018353129, + "tokens_seen": 254541824 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007457412236710131, + "loss": 1.7451, + "theoretical_loss": 4.256566883118958, + "tokens_seen": 254607360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007457251755265798, + "loss": 1.687, + "theoretical_loss": 4.256435791083101, + "tokens_seen": 254672896 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007457091273821465, + "loss": 1.7698, + "theoretical_loss": 4.256304742220219, + "tokens_seen": 254738432 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007456930792377133, + "loss": 1.7307, + "theoretical_loss": 4.25617373650499, + "tokens_seen": 254803968 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007456770310932799, + "loss": 1.5561, + "theoretical_loss": 4.256042773912117, + "tokens_seen": 254869504 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007456609829488467, + "loss": 1.7058, + "theoretical_loss": 4.255911854416323, + "tokens_seen": 254935040 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007456449348044133, + "loss": 1.7615, + "theoretical_loss": 4.255780977992352, + "tokens_seen": 255000576 + }, + { + "epoch": 0.08, + "learning_rate": 0.00074562888665998, + "loss": 1.5752, + "theoretical_loss": 4.255650144614968, + "tokens_seen": 255066112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007456128385155467, + "loss": 1.7385, + "theoretical_loss": 4.255519354258959, + "tokens_seen": 255131648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007455967903711135, + "loss": 1.7003, + "theoretical_loss": 4.255388606899133, + "tokens_seen": 255197184 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007455807422266801, + "loss": 1.6726, + "theoretical_loss": 4.255257902510319, + "tokens_seen": 255262720 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007455646940822469, + "loss": 1.7307, + "theoretical_loss": 4.255127241067368, + "tokens_seen": 255328256 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007455486459378135, + "loss": 1.7527, + "theoretical_loss": 4.25499662254515, + "tokens_seen": 255393792 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007455325977933803, + "loss": 1.6548, + "theoretical_loss": 4.254866046918559, + "tokens_seen": 255459328 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007455165496489469, + "loss": 1.7405, + "theoretical_loss": 4.254735514162508, + "tokens_seen": 255524864 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.9922325611114502, + "objective/train/avg_token_score": 0.00776749849319458, + "objective/train/docs_used": 99403, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.7506349086761475, + "objective/train/original_loss": 1.7650294303894043, + "objective/train/theoretical_loss": 4.254605024251933, + "objective/train/tokens_used": 276050400, + "objective/train/unlikelihood_loss": 0.014327810145914555, + "theoretical_loss": 4.254605024251933, + "tokens_seen": 255590400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007455005015045136, + "loss": 1.6871, + "theoretical_loss": 4.254605024251933, + "tokens_seen": 255590400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007454844533600803, + "loss": 1.6841, + "theoretical_loss": 4.254474577161789, + "tokens_seen": 255655936 + }, + { + "epoch": 0.08, + "learning_rate": 0.000745468405215647, + "loss": 1.5348, + "theoretical_loss": 4.254344172867053, + "tokens_seen": 255721472 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007454523570712137, + "loss": 1.7119, + "theoretical_loss": 4.254213811342723, + "tokens_seen": 255787008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007454363089267804, + "loss": 1.6979, + "theoretical_loss": 4.254083492563819, + "tokens_seen": 255852544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007454202607823471, + "loss": 1.6338, + "theoretical_loss": 4.253953216505381, + "tokens_seen": 255918080 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007454042126379138, + "loss": 1.7678, + "theoretical_loss": 4.253822983142468, + "tokens_seen": 255983616 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007453881644934805, + "loss": 1.6223, + "theoretical_loss": 4.253692792450165, + "tokens_seen": 256049152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007453721163490472, + "loss": 1.7098, + "theoretical_loss": 4.253562644403573, + "tokens_seen": 256114688 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007453560682046138, + "loss": 1.687, + "theoretical_loss": 4.253432538977816, + "tokens_seen": 256180224 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007453400200601806, + "loss": 1.6215, + "theoretical_loss": 4.253302476148039, + "tokens_seen": 256245760 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007453239719157472, + "loss": 1.7388, + "theoretical_loss": 4.253172455889407, + "tokens_seen": 256311296 + }, + { + "epoch": 0.08, + "learning_rate": 0.000745307923771314, + "loss": 1.6741, + "theoretical_loss": 4.2530424781771075, + "tokens_seen": 256376832 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007452918756268806, + "loss": 1.8368, + "theoretical_loss": 4.252912542986345, + "tokens_seen": 256442368 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007452758274824474, + "loss": 1.5702, + "theoretical_loss": 4.2527826502923505, + "tokens_seen": 256507904 + }, + { + "epoch": 0.08, + "learning_rate": 0.000745259779338014, + "loss": 1.7219, + "theoretical_loss": 4.25265280007037, + "tokens_seen": 256573440 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007452437311935808, + "loss": 1.6292, + "theoretical_loss": 4.252522992295676, + "tokens_seen": 256638976 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007452276830491474, + "loss": 1.6227, + "theoretical_loss": 4.252393226943555, + "tokens_seen": 256704512 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007452116349047142, + "loss": 1.6683, + "theoretical_loss": 4.252263503989321, + "tokens_seen": 256770048 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007451955867602808, + "loss": 1.6791, + "theoretical_loss": 4.252133823408303, + "tokens_seen": 256835584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007451795386158476, + "loss": 1.7457, + "theoretical_loss": 4.2520041851758545, + "tokens_seen": 256901120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007451634904714142, + "loss": 1.5588, + "theoretical_loss": 4.251874589267347, + "tokens_seen": 256966656 + }, + { + "epoch": 0.08, + "learning_rate": 0.000745147442326981, + "loss": 1.6808, + "theoretical_loss": 4.251745035658176, + "tokens_seen": 257032192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007451313941825476, + "loss": 1.7986, + "theoretical_loss": 4.251615524323752, + "tokens_seen": 257097728 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007451153460381144, + "loss": 1.7778, + "theoretical_loss": 4.251486055239512, + "tokens_seen": 257163264 + }, + { + "epoch": 0.08, + "learning_rate": 0.000745099297893681, + "loss": 1.7848, + "theoretical_loss": 4.251356628380911, + "tokens_seen": 257228800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007450832497492478, + "loss": 1.8092, + "theoretical_loss": 4.251227243723424, + "tokens_seen": 257294336 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007450672016048144, + "loss": 1.6831, + "theoretical_loss": 4.251097901242546, + "tokens_seen": 257359872 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007450511534603812, + "loss": 1.833, + "theoretical_loss": 4.250968600913795, + "tokens_seen": 257425408 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007450351053159478, + "loss": 1.6974, + "theoretical_loss": 4.250839342712707, + "tokens_seen": 257490944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007450190571715146, + "loss": 1.6069, + "theoretical_loss": 4.250710126614838, + "tokens_seen": 257556480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007450030090270812, + "loss": 1.8393, + "theoretical_loss": 4.250580952595769, + "tokens_seen": 257622016 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744986960882648, + "loss": 1.7304, + "theoretical_loss": 4.250451820631095, + "tokens_seen": 257687552 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007449709127382146, + "loss": 1.664, + "theoretical_loss": 4.2503227306964355, + "tokens_seen": 257753088 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007449548645937814, + "loss": 1.7117, + "theoretical_loss": 4.25019368276743, + "tokens_seen": 257818624 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744938816449348, + "loss": 1.6777, + "theoretical_loss": 4.250064676819735, + "tokens_seen": 257884160 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007449227683049148, + "loss": 1.568, + "theoretical_loss": 4.249935712829033, + "tokens_seen": 257949696 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007449067201604814, + "loss": 1.9205, + "theoretical_loss": 4.249806790771022, + "tokens_seen": 258015232 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007448906720160482, + "loss": 1.7147, + "theoretical_loss": 4.249677910621422, + "tokens_seen": 258080768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007448746238716148, + "loss": 1.792, + "theoretical_loss": 4.249549072355974, + "tokens_seen": 258146304 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007448585757271816, + "loss": 1.7191, + "theoretical_loss": 4.2494202759504365, + "tokens_seen": 258211840 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007448425275827482, + "loss": 1.7045, + "theoretical_loss": 4.249291521380591, + "tokens_seen": 258277376 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744826479438315, + "loss": 1.733, + "theoretical_loss": 4.249162808622237, + "tokens_seen": 258342912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007448104312938816, + "loss": 1.8137, + "theoretical_loss": 4.249034137651197, + "tokens_seen": 258408448 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007447943831494484, + "loss": 1.6626, + "theoretical_loss": 4.248905508443311, + "tokens_seen": 258473984 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744778335005015, + "loss": 1.689, + "theoretical_loss": 4.248776920974439, + "tokens_seen": 258539520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007447622868605818, + "loss": 1.6402, + "theoretical_loss": 4.248648375220464, + "tokens_seen": 258605056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007447462387161484, + "loss": 1.6385, + "theoretical_loss": 4.248519871157285, + "tokens_seen": 258670592 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007447301905717152, + "loss": 1.7859, + "theoretical_loss": 4.248391408760824, + "tokens_seen": 258736128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007447141424272819, + "loss": 1.6156, + "theoretical_loss": 4.248262988007022, + "tokens_seen": 258801664 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.9922111630439758, + "objective/train/avg_token_score": 0.007788830436766148, + "objective/train/docs_used": 100739, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5748742818832397, + "objective/train/original_loss": 1.5889265537261963, + "objective/train/theoretical_loss": 4.248134608871841, + "objective/train/tokens_used": 279327200, + "objective/train/unlikelihood_loss": 0.019068362191319466, + "theoretical_loss": 4.248134608871841, + "tokens_seen": 258867200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007446980942828486, + "loss": 1.6656, + "theoretical_loss": 4.248134608871841, + "tokens_seen": 258867200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007446820461384153, + "loss": 1.76, + "theoretical_loss": 4.24800627133126, + "tokens_seen": 258932736 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744665997993982, + "loss": 1.8246, + "theoretical_loss": 4.247877975361281, + "tokens_seen": 258998272 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007446499498495487, + "loss": 1.6539, + "theoretical_loss": 4.247749720937925, + "tokens_seen": 259063808 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007446339017051154, + "loss": 1.718, + "theoretical_loss": 4.247621508037232, + "tokens_seen": 259129344 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007446178535606821, + "loss": 1.6954, + "theoretical_loss": 4.247493336635264, + "tokens_seen": 259194880 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007446018054162488, + "loss": 1.6578, + "theoretical_loss": 4.2473652067081, + "tokens_seen": 259260416 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007445857572718155, + "loss": 1.6479, + "theoretical_loss": 4.247237118231841, + "tokens_seen": 259325952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007445697091273822, + "loss": 1.7226, + "theoretical_loss": 4.247109071182607, + "tokens_seen": 259391488 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007445536609829489, + "loss": 1.7513, + "theoretical_loss": 4.246981065536538, + "tokens_seen": 259457024 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007445376128385156, + "loss": 1.7526, + "theoretical_loss": 4.246853101269793, + "tokens_seen": 259522560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007445215646940823, + "loss": 1.8571, + "theoretical_loss": 4.2467251783585525, + "tokens_seen": 259588096 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744505516549649, + "loss": 1.7127, + "theoretical_loss": 4.246597296779014, + "tokens_seen": 259653632 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007444894684052157, + "loss": 1.7481, + "theoretical_loss": 4.246469456507398, + "tokens_seen": 259719168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007444734202607824, + "loss": 1.7168, + "theoretical_loss": 4.246341657519943, + "tokens_seen": 259784704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007444573721163491, + "loss": 1.7561, + "theoretical_loss": 4.246213899792905, + "tokens_seen": 259850240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007444413239719158, + "loss": 1.7298, + "theoretical_loss": 4.246086183302564, + "tokens_seen": 259915776 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007444252758274825, + "loss": 1.5559, + "theoretical_loss": 4.2459585080252165, + "tokens_seen": 259981312 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007444092276830492, + "loss": 1.6725, + "theoretical_loss": 4.245830873937179, + "tokens_seen": 260046848 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744393179538616, + "loss": 1.7795, + "theoretical_loss": 4.24570328101479, + "tokens_seen": 260112384 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007443771313941826, + "loss": 1.7076, + "theoretical_loss": 4.245575729234403, + "tokens_seen": 260177920 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007443610832497493, + "loss": 1.7832, + "theoretical_loss": 4.245448218572396, + "tokens_seen": 260243456 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744345035105316, + "loss": 1.7738, + "theoretical_loss": 4.245320749005163, + "tokens_seen": 260308992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007443289869608828, + "loss": 1.641, + "theoretical_loss": 4.245193320509118, + "tokens_seen": 260374528 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007443129388164494, + "loss": 1.7419, + "theoretical_loss": 4.245065933060697, + "tokens_seen": 260440064 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007442968906720162, + "loss": 1.6923, + "theoretical_loss": 4.244938586636351, + "tokens_seen": 260505600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007442808425275828, + "loss": 1.7601, + "theoretical_loss": 4.244811281212556, + "tokens_seen": 260571136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007442647943831496, + "loss": 1.6368, + "theoretical_loss": 4.244684016765802, + "tokens_seen": 260636672 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007442487462387162, + "loss": 1.7913, + "theoretical_loss": 4.2445567932726025, + "tokens_seen": 260702208 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744232698094283, + "loss": 1.805, + "theoretical_loss": 4.244429610709488, + "tokens_seen": 260767744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007442166499498496, + "loss": 1.7097, + "theoretical_loss": 4.244302469053007, + "tokens_seen": 260833280 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007442006018054164, + "loss": 1.6873, + "theoretical_loss": 4.244175368279733, + "tokens_seen": 260898816 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744184553660983, + "loss": 1.5869, + "theoretical_loss": 4.244048308366253, + "tokens_seen": 260964352 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007441685055165498, + "loss": 1.5924, + "theoretical_loss": 4.243921289289175, + "tokens_seen": 261029888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007441524573721164, + "loss": 1.6635, + "theoretical_loss": 4.243794311025128, + "tokens_seen": 261095424 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007441364092276832, + "loss": 1.7181, + "theoretical_loss": 4.243667373550757, + "tokens_seen": 261160960 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007441203610832498, + "loss": 1.7301, + "theoretical_loss": 4.24354047684273, + "tokens_seen": 261226496 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007441043129388166, + "loss": 1.7705, + "theoretical_loss": 4.243413620877732, + "tokens_seen": 261292032 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007440882647943832, + "loss": 1.6035, + "theoretical_loss": 4.243286805632467, + "tokens_seen": 261357568 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007440722166499499, + "loss": 1.7168, + "theoretical_loss": 4.243160031083658, + "tokens_seen": 261423104 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007440561685055166, + "loss": 1.7985, + "theoretical_loss": 4.24303329720805, + "tokens_seen": 261488640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007440401203610833, + "loss": 1.8321, + "theoretical_loss": 4.242906603982402, + "tokens_seen": 261554176 + }, + { + "epoch": 0.08, + "learning_rate": 0.00074402407221665, + "loss": 1.7769, + "theoretical_loss": 4.242779951383497, + "tokens_seen": 261619712 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007440080240722167, + "loss": 1.7236, + "theoretical_loss": 4.242653339388135, + "tokens_seen": 261685248 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007439919759277834, + "loss": 1.7097, + "theoretical_loss": 4.242526767973134, + "tokens_seen": 261750784 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007439759277833501, + "loss": 1.7807, + "theoretical_loss": 4.242400237115333, + "tokens_seen": 261816320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007439598796389168, + "loss": 1.6996, + "theoretical_loss": 4.242273746791589, + "tokens_seen": 261881856 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007439438314944835, + "loss": 1.6509, + "theoretical_loss": 4.242147296978778, + "tokens_seen": 261947392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007439277833500501, + "loss": 1.5987, + "theoretical_loss": 4.242020887653795, + "tokens_seen": 262012928 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007439117352056169, + "loss": 1.5883, + "theoretical_loss": 4.241894518793555, + "tokens_seen": 262078464 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.9959133267402649, + "objective/train/avg_token_score": 0.004086651839315891, + "objective/train/docs_used": 101934, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.844117283821106, + "objective/train/original_loss": 1.8505632877349854, + "objective/train/theoretical_loss": 4.24176819037499, + "objective/train/tokens_used": 282604000, + "objective/train/unlikelihood_loss": 0.010299565270543098, + "theoretical_loss": 4.24176819037499, + "tokens_seen": 262144000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007438956870611835, + "loss": 1.7656, + "theoretical_loss": 4.24176819037499, + "tokens_seen": 262144000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007438796389167503, + "loss": 1.7088, + "theoretical_loss": 4.2416419023750525, + "tokens_seen": 262209536 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007438635907723169, + "loss": 1.6223, + "theoretical_loss": 4.241515654770713, + "tokens_seen": 262275072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007438475426278837, + "loss": 1.6989, + "theoretical_loss": 4.241389447538961, + "tokens_seen": 262340608 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007438314944834503, + "loss": 1.7144, + "theoretical_loss": 4.241263280656806, + "tokens_seen": 262406144 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007438154463390171, + "loss": 1.7134, + "theoretical_loss": 4.241137154101275, + "tokens_seen": 262471680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007437993981945837, + "loss": 1.8034, + "theoretical_loss": 4.2410110678494135, + "tokens_seen": 262537216 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007437833500501505, + "loss": 1.6263, + "theoretical_loss": 4.240885021878286, + "tokens_seen": 262602752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007437673019057171, + "loss": 1.8085, + "theoretical_loss": 4.240759016164978, + "tokens_seen": 262668288 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007437512537612839, + "loss": 1.7133, + "theoretical_loss": 4.240633050686592, + "tokens_seen": 262733824 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007437352056168505, + "loss": 1.6733, + "theoretical_loss": 4.2405071254202475, + "tokens_seen": 262799360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007437191574724173, + "loss": 1.6894, + "theoretical_loss": 4.240381240343085, + "tokens_seen": 262864896 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007437031093279839, + "loss": 1.5885, + "theoretical_loss": 4.240255395432264, + "tokens_seen": 262930432 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007436870611835507, + "loss": 1.5901, + "theoretical_loss": 4.240129590664961, + "tokens_seen": 262995968 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007436710130391173, + "loss": 1.6911, + "theoretical_loss": 4.240003826018373, + "tokens_seen": 263061504 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007436549648946841, + "loss": 1.6499, + "theoretical_loss": 4.239878101469713, + "tokens_seen": 263127040 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007436389167502507, + "loss": 1.6993, + "theoretical_loss": 4.239752416996215, + "tokens_seen": 263192576 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007436228686058175, + "loss": 1.6095, + "theoretical_loss": 4.239626772575131, + "tokens_seen": 263258112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007436068204613841, + "loss": 1.8097, + "theoretical_loss": 4.239501168183732, + "tokens_seen": 263323648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007435907723169509, + "loss": 1.6452, + "theoretical_loss": 4.239375603799305, + "tokens_seen": 263389184 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007435747241725175, + "loss": 1.7167, + "theoretical_loss": 4.2392500793991585, + "tokens_seen": 263454720 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007435586760280843, + "loss": 1.7911, + "theoretical_loss": 4.239124594960619, + "tokens_seen": 263520256 + }, + { + "epoch": 0.08, + "learning_rate": 0.000743542627883651, + "loss": 1.7553, + "theoretical_loss": 4.23899915046103, + "tokens_seen": 263585792 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007435265797392177, + "loss": 1.7169, + "theoretical_loss": 4.238873745877756, + "tokens_seen": 263651328 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007435105315947844, + "loss": 1.7147, + "theoretical_loss": 4.238748381188176, + "tokens_seen": 263716864 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007434944834503511, + "loss": 1.7287, + "theoretical_loss": 4.238623056369692, + "tokens_seen": 263782400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007434784353059178, + "loss": 1.757, + "theoretical_loss": 4.238497771399721, + "tokens_seen": 263847936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007434623871614845, + "loss": 1.6899, + "theoretical_loss": 4.2383725262556995, + "tokens_seen": 263913472 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007434463390170512, + "loss": 1.7292, + "theoretical_loss": 4.238247320915083, + "tokens_seen": 263979008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007434302908726179, + "loss": 1.6601, + "theoretical_loss": 4.238122155355345, + "tokens_seen": 264044544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007434142427281846, + "loss": 1.7898, + "theoretical_loss": 4.237997029553977, + "tokens_seen": 264110080 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007433981945837513, + "loss": 1.7964, + "theoretical_loss": 4.237871943488488, + "tokens_seen": 264175616 + }, + { + "epoch": 0.08, + "learning_rate": 0.000743382146439318, + "loss": 1.6927, + "theoretical_loss": 4.237746897136408, + "tokens_seen": 264241152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007433660982948847, + "loss": 1.7225, + "theoretical_loss": 4.237621890475284, + "tokens_seen": 264306688 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007433500501504514, + "loss": 1.6778, + "theoretical_loss": 4.237496923482678, + "tokens_seen": 264372224 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007433340020060181, + "loss": 1.6421, + "theoretical_loss": 4.237371996136175, + "tokens_seen": 264437760 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007433179538615848, + "loss": 1.6932, + "theoretical_loss": 4.237247108413376, + "tokens_seen": 264503296 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007433019057171515, + "loss": 1.552, + "theoretical_loss": 4.237122260291901, + "tokens_seen": 264568832 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007432858575727182, + "loss": 1.7012, + "theoretical_loss": 4.236997451749387, + "tokens_seen": 264634368 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007432698094282849, + "loss": 1.603, + "theoretical_loss": 4.236872682763489, + "tokens_seen": 264699904 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007432537612838516, + "loss": 1.6979, + "theoretical_loss": 4.236747953311884, + "tokens_seen": 264765440 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007432377131394183, + "loss": 1.7527, + "theoretical_loss": 4.236623263372261, + "tokens_seen": 264830976 + }, + { + "epoch": 0.08, + "learning_rate": 0.000743221664994985, + "loss": 1.7791, + "theoretical_loss": 4.236498612922331, + "tokens_seen": 264896512 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007432056168505517, + "loss": 1.7403, + "theoretical_loss": 4.236374001939822, + "tokens_seen": 264962048 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007431895687061184, + "loss": 1.8027, + "theoretical_loss": 4.2362494304024825, + "tokens_seen": 265027584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007431735205616851, + "loss": 1.7812, + "theoretical_loss": 4.236124898288074, + "tokens_seen": 265093120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007431574724172518, + "loss": 1.6923, + "theoretical_loss": 4.236000405574382, + "tokens_seen": 265158656 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007431414242728185, + "loss": 1.7207, + "theoretical_loss": 4.235875952239204, + "tokens_seen": 265224192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007431253761283852, + "loss": 1.645, + "theoretical_loss": 4.235751538260359, + "tokens_seen": 265289728 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007431093279839519, + "loss": 1.76, + "theoretical_loss": 4.2356271636156855, + "tokens_seen": 265355264 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.9950775504112244, + "objective/train/avg_token_score": 0.004922426305711269, + "objective/train/docs_used": 103026, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5698250532150269, + "objective/train/original_loss": 1.577757716178894, + "objective/train/theoretical_loss": 4.235502828283035, + "objective/train/tokens_used": 285880800, + "objective/train/unlikelihood_loss": 0.012746967375278473, + "theoretical_loss": 4.235502828283035, + "tokens_seen": 265420800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007430932798395186, + "loss": 1.7214, + "theoretical_loss": 4.235502828283035, + "tokens_seen": 265420800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007430772316950853, + "loss": 1.8007, + "theoretical_loss": 4.235378532240283, + "tokens_seen": 265486336 + }, + { + "epoch": 0.08, + "learning_rate": 0.000743061183550652, + "loss": 1.8168, + "theoretical_loss": 4.235254275465317, + "tokens_seen": 265551872 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007430451354062187, + "loss": 1.6161, + "theoretical_loss": 4.235130057936045, + "tokens_seen": 265617408 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007430290872617854, + "loss": 1.6177, + "theoretical_loss": 4.235005879630395, + "tokens_seen": 265682944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007430130391173521, + "loss": 1.7165, + "theoretical_loss": 4.23488174052631, + "tokens_seen": 265748480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007429969909729188, + "loss": 1.6735, + "theoretical_loss": 4.234757640601751, + "tokens_seen": 265814016 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007429809428284855, + "loss": 1.6986, + "theoretical_loss": 4.234633579834698, + "tokens_seen": 265879552 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007429648946840523, + "loss": 1.6532, + "theoretical_loss": 4.234509558203149, + "tokens_seen": 265945088 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007429488465396189, + "loss": 1.7512, + "theoretical_loss": 4.234385575685116, + "tokens_seen": 266010624 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007429327983951857, + "loss": 1.6823, + "theoretical_loss": 4.234261632258636, + "tokens_seen": 266076160 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007429167502507523, + "loss": 1.7079, + "theoretical_loss": 4.234137727901757, + "tokens_seen": 266141696 + }, + { + "epoch": 0.08, + "learning_rate": 0.000742900702106319, + "loss": 1.8109, + "theoretical_loss": 4.234013862592548, + "tokens_seen": 266207232 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007428846539618857, + "loss": 1.7044, + "theoretical_loss": 4.233890036309095, + "tokens_seen": 266272768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007428686058174525, + "loss": 1.5844, + "theoretical_loss": 4.233766249029501, + "tokens_seen": 266338304 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007428525576730191, + "loss": 1.7244, + "theoretical_loss": 4.233642500731889, + "tokens_seen": 266403840 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007428365095285859, + "loss": 1.8177, + "theoretical_loss": 4.233518791394396, + "tokens_seen": 266469376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007428204613841525, + "loss": 1.7833, + "theoretical_loss": 4.233395120995181, + "tokens_seen": 266534912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007428044132397193, + "loss": 1.7393, + "theoretical_loss": 4.233271489512416, + "tokens_seen": 266600448 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007427883650952859, + "loss": 1.7159, + "theoretical_loss": 4.233147896924294, + "tokens_seen": 266665984 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007427723169508527, + "loss": 1.783, + "theoretical_loss": 4.233024343209023, + "tokens_seen": 266731520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007427562688064193, + "loss": 1.7245, + "theoretical_loss": 4.232900828344833, + "tokens_seen": 266797056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007427402206619861, + "loss": 1.6472, + "theoretical_loss": 4.232777352309966, + "tokens_seen": 266862592 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007427241725175527, + "loss": 1.7545, + "theoretical_loss": 4.232653915082684, + "tokens_seen": 266928128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007427081243731195, + "loss": 1.6979, + "theoretical_loss": 4.232530516641267, + "tokens_seen": 266993664 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007426920762286861, + "loss": 1.6861, + "theoretical_loss": 4.232407156964014, + "tokens_seen": 267059200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007426760280842528, + "loss": 1.6705, + "theoretical_loss": 4.232283836029237, + "tokens_seen": 267124736 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007426599799398195, + "loss": 1.6525, + "theoretical_loss": 4.232160553815269, + "tokens_seen": 267190272 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007426439317953862, + "loss": 1.7308, + "theoretical_loss": 4.232037310300458, + "tokens_seen": 267255808 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007426278836509529, + "loss": 1.6689, + "theoretical_loss": 4.231914105463173, + "tokens_seen": 267321344 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007426118355065196, + "loss": 1.7994, + "theoretical_loss": 4.231790939281797, + "tokens_seen": 267386880 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007425957873620863, + "loss": 1.7506, + "theoretical_loss": 4.231667811734732, + "tokens_seen": 267452416 + }, + { + "epoch": 0.08, + "learning_rate": 0.000742579739217653, + "loss": 1.7976, + "theoretical_loss": 4.231544722800395, + "tokens_seen": 267517952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007425636910732197, + "loss": 1.6534, + "theoretical_loss": 4.231421672457225, + "tokens_seen": 267583488 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007425476429287864, + "loss": 1.8229, + "theoretical_loss": 4.231298660683675, + "tokens_seen": 267649024 + }, + { + "epoch": 0.08, + "learning_rate": 0.000742531594784353, + "loss": 1.6398, + "theoretical_loss": 4.231175687458214, + "tokens_seen": 267714560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007425155466399198, + "loss": 1.7233, + "theoretical_loss": 4.231052752759332, + "tokens_seen": 267780096 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007424994984954864, + "loss": 1.6159, + "theoretical_loss": 4.230929856565535, + "tokens_seen": 267845632 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007424834503510532, + "loss": 1.6678, + "theoretical_loss": 4.230806998855343, + "tokens_seen": 267911168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007424674022066198, + "loss": 1.7349, + "theoretical_loss": 4.230684179607297, + "tokens_seen": 267976704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007424513540621866, + "loss": 1.6972, + "theoretical_loss": 4.2305613987999555, + "tokens_seen": 268042240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007424353059177532, + "loss": 1.6641, + "theoretical_loss": 4.230438656411891, + "tokens_seen": 268107776 + }, + { + "epoch": 0.08, + "learning_rate": 0.00074241925777332, + "loss": 1.6813, + "theoretical_loss": 4.230315952421697, + "tokens_seen": 268173312 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007424032096288866, + "loss": 1.6953, + "theoretical_loss": 4.230193286807981, + "tokens_seen": 268238848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007423871614844534, + "loss": 1.7293, + "theoretical_loss": 4.230070659549368, + "tokens_seen": 268304384 + }, + { + "epoch": 0.08, + "learning_rate": 0.00074237111334002, + "loss": 1.7146, + "theoretical_loss": 4.229948070624502, + "tokens_seen": 268369920 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007423550651955868, + "loss": 1.6868, + "theoretical_loss": 4.229825520012042, + "tokens_seen": 268435456 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007423390170511535, + "loss": 1.7934, + "theoretical_loss": 4.2297030076906665, + "tokens_seen": 268500992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007423229689067202, + "loss": 1.7561, + "theoretical_loss": 4.229580533639068, + "tokens_seen": 268566528 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007423069207622869, + "loss": 1.6827, + "theoretical_loss": 4.229458097835959, + "tokens_seen": 268632064 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.9923770427703857, + "objective/train/avg_token_score": 0.007622957695275545, + "objective/train/docs_used": 104173, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5681957006454468, + "objective/train/original_loss": 1.5814659595489502, + "objective/train/theoretical_loss": 4.2293357002600676, + "objective/train/tokens_used": 289157600, + "objective/train/unlikelihood_loss": 0.013908402994275093, + "theoretical_loss": 4.2293357002600676, + "tokens_seen": 268697600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007422908726178536, + "loss": 1.5858, + "theoretical_loss": 4.2293357002600676, + "tokens_seen": 268697600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007422748244734203, + "loss": 1.6768, + "theoretical_loss": 4.229213340890139, + "tokens_seen": 268763136 + }, + { + "epoch": 0.08, + "learning_rate": 0.000742258776328987, + "loss": 1.731, + "theoretical_loss": 4.229091019704935, + "tokens_seen": 268828672 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007422427281845537, + "loss": 1.7924, + "theoretical_loss": 4.2289687366832345, + "tokens_seen": 268894208 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007422266800401204, + "loss": 1.7336, + "theoretical_loss": 4.228846491803835, + "tokens_seen": 268959744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007422106318956871, + "loss": 1.7442, + "theoretical_loss": 4.228724285045549, + "tokens_seen": 269025280 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007421945837512538, + "loss": 1.656, + "theoretical_loss": 4.228602116387206, + "tokens_seen": 269090816 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007421785356068205, + "loss": 1.71, + "theoretical_loss": 4.228479985807653, + "tokens_seen": 269156352 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007421624874623872, + "loss": 1.5211, + "theoretical_loss": 4.228357893285755, + "tokens_seen": 269221888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007421464393179539, + "loss": 1.7, + "theoretical_loss": 4.2282358388003916, + "tokens_seen": 269287424 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007421303911735206, + "loss": 1.6284, + "theoretical_loss": 4.228113822330461, + "tokens_seen": 269352960 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007421143430290873, + "loss": 1.5855, + "theoretical_loss": 4.227991843854878, + "tokens_seen": 269418496 + }, + { + "epoch": 0.08, + "learning_rate": 0.000742098294884654, + "loss": 1.6765, + "theoretical_loss": 4.227869903352572, + "tokens_seen": 269484032 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007420822467402207, + "loss": 1.6274, + "theoretical_loss": 4.227748000802493, + "tokens_seen": 269549568 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007420661985957874, + "loss": 1.7435, + "theoretical_loss": 4.227626136183606, + "tokens_seen": 269615104 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007420501504513541, + "loss": 1.7102, + "theoretical_loss": 4.227504309474892, + "tokens_seen": 269680640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007420341023069208, + "loss": 1.7142, + "theoretical_loss": 4.227382520655348, + "tokens_seen": 269746176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007420180541624875, + "loss": 1.7625, + "theoretical_loss": 4.227260769703991, + "tokens_seen": 269811712 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007420020060180542, + "loss": 1.5951, + "theoretical_loss": 4.227139056599851, + "tokens_seen": 269877248 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007419859578736209, + "loss": 1.6403, + "theoretical_loss": 4.2270173813219785, + "tokens_seen": 269942784 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007419699097291876, + "loss": 1.6429, + "theoretical_loss": 4.226895743849437, + "tokens_seen": 270008320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007419538615847543, + "loss": 1.7425, + "theoretical_loss": 4.226774144161309, + "tokens_seen": 270073856 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741937813440321, + "loss": 1.7235, + "theoretical_loss": 4.226652582236694, + "tokens_seen": 270139392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007419217652958877, + "loss": 1.7527, + "theoretical_loss": 4.226531058054706, + "tokens_seen": 270204928 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007419057171514544, + "loss": 1.7459, + "theoretical_loss": 4.226409571594477, + "tokens_seen": 270270464 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007418896690070211, + "loss": 1.7338, + "theoretical_loss": 4.226288122835156, + "tokens_seen": 270336000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007418736208625878, + "loss": 1.6711, + "theoretical_loss": 4.226166711755906, + "tokens_seen": 270401536 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007418575727181545, + "loss": 1.646, + "theoretical_loss": 4.2260453383359105, + "tokens_seen": 270467072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007418415245737212, + "loss": 1.8201, + "theoretical_loss": 4.2259240025543665, + "tokens_seen": 270532608 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007418254764292879, + "loss": 1.7804, + "theoretical_loss": 4.225802704390488, + "tokens_seen": 270598144 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007418094282848546, + "loss": 1.5929, + "theoretical_loss": 4.2256814438235075, + "tokens_seen": 270663680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007417933801404213, + "loss": 1.6235, + "theoretical_loss": 4.225560220832672, + "tokens_seen": 270729216 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741777331995988, + "loss": 1.5941, + "theoretical_loss": 4.225439035397246, + "tokens_seen": 270794752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007417612838515547, + "loss": 1.6917, + "theoretical_loss": 4.225317887496508, + "tokens_seen": 270860288 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007417452357071214, + "loss": 1.6194, + "theoretical_loss": 4.2251967771097565, + "tokens_seen": 270925824 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007417291875626881, + "loss": 1.7763, + "theoretical_loss": 4.225075704216305, + "tokens_seen": 270991360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007417131394182548, + "loss": 1.616, + "theoretical_loss": 4.224954668795483, + "tokens_seen": 271056896 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007416970912738215, + "loss": 1.7015, + "theoretical_loss": 4.224833670826636, + "tokens_seen": 271122432 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007416810431293882, + "loss": 1.7482, + "theoretical_loss": 4.224712710289127, + "tokens_seen": 271187968 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741664994984955, + "loss": 1.7837, + "theoretical_loss": 4.2245917871623355, + "tokens_seen": 271253504 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007416489468405216, + "loss": 1.6414, + "theoretical_loss": 4.224470901425655, + "tokens_seen": 271319040 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007416328986960883, + "loss": 1.8345, + "theoretical_loss": 4.224350053058499, + "tokens_seen": 271384576 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741616850551655, + "loss": 1.758, + "theoretical_loss": 4.224229242040295, + "tokens_seen": 271450112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007416008024072218, + "loss": 1.6907, + "theoretical_loss": 4.224108468350485, + "tokens_seen": 271515648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007415847542627884, + "loss": 1.8171, + "theoretical_loss": 4.223987731968532, + "tokens_seen": 271581184 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007415687061183552, + "loss": 1.8247, + "theoretical_loss": 4.223867032873912, + "tokens_seen": 271646720 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007415526579739218, + "loss": 1.7332, + "theoretical_loss": 4.223746371046116, + "tokens_seen": 271712256 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007415366098294886, + "loss": 1.649, + "theoretical_loss": 4.223625746464656, + "tokens_seen": 271777792 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007415205616850552, + "loss": 1.8229, + "theoretical_loss": 4.223505159109054, + "tokens_seen": 271843328 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741504513540622, + "loss": 1.6504, + "theoretical_loss": 4.223384608958855, + "tokens_seen": 271908864 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.9953818321228027, + "objective/train/avg_token_score": 0.004618153907358646, + "objective/train/docs_used": 105443, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.65164315700531, + "objective/train/original_loss": 1.6592209339141846, + "objective/train/theoretical_loss": 4.223264095993615, + "objective/train/tokens_used": 292434400, + "objective/train/unlikelihood_loss": 0.010537678375840187, + "theoretical_loss": 4.223264095993615, + "tokens_seen": 271974400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007414884653961886, + "loss": 1.7169, + "theoretical_loss": 4.223264095993615, + "tokens_seen": 271974400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007414724172517554, + "loss": 1.5846, + "theoretical_loss": 4.223143620192907, + "tokens_seen": 272039936 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741456369107322, + "loss": 1.6768, + "theoretical_loss": 4.223023181536323, + "tokens_seen": 272105472 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007414403209628888, + "loss": 1.6579, + "theoretical_loss": 4.2229027800034675, + "tokens_seen": 272171008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007414242728184554, + "loss": 1.636, + "theoretical_loss": 4.222782415573962, + "tokens_seen": 272236544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007414082246740222, + "loss": 1.6655, + "theoretical_loss": 4.222662088227447, + "tokens_seen": 272302080 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007413921765295888, + "loss": 1.8074, + "theoretical_loss": 4.222541797943576, + "tokens_seen": 272367616 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007413761283851556, + "loss": 1.7076, + "theoretical_loss": 4.222421544702018, + "tokens_seen": 272433152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007413600802407222, + "loss": 1.682, + "theoretical_loss": 4.2223013284824615, + "tokens_seen": 272498688 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741344032096289, + "loss": 1.6367, + "theoretical_loss": 4.222181149264608, + "tokens_seen": 272564224 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007413279839518556, + "loss": 1.6686, + "theoretical_loss": 4.222061007028175, + "tokens_seen": 272629760 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007413119358074224, + "loss": 1.6947, + "theoretical_loss": 4.221940901752899, + "tokens_seen": 272695296 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741295887662989, + "loss": 1.7546, + "theoretical_loss": 4.221820833418529, + "tokens_seen": 272760832 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007412798395185557, + "loss": 1.7051, + "theoretical_loss": 4.221700802004832, + "tokens_seen": 272826368 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007412637913741224, + "loss": 1.8058, + "theoretical_loss": 4.2215808074915895, + "tokens_seen": 272891904 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007412477432296891, + "loss": 1.7982, + "theoretical_loss": 4.221460849858601, + "tokens_seen": 272957440 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007412316950852558, + "loss": 1.718, + "theoretical_loss": 4.221340929085679, + "tokens_seen": 273022976 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007412156469408225, + "loss": 1.6095, + "theoretical_loss": 4.221221045152655, + "tokens_seen": 273088512 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007411995987963892, + "loss": 1.8157, + "theoretical_loss": 4.221101198039375, + "tokens_seen": 273154048 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007411835506519559, + "loss": 1.7946, + "theoretical_loss": 4.2209813877257, + "tokens_seen": 273219584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007411675025075226, + "loss": 1.6397, + "theoretical_loss": 4.220861614191508, + "tokens_seen": 273285120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007411514543630893, + "loss": 1.658, + "theoretical_loss": 4.220741877416691, + "tokens_seen": 273350656 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741135406218656, + "loss": 1.6263, + "theoretical_loss": 4.22062217738116, + "tokens_seen": 273416192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007411193580742227, + "loss": 1.7019, + "theoretical_loss": 4.22050251406484, + "tokens_seen": 273481728 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007411033099297894, + "loss": 1.7972, + "theoretical_loss": 4.220382887447671, + "tokens_seen": 273547264 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007410872617853561, + "loss": 1.7763, + "theoretical_loss": 4.22026329750961, + "tokens_seen": 273612800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007410712136409228, + "loss": 1.6669, + "theoretical_loss": 4.220143744230629, + "tokens_seen": 273678336 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007410551654964895, + "loss": 1.7063, + "theoretical_loss": 4.220024227590717, + "tokens_seen": 273743872 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007410391173520562, + "loss": 1.6753, + "theoretical_loss": 4.219904747569877, + "tokens_seen": 273809408 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007410230692076229, + "loss": 1.7729, + "theoretical_loss": 4.219785304148129, + "tokens_seen": 273874944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007410070210631896, + "loss": 1.7853, + "theoretical_loss": 4.219665897305506, + "tokens_seen": 273940480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007409909729187563, + "loss": 1.7074, + "theoretical_loss": 4.219546527022062, + "tokens_seen": 274006016 + }, + { + "epoch": 0.08, + "learning_rate": 0.000740974924774323, + "loss": 1.6937, + "theoretical_loss": 4.219427193277861, + "tokens_seen": 274071552 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007409588766298897, + "loss": 1.6927, + "theoretical_loss": 4.219307896052987, + "tokens_seen": 274137088 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007409428284854564, + "loss": 1.7776, + "theoretical_loss": 4.219188635327535, + "tokens_seen": 274202624 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007409267803410231, + "loss": 1.7093, + "theoretical_loss": 4.219069411081622, + "tokens_seen": 274268160 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007409107321965898, + "loss": 1.5864, + "theoretical_loss": 4.218950223295374, + "tokens_seen": 274333696 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007408946840521565, + "loss": 1.617, + "theoretical_loss": 4.218831071948936, + "tokens_seen": 274399232 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007408786359077232, + "loss": 1.7415, + "theoretical_loss": 4.218711957022469, + "tokens_seen": 274464768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007408625877632899, + "loss": 1.5896, + "theoretical_loss": 4.218592878496148, + "tokens_seen": 274530304 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007408465396188566, + "loss": 1.6989, + "theoretical_loss": 4.218473836350164, + "tokens_seen": 274595840 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007408304914744233, + "loss": 1.6196, + "theoretical_loss": 4.2183548305647225, + "tokens_seen": 274661376 + }, + { + "epoch": 0.08, + "learning_rate": 0.00074081444332999, + "loss": 1.7062, + "theoretical_loss": 4.218235861120047, + "tokens_seen": 274726912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007407983951855567, + "loss": 1.803, + "theoretical_loss": 4.218116927996375, + "tokens_seen": 274792448 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007407823470411234, + "loss": 1.6387, + "theoretical_loss": 4.217998031173959, + "tokens_seen": 274857984 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007407662988966901, + "loss": 1.6956, + "theoretical_loss": 4.217879170633067, + "tokens_seen": 274923520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007407502507522568, + "loss": 1.726, + "theoretical_loss": 4.217760346353984, + "tokens_seen": 274989056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007407342026078235, + "loss": 1.7503, + "theoretical_loss": 4.217641558317008, + "tokens_seen": 275054592 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007407181544633902, + "loss": 1.589, + "theoretical_loss": 4.217522806502455, + "tokens_seen": 275120128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007407021063189569, + "loss": 1.6258, + "theoretical_loss": 4.217404090890653, + "tokens_seen": 275185664 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.9921970367431641, + "objective/train/avg_token_score": 0.007803000044077635, + "objective/train/docs_used": 106703, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.630520224571228, + "objective/train/original_loss": 1.644268274307251, + "objective/train/theoretical_loss": 4.217285411461949, + "objective/train/tokens_used": 295711200, + "objective/train/unlikelihood_loss": 0.01716526970267296, + "theoretical_loss": 4.217285411461949, + "tokens_seen": 275251200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007406860581745236, + "loss": 1.6183, + "theoretical_loss": 4.217285411461949, + "tokens_seen": 275251200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007406700100300903, + "loss": 1.6592, + "theoretical_loss": 4.217166768196704, + "tokens_seen": 275316736 + }, + { + "epoch": 0.08, + "learning_rate": 0.000740653961885657, + "loss": 1.758, + "theoretical_loss": 4.217048161075294, + "tokens_seen": 275382272 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007406379137412237, + "loss": 1.624, + "theoretical_loss": 4.2169295900781085, + "tokens_seen": 275447808 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007406218655967904, + "loss": 1.7621, + "theoretical_loss": 4.216811055185556, + "tokens_seen": 275513344 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007406058174523571, + "loss": 1.7228, + "theoretical_loss": 4.216692556378058, + "tokens_seen": 275578880 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007405897693079238, + "loss": 1.6843, + "theoretical_loss": 4.216574093636053, + "tokens_seen": 275644416 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007405737211634905, + "loss": 1.7134, + "theoretical_loss": 4.216455666939991, + "tokens_seen": 275709952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007405576730190572, + "loss": 1.7134, + "theoretical_loss": 4.216337276270341, + "tokens_seen": 275775488 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007405416248746239, + "loss": 1.5855, + "theoretical_loss": 4.216218921607586, + "tokens_seen": 275841024 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007405255767301906, + "loss": 1.786, + "theoretical_loss": 4.216100602932224, + "tokens_seen": 275906560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007405095285857573, + "loss": 1.7365, + "theoretical_loss": 4.21598232022477, + "tokens_seen": 275972096 + }, + { + "epoch": 0.08, + "learning_rate": 0.000740493480441324, + "loss": 1.8074, + "theoretical_loss": 4.215864073465749, + "tokens_seen": 276037632 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007404774322968907, + "loss": 1.7532, + "theoretical_loss": 4.215745862635708, + "tokens_seen": 276103168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007404613841524574, + "loss": 1.6894, + "theoretical_loss": 4.2156276877152035, + "tokens_seen": 276168704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007404453360080241, + "loss": 1.8588, + "theoretical_loss": 4.21550954868481, + "tokens_seen": 276234240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007404292878635908, + "loss": 1.6965, + "theoretical_loss": 4.2153914455251185, + "tokens_seen": 276299776 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007404132397191575, + "loss": 1.7794, + "theoretical_loss": 4.215273378216731, + "tokens_seen": 276365312 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007403971915747242, + "loss": 1.7505, + "theoretical_loss": 4.215155346740268, + "tokens_seen": 276430848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007403811434302909, + "loss": 1.8021, + "theoretical_loss": 4.215037351076363, + "tokens_seen": 276496384 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007403650952858576, + "loss": 1.5533, + "theoretical_loss": 4.214919391205665, + "tokens_seen": 276561920 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007403490471414243, + "loss": 1.7341, + "theoretical_loss": 4.21480146710884, + "tokens_seen": 276627456 + }, + { + "epoch": 0.08, + "learning_rate": 0.000740332998996991, + "loss": 1.7859, + "theoretical_loss": 4.214683578766566, + "tokens_seen": 276692992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007403169508525577, + "loss": 1.6928, + "theoretical_loss": 4.214565726159539, + "tokens_seen": 276758528 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007403009027081244, + "loss": 1.6914, + "theoretical_loss": 4.214447909268467, + "tokens_seen": 276824064 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007402848545636911, + "loss": 1.7631, + "theoretical_loss": 4.214330128074074, + "tokens_seen": 276889600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007402688064192578, + "loss": 1.6087, + "theoretical_loss": 4.214212382557101, + "tokens_seen": 276955136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007402527582748245, + "loss": 1.674, + "theoretical_loss": 4.214094672698302, + "tokens_seen": 277020672 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007402367101303912, + "loss": 1.6281, + "theoretical_loss": 4.213976998478445, + "tokens_seen": 277086208 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007402206619859579, + "loss": 1.7583, + "theoretical_loss": 4.213859359878316, + "tokens_seen": 277151744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007402046138415247, + "loss": 1.7528, + "theoretical_loss": 4.213741756878713, + "tokens_seen": 277217280 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007401885656970913, + "loss": 1.656, + "theoretical_loss": 4.213624189460449, + "tokens_seen": 277282816 + }, + { + "epoch": 0.08, + "learning_rate": 0.000740172517552658, + "loss": 1.6746, + "theoretical_loss": 4.213506657604356, + "tokens_seen": 277348352 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007401564694082247, + "loss": 1.6806, + "theoretical_loss": 4.213389161291273, + "tokens_seen": 277413888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007401404212637915, + "loss": 1.6625, + "theoretical_loss": 4.213271700502062, + "tokens_seen": 277479424 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007401243731193581, + "loss": 1.7902, + "theoretical_loss": 4.213154275217597, + "tokens_seen": 277544960 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007401083249749249, + "loss": 1.7582, + "theoretical_loss": 4.213036885418763, + "tokens_seen": 277610496 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007400922768304915, + "loss": 1.6914, + "theoretical_loss": 4.212919531086466, + "tokens_seen": 277676032 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007400762286860583, + "loss": 1.7123, + "theoretical_loss": 4.2128022122016215, + "tokens_seen": 277741568 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007400601805416249, + "loss": 1.722, + "theoretical_loss": 4.212684928745163, + "tokens_seen": 277807104 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007400441323971917, + "loss": 1.7611, + "theoretical_loss": 4.212567680698038, + "tokens_seen": 277872640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007400280842527583, + "loss": 1.717, + "theoretical_loss": 4.212450468041208, + "tokens_seen": 277938176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007400120361083251, + "loss": 1.8006, + "theoretical_loss": 4.21233329075565, + "tokens_seen": 278003712 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007399959879638917, + "loss": 1.6698, + "theoretical_loss": 4.212216148822355, + "tokens_seen": 278069248 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007399799398194585, + "loss": 1.6619, + "theoretical_loss": 4.212099042222331, + "tokens_seen": 278134784 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007399638916750251, + "loss": 1.8305, + "theoretical_loss": 4.211981970936595, + "tokens_seen": 278200320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007399478435305919, + "loss": 1.6663, + "theoretical_loss": 4.2118649349461865, + "tokens_seen": 278265856 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007399317953861585, + "loss": 1.7736, + "theoretical_loss": 4.211747934232153, + "tokens_seen": 278331392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007399157472417253, + "loss": 1.7548, + "theoretical_loss": 4.211630968775561, + "tokens_seen": 278396928 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007398996990972919, + "loss": 1.7065, + "theoretical_loss": 4.211514038557488, + "tokens_seen": 278462464 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.9929666519165039, + "objective/train/avg_token_score": 0.007033352740108967, + "objective/train/docs_used": 107929, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.7224996089935303, + "objective/train/original_loss": 1.735703706741333, + "objective/train/theoretical_loss": 4.211397143559029, + "objective/train/tokens_used": 298988000, + "objective/train/unlikelihood_loss": 0.01389598660171032, + "theoretical_loss": 4.211397143559029, + "tokens_seen": 278528000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007398836509528586, + "loss": 1.7296, + "theoretical_loss": 4.211397143559029, + "tokens_seen": 278528000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007398676028084253, + "loss": 1.7271, + "theoretical_loss": 4.211280283761294, + "tokens_seen": 278593536 + }, + { + "epoch": 0.08, + "learning_rate": 0.000739851554663992, + "loss": 1.7526, + "theoretical_loss": 4.211163459145403, + "tokens_seen": 278659072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007398355065195587, + "loss": 1.767, + "theoretical_loss": 4.211046669692496, + "tokens_seen": 278724608 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007398194583751254, + "loss": 1.7714, + "theoretical_loss": 4.2109299153837245, + "tokens_seen": 278790144 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007398034102306921, + "loss": 1.8037, + "theoretical_loss": 4.210813196200255, + "tokens_seen": 278855680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007397873620862588, + "loss": 1.6539, + "theoretical_loss": 4.21069651212327, + "tokens_seen": 278921216 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007397713139418255, + "loss": 1.8859, + "theoretical_loss": 4.2105798631339635, + "tokens_seen": 278986752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007397552657973922, + "loss": 1.716, + "theoretical_loss": 4.2104632492135465, + "tokens_seen": 279052288 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007397392176529589, + "loss": 1.6487, + "theoretical_loss": 4.210346670343244, + "tokens_seen": 279117824 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007397231695085256, + "loss": 1.72, + "theoretical_loss": 4.210230126504296, + "tokens_seen": 279183360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007397071213640923, + "loss": 1.7737, + "theoretical_loss": 4.210113617677955, + "tokens_seen": 279248896 + }, + { + "epoch": 0.08, + "learning_rate": 0.000739691073219659, + "loss": 1.876, + "theoretical_loss": 4.20999714384549, + "tokens_seen": 279314432 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007396750250752257, + "loss": 1.8924, + "theoretical_loss": 4.209880704988181, + "tokens_seen": 279379968 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007396589769307924, + "loss": 1.7939, + "theoretical_loss": 4.209764301087327, + "tokens_seen": 279445504 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007396429287863591, + "loss": 1.7545, + "theoretical_loss": 4.20964793212424, + "tokens_seen": 279511040 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007396268806419258, + "loss": 1.7449, + "theoretical_loss": 4.209531598080245, + "tokens_seen": 279576576 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007396108324974925, + "loss": 1.8539, + "theoretical_loss": 4.20941529893668, + "tokens_seen": 279642112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007395947843530592, + "loss": 1.7373, + "theoretical_loss": 4.209299034674902, + "tokens_seen": 279707648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007395787362086259, + "loss": 1.9158, + "theoretical_loss": 4.209182805276278, + "tokens_seen": 279773184 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007395626880641926, + "loss": 1.8409, + "theoretical_loss": 4.209066610722192, + "tokens_seen": 279838720 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007395466399197593, + "loss": 2.1326, + "theoretical_loss": 4.208950450994041, + "tokens_seen": 279904256 + }, + { + "epoch": 0.08, + "learning_rate": 0.000739530591775326, + "loss": 1.9939, + "theoretical_loss": 4.2088343260732355, + "tokens_seen": 279969792 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007395145436308927, + "loss": 1.7964, + "theoretical_loss": 4.208718235941204, + "tokens_seen": 280035328 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007394984954864594, + "loss": 1.903, + "theoretical_loss": 4.2086021805793825, + "tokens_seen": 280100864 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007394824473420261, + "loss": 1.8401, + "theoretical_loss": 4.208486159969229, + "tokens_seen": 280166400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007394663991975928, + "loss": 1.7473, + "theoretical_loss": 4.20837017409221, + "tokens_seen": 280231936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007394503510531595, + "loss": 1.7886, + "theoretical_loss": 4.20825422292981, + "tokens_seen": 280297472 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007394343029087262, + "loss": 1.9122, + "theoretical_loss": 4.208138306463524, + "tokens_seen": 280363008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007394182547642929, + "loss": 1.8554, + "theoretical_loss": 4.208022424674864, + "tokens_seen": 280428544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007394022066198596, + "loss": 1.9421, + "theoretical_loss": 4.207906577545356, + "tokens_seen": 280494080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007393861584754263, + "loss": 1.7439, + "theoretical_loss": 4.207790765056538, + "tokens_seen": 280559616 + }, + { + "epoch": 0.09, + "learning_rate": 0.000739370110330993, + "loss": 1.8375, + "theoretical_loss": 4.207674987189965, + "tokens_seen": 280625152 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007393540621865597, + "loss": 1.8, + "theoretical_loss": 4.2075592439272045, + "tokens_seen": 280690688 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007393380140421264, + "loss": 1.8377, + "theoretical_loss": 4.207443535249838, + "tokens_seen": 280756224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007393219658976931, + "loss": 1.6957, + "theoretical_loss": 4.207327861139461, + "tokens_seen": 280821760 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007393059177532598, + "loss": 1.7269, + "theoretical_loss": 4.207212221577684, + "tokens_seen": 280887296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007392898696088265, + "loss": 1.8894, + "theoretical_loss": 4.207096616546132, + "tokens_seen": 280952832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007392738214643932, + "loss": 1.7683, + "theoretical_loss": 4.206981046026442, + "tokens_seen": 281018368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007392577733199599, + "loss": 1.7632, + "theoretical_loss": 4.2068655100002665, + "tokens_seen": 281083904 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007392417251755266, + "loss": 1.7455, + "theoretical_loss": 4.206750008449272, + "tokens_seen": 281149440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007392256770310933, + "loss": 1.7545, + "theoretical_loss": 4.206634541355138, + "tokens_seen": 281214976 + }, + { + "epoch": 0.09, + "learning_rate": 0.00073920962888666, + "loss": 1.6972, + "theoretical_loss": 4.20651910869956, + "tokens_seen": 281280512 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007391935807422267, + "loss": 1.88, + "theoretical_loss": 4.2064037104642455, + "tokens_seen": 281346048 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007391775325977934, + "loss": 1.8035, + "theoretical_loss": 4.206288346630917, + "tokens_seen": 281411584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007391614844533601, + "loss": 1.691, + "theoretical_loss": 4.206173017181309, + "tokens_seen": 281477120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007391454363089268, + "loss": 1.6731, + "theoretical_loss": 4.206057722097174, + "tokens_seen": 281542656 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007391293881644935, + "loss": 1.6688, + "theoretical_loss": 4.205942461360276, + "tokens_seen": 281608192 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007391133400200602, + "loss": 1.7237, + "theoretical_loss": 4.205827234952391, + "tokens_seen": 281673728 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007390972918756269, + "loss": 1.8142, + "theoretical_loss": 4.205712042855312, + "tokens_seen": 281739264 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.994278073310852, + "objective/train/avg_token_score": 0.005721868947148323, + "objective/train/docs_used": 109222, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6545989513397217, + "objective/train/original_loss": 1.6650059223175049, + "objective/train/theoretical_loss": 4.205596885050846, + "objective/train/tokens_used": 302264800, + "objective/train/unlikelihood_loss": 0.011938439682126045, + "theoretical_loss": 4.205596885050846, + "tokens_seen": 281804800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007390812437311936, + "loss": 1.7451, + "theoretical_loss": 4.205596885050846, + "tokens_seen": 281804800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007390651955867603, + "loss": 1.7625, + "theoretical_loss": 4.205481761520811, + "tokens_seen": 281870336 + }, + { + "epoch": 0.09, + "learning_rate": 0.000739049147442327, + "loss": 1.6838, + "theoretical_loss": 4.20536667224704, + "tokens_seen": 281935872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007390330992978937, + "loss": 1.7211, + "theoretical_loss": 4.205251617211382, + "tokens_seen": 282001408 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007390170511534604, + "loss": 1.6605, + "theoretical_loss": 4.205136596395698, + "tokens_seen": 282066944 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007390010030090271, + "loss": 1.713, + "theoretical_loss": 4.205021609781861, + "tokens_seen": 282132480 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007389849548645938, + "loss": 1.7144, + "theoretical_loss": 4.204906657351763, + "tokens_seen": 282198016 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007389689067201605, + "loss": 1.7445, + "theoretical_loss": 4.2047917390873035, + "tokens_seen": 282263552 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007389528585757272, + "loss": 1.8033, + "theoretical_loss": 4.204676854970401, + "tokens_seen": 282329088 + }, + { + "epoch": 0.09, + "learning_rate": 0.000738936810431294, + "loss": 1.731, + "theoretical_loss": 4.204562004982986, + "tokens_seen": 282394624 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007389207622868606, + "loss": 1.6874, + "theoretical_loss": 4.204447189107, + "tokens_seen": 282460160 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007389047141424273, + "loss": 1.7391, + "theoretical_loss": 4.204332407324403, + "tokens_seen": 282525696 + }, + { + "epoch": 0.09, + "learning_rate": 0.000738888665997994, + "loss": 1.7413, + "theoretical_loss": 4.204217659617164, + "tokens_seen": 282591232 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007388726178535607, + "loss": 1.7963, + "theoretical_loss": 4.2041029459672705, + "tokens_seen": 282656768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007388565697091274, + "loss": 1.6605, + "theoretical_loss": 4.2039882663567205, + "tokens_seen": 282722304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007388405215646942, + "loss": 1.7686, + "theoretical_loss": 4.203873620767526, + "tokens_seen": 282787840 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007388244734202608, + "loss": 1.7182, + "theoretical_loss": 4.203759009181714, + "tokens_seen": 282853376 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007388084252758276, + "loss": 1.6912, + "theoretical_loss": 4.203644431581323, + "tokens_seen": 282918912 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007387923771313942, + "loss": 1.7263, + "theoretical_loss": 4.203529887948408, + "tokens_seen": 282984448 + }, + { + "epoch": 0.09, + "learning_rate": 0.000738776328986961, + "loss": 1.73, + "theoretical_loss": 4.203415378265034, + "tokens_seen": 283049984 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007387602808425276, + "loss": 1.7009, + "theoretical_loss": 4.203300902513284, + "tokens_seen": 283115520 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007387442326980944, + "loss": 1.8133, + "theoretical_loss": 4.203186460675251, + "tokens_seen": 283181056 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007387281845536611, + "loss": 1.6982, + "theoretical_loss": 4.203072052733043, + "tokens_seen": 283246592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007387121364092278, + "loss": 1.6929, + "theoretical_loss": 4.202957678668782, + "tokens_seen": 283312128 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007386960882647945, + "loss": 1.6801, + "theoretical_loss": 4.202843338464602, + "tokens_seen": 283377664 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007386800401203612, + "loss": 1.869, + "theoretical_loss": 4.202729032102653, + "tokens_seen": 283443200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007386639919759279, + "loss": 1.732, + "theoretical_loss": 4.202614759565096, + "tokens_seen": 283508736 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007386479438314946, + "loss": 1.782, + "theoretical_loss": 4.202500520834106, + "tokens_seen": 283574272 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007386318956870613, + "loss": 1.7203, + "theoretical_loss": 4.202386315891874, + "tokens_seen": 283639808 + }, + { + "epoch": 0.09, + "learning_rate": 0.000738615847542628, + "loss": 1.7094, + "theoretical_loss": 4.202272144720602, + "tokens_seen": 283705344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007385997993981946, + "loss": 1.725, + "theoretical_loss": 4.202158007302506, + "tokens_seen": 283770880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007385837512537614, + "loss": 1.7421, + "theoretical_loss": 4.202043903619814, + "tokens_seen": 283836416 + }, + { + "epoch": 0.09, + "learning_rate": 0.000738567703109328, + "loss": 1.6697, + "theoretical_loss": 4.201929833654772, + "tokens_seen": 283901952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007385516549648948, + "loss": 1.6625, + "theoretical_loss": 4.201815797389634, + "tokens_seen": 283967488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007385356068204614, + "loss": 1.6896, + "theoretical_loss": 4.201701794806672, + "tokens_seen": 284033024 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007385195586760282, + "loss": 1.8545, + "theoretical_loss": 4.201587825888167, + "tokens_seen": 284098560 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007385035105315948, + "loss": 1.6042, + "theoretical_loss": 4.201473890616418, + "tokens_seen": 284164096 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007384874623871615, + "loss": 1.8592, + "theoretical_loss": 4.201359988973733, + "tokens_seen": 284229632 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007384714142427282, + "loss": 1.745, + "theoretical_loss": 4.201246120942438, + "tokens_seen": 284295168 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007384553660982949, + "loss": 1.8088, + "theoretical_loss": 4.201132286504867, + "tokens_seen": 284360704 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007384393179538616, + "loss": 1.8036, + "theoretical_loss": 4.201018485643373, + "tokens_seen": 284426240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007384232698094283, + "loss": 1.7059, + "theoretical_loss": 4.200904718340317, + "tokens_seen": 284491776 + }, + { + "epoch": 0.09, + "learning_rate": 0.000738407221664995, + "loss": 1.7707, + "theoretical_loss": 4.200790984578078, + "tokens_seen": 284557312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007383911735205617, + "loss": 1.701, + "theoretical_loss": 4.200677284339045, + "tokens_seen": 284622848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007383751253761284, + "loss": 1.8115, + "theoretical_loss": 4.2005636176056225, + "tokens_seen": 284688384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007383590772316951, + "loss": 1.7483, + "theoretical_loss": 4.200449984360226, + "tokens_seen": 284753920 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007383430290872618, + "loss": 1.7643, + "theoretical_loss": 4.200336384585286, + "tokens_seen": 284819456 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007383269809428285, + "loss": 1.741, + "theoretical_loss": 4.200222818263246, + "tokens_seen": 284884992 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007383109327983952, + "loss": 1.7391, + "theoretical_loss": 4.200109285376563, + "tokens_seen": 284950528 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007382948846539619, + "loss": 1.6715, + "theoretical_loss": 4.1999957859077055, + "tokens_seen": 285016064 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.9956050515174866, + "objective/train/avg_token_score": 0.0043949587270617485, + "objective/train/docs_used": 110428, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6699156761169434, + "objective/train/original_loss": 1.6774662733078003, + "objective/train/theoretical_loss": 4.1998823198391575, + "objective/train/tokens_used": 305541600, + "objective/train/unlikelihood_loss": 0.00903965625911951, + "theoretical_loss": 4.1998823198391575, + "tokens_seen": 285081600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007382788365095286, + "loss": 1.72, + "theoretical_loss": 4.1998823198391575, + "tokens_seen": 285081600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007382627883650953, + "loss": 1.6954, + "theoretical_loss": 4.199768887153415, + "tokens_seen": 285147136 + }, + { + "epoch": 0.09, + "learning_rate": 0.000738246740220662, + "loss": 1.7931, + "theoretical_loss": 4.199655487832987, + "tokens_seen": 285212672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007382306920762287, + "loss": 1.7774, + "theoretical_loss": 4.199542121860397, + "tokens_seen": 285278208 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007382146439317954, + "loss": 1.7157, + "theoretical_loss": 4.199428789218181, + "tokens_seen": 285343744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007381985957873621, + "loss": 1.7028, + "theoretical_loss": 4.199315489888885, + "tokens_seen": 285409280 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007381825476429288, + "loss": 1.7105, + "theoretical_loss": 4.199202223855075, + "tokens_seen": 285474816 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007381664994984955, + "loss": 1.7265, + "theoretical_loss": 4.199088991099323, + "tokens_seen": 285540352 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007381504513540622, + "loss": 1.6284, + "theoretical_loss": 4.19897579160422, + "tokens_seen": 285605888 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007381344032096289, + "loss": 1.7323, + "theoretical_loss": 4.1988626253523655, + "tokens_seen": 285671424 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007381183550651956, + "loss": 1.7412, + "theoretical_loss": 4.198749492326374, + "tokens_seen": 285736960 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007381023069207623, + "loss": 1.7572, + "theoretical_loss": 4.198636392508874, + "tokens_seen": 285802496 + }, + { + "epoch": 0.09, + "learning_rate": 0.000738086258776329, + "loss": 1.6881, + "theoretical_loss": 4.198523325882505, + "tokens_seen": 285868032 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007380702106318957, + "loss": 1.6655, + "theoretical_loss": 4.198410292429922, + "tokens_seen": 285933568 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007380541624874624, + "loss": 1.6304, + "theoretical_loss": 4.19829729213379, + "tokens_seen": 285999104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007380381143430291, + "loss": 1.7001, + "theoretical_loss": 4.198184324976791, + "tokens_seen": 286064640 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007380220661985958, + "loss": 1.6947, + "theoretical_loss": 4.198071390941616, + "tokens_seen": 286130176 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007380060180541625, + "loss": 1.6434, + "theoretical_loss": 4.197958490010971, + "tokens_seen": 286195712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007379899699097292, + "loss": 1.7291, + "theoretical_loss": 4.197845622167575, + "tokens_seen": 286261248 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007379739217652959, + "loss": 1.7478, + "theoretical_loss": 4.197732787394161, + "tokens_seen": 286326784 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007379578736208626, + "loss": 1.7196, + "theoretical_loss": 4.197619985673471, + "tokens_seen": 286392320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007379418254764293, + "loss": 1.6591, + "theoretical_loss": 4.197507216988265, + "tokens_seen": 286457856 + }, + { + "epoch": 0.09, + "learning_rate": 0.000737925777331996, + "loss": 1.6475, + "theoretical_loss": 4.197394481321312, + "tokens_seen": 286523392 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007379097291875627, + "loss": 1.7824, + "theoretical_loss": 4.1972817786553955, + "tokens_seen": 286588928 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007378936810431294, + "loss": 1.6794, + "theoretical_loss": 4.197169108973313, + "tokens_seen": 286654464 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007378776328986961, + "loss": 1.7249, + "theoretical_loss": 4.197056472257874, + "tokens_seen": 286720000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007378615847542628, + "loss": 1.7829, + "theoretical_loss": 4.196943868491899, + "tokens_seen": 286785536 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007378455366098295, + "loss": 1.6961, + "theoretical_loss": 4.196831297658225, + "tokens_seen": 286851072 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007378294884653962, + "loss": 1.7219, + "theoretical_loss": 4.1967187597396975, + "tokens_seen": 286916608 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007378134403209629, + "loss": 1.6522, + "theoretical_loss": 4.196606254719179, + "tokens_seen": 286982144 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007377973921765296, + "loss": 1.747, + "theoretical_loss": 4.196493782579544, + "tokens_seen": 287047680 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007377813440320963, + "loss": 1.8036, + "theoretical_loss": 4.196381343303676, + "tokens_seen": 287113216 + }, + { + "epoch": 0.09, + "learning_rate": 0.000737765295887663, + "loss": 1.6951, + "theoretical_loss": 4.1962689368744766, + "tokens_seen": 287178752 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007377492477432297, + "loss": 1.6258, + "theoretical_loss": 4.196156563274858, + "tokens_seen": 287244288 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007377331995987964, + "loss": 1.6439, + "theoretical_loss": 4.1960442224877434, + "tokens_seen": 287309824 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007377171514543631, + "loss": 1.6594, + "theoretical_loss": 4.195931914496072, + "tokens_seen": 287375360 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007377011033099298, + "loss": 1.6419, + "theoretical_loss": 4.195819639282792, + "tokens_seen": 287440896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007376850551654965, + "loss": 1.7915, + "theoretical_loss": 4.195707396830869, + "tokens_seen": 287506432 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007376690070210632, + "loss": 1.6659, + "theoretical_loss": 4.195595187123277, + "tokens_seen": 287571968 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007376529588766299, + "loss": 1.9181, + "theoretical_loss": 4.195483010143006, + "tokens_seen": 287637504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007376369107321966, + "loss": 1.821, + "theoretical_loss": 4.1953708658730555, + "tokens_seen": 287703040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007376208625877633, + "loss": 1.7387, + "theoretical_loss": 4.195258754296441, + "tokens_seen": 287768576 + }, + { + "epoch": 0.09, + "learning_rate": 0.00073760481444333, + "loss": 1.7456, + "theoretical_loss": 4.195146675396189, + "tokens_seen": 287834112 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007375887662988967, + "loss": 1.7044, + "theoretical_loss": 4.195034629155338, + "tokens_seen": 287899648 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007375727181544634, + "loss": 1.6345, + "theoretical_loss": 4.194922615556941, + "tokens_seen": 287965184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007375566700100301, + "loss": 1.7056, + "theoretical_loss": 4.194810634584061, + "tokens_seen": 288030720 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007375406218655968, + "loss": 1.7121, + "theoretical_loss": 4.194698686219778, + "tokens_seen": 288096256 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007375245737211636, + "loss": 1.6854, + "theoretical_loss": 4.194586770447179, + "tokens_seen": 288161792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007375085255767302, + "loss": 1.6744, + "theoretical_loss": 4.194474887249369, + "tokens_seen": 288227328 + }, + { + "epoch": 0.09, + "learning_rate": 0.000737492477432297, + "loss": 1.6605, + "theoretical_loss": 4.19436303660946, + "tokens_seen": 288292864 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.9943329095840454, + "objective/train/avg_token_score": 0.005667079705744982, + "objective/train/docs_used": 111591, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6577270030975342, + "objective/train/original_loss": 1.667661428451538, + "objective/train/theoretical_loss": 4.194251218510583, + "objective/train/tokens_used": 308818400, + "objective/train/unlikelihood_loss": 0.012920353561639786, + "theoretical_loss": 4.194251218510583, + "tokens_seen": 288358400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007374764292878637, + "loss": 1.6288, + "theoretical_loss": 4.194251218510583, + "tokens_seen": 288358400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007374603811434304, + "loss": 1.7176, + "theoretical_loss": 4.194139432935877, + "tokens_seen": 288423936 + }, + { + "epoch": 0.09, + "learning_rate": 0.000737444332998997, + "loss": 1.7369, + "theoretical_loss": 4.194027679868495, + "tokens_seen": 288489472 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007374282848545638, + "loss": 1.6797, + "theoretical_loss": 4.193915959291601, + "tokens_seen": 288555008 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007374122367101305, + "loss": 1.7732, + "theoretical_loss": 4.193804271188376, + "tokens_seen": 288620544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007373961885656972, + "loss": 1.7748, + "theoretical_loss": 4.193692615542007, + "tokens_seen": 288686080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007373801404212639, + "loss": 1.8299, + "theoretical_loss": 4.193580992335699, + "tokens_seen": 288751616 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007373640922768306, + "loss": 1.7772, + "theoretical_loss": 4.193469401552667, + "tokens_seen": 288817152 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007373480441323973, + "loss": 1.7097, + "theoretical_loss": 4.193357843176139, + "tokens_seen": 288882688 + }, + { + "epoch": 0.09, + "learning_rate": 0.000737331995987964, + "loss": 1.6683, + "theoretical_loss": 4.193246317189356, + "tokens_seen": 288948224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007373159478435307, + "loss": 1.7662, + "theoretical_loss": 4.19313482357557, + "tokens_seen": 289013760 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007372998996990974, + "loss": 1.8484, + "theoretical_loss": 4.193023362318047, + "tokens_seen": 289079296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007372838515546641, + "loss": 1.7459, + "theoretical_loss": 4.1929119334000635, + "tokens_seen": 289144832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007372678034102308, + "loss": 1.6927, + "theoretical_loss": 4.192800536804912, + "tokens_seen": 289210368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007372517552657975, + "loss": 1.7054, + "theoretical_loss": 4.192689172515893, + "tokens_seen": 289275904 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007372357071213642, + "loss": 1.7032, + "theoretical_loss": 4.192577840516323, + "tokens_seen": 289341440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007372196589769309, + "loss": 1.6955, + "theoretical_loss": 4.192466540789528, + "tokens_seen": 289406976 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007372036108324975, + "loss": 1.7012, + "theoretical_loss": 4.19235527331885, + "tokens_seen": 289472512 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007371875626880643, + "loss": 1.6343, + "theoretical_loss": 4.19224403808764, + "tokens_seen": 289538048 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007371715145436309, + "loss": 1.766, + "theoretical_loss": 4.192132835079262, + "tokens_seen": 289603584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007371554663991977, + "loss": 1.6647, + "theoretical_loss": 4.192021664277094, + "tokens_seen": 289669120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007371394182547643, + "loss": 1.8121, + "theoretical_loss": 4.191910525664524, + "tokens_seen": 289734656 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007371233701103311, + "loss": 1.6457, + "theoretical_loss": 4.191799419224955, + "tokens_seen": 289800192 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007371073219658977, + "loss": 1.7834, + "theoretical_loss": 4.1916883449418, + "tokens_seen": 289865728 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007370912738214645, + "loss": 1.7966, + "theoretical_loss": 4.1915773027984855, + "tokens_seen": 289931264 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007370752256770311, + "loss": 1.724, + "theoretical_loss": 4.19146629277845, + "tokens_seen": 289996800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007370591775325978, + "loss": 1.7636, + "theoretical_loss": 4.1913553148651435, + "tokens_seen": 290062336 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007370431293881645, + "loss": 1.6831, + "theoretical_loss": 4.191244369042029, + "tokens_seen": 290127872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007370270812437312, + "loss": 1.7328, + "theoretical_loss": 4.191133455292583, + "tokens_seen": 290193408 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007370110330992979, + "loss": 1.7513, + "theoretical_loss": 4.191022573600293, + "tokens_seen": 290258944 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007369949849548646, + "loss": 1.7393, + "theoretical_loss": 4.190911723948657, + "tokens_seen": 290324480 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007369789368104313, + "loss": 1.6543, + "theoretical_loss": 4.190800906321188, + "tokens_seen": 290390016 + }, + { + "epoch": 0.09, + "learning_rate": 0.000736962888665998, + "loss": 1.7516, + "theoretical_loss": 4.190690120701412, + "tokens_seen": 290455552 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007369468405215647, + "loss": 1.6035, + "theoretical_loss": 4.190579367072861, + "tokens_seen": 290521088 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007369307923771314, + "loss": 1.6588, + "theoretical_loss": 4.190468645419086, + "tokens_seen": 290586624 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007369147442326981, + "loss": 1.7415, + "theoretical_loss": 4.190357955723649, + "tokens_seen": 290652160 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007368986960882648, + "loss": 1.6015, + "theoretical_loss": 4.190247297970121, + "tokens_seen": 290717696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007368826479438315, + "loss": 1.7648, + "theoretical_loss": 4.190136672142088, + "tokens_seen": 290783232 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007368665997993982, + "loss": 1.7645, + "theoretical_loss": 4.190026078223145, + "tokens_seen": 290848768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007368505516549649, + "loss": 1.7249, + "theoretical_loss": 4.189915516196904, + "tokens_seen": 290914304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007368345035105316, + "loss": 1.7157, + "theoretical_loss": 4.189804986046984, + "tokens_seen": 290979840 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007368184553660983, + "loss": 1.8322, + "theoretical_loss": 4.18969448775702, + "tokens_seen": 291045376 + }, + { + "epoch": 0.09, + "learning_rate": 0.000736802407221665, + "loss": 1.854, + "theoretical_loss": 4.189584021310657, + "tokens_seen": 291110912 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007367863590772317, + "loss": 1.6284, + "theoretical_loss": 4.189473586691554, + "tokens_seen": 291176448 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007367703109327984, + "loss": 1.6606, + "theoretical_loss": 4.189363183883377, + "tokens_seen": 291241984 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007367542627883651, + "loss": 1.6639, + "theoretical_loss": 4.189252812869811, + "tokens_seen": 291307520 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007367382146439318, + "loss": 1.775, + "theoretical_loss": 4.18914247363455, + "tokens_seen": 291373056 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007367221664994985, + "loss": 1.6875, + "theoretical_loss": 4.189032166161297, + "tokens_seen": 291438592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007367061183550652, + "loss": 1.6348, + "theoretical_loss": 4.188921890433772, + "tokens_seen": 291504128 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007366900702106319, + "loss": 1.71, + "theoretical_loss": 4.1888116464357035, + "tokens_seen": 291569664 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.9927524328231812, + "objective/train/avg_token_score": 0.007247562985867262, + "objective/train/docs_used": 112844, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6681028604507446, + "objective/train/original_loss": 1.6812455654144287, + "objective/train/theoretical_loss": 4.188701434150834, + "objective/train/tokens_used": 312095200, + "objective/train/unlikelihood_loss": 0.015475046820938587, + "theoretical_loss": 4.188701434150834, + "tokens_seen": 291635200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007366740220661986, + "loss": 1.765, + "theoretical_loss": 4.188701434150834, + "tokens_seen": 291635200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007366579739217653, + "loss": 1.6976, + "theoretical_loss": 4.188591253562918, + "tokens_seen": 291700736 + }, + { + "epoch": 0.09, + "learning_rate": 0.000736641925777332, + "loss": 1.6533, + "theoretical_loss": 4.18848110465572, + "tokens_seen": 291766272 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007366258776328987, + "loss": 1.5795, + "theoretical_loss": 4.188370987413018, + "tokens_seen": 291831808 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007366098294884654, + "loss": 1.8085, + "theoretical_loss": 4.188260901818602, + "tokens_seen": 291897344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007365937813440321, + "loss": 1.7958, + "theoretical_loss": 4.1881508478562735, + "tokens_seen": 291962880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007365777331995988, + "loss": 1.7167, + "theoretical_loss": 4.188040825509845, + "tokens_seen": 292028416 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007365616850551655, + "loss": 1.7282, + "theoretical_loss": 4.1879308347631445, + "tokens_seen": 292093952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007365456369107322, + "loss": 1.7417, + "theoretical_loss": 4.187820875600006, + "tokens_seen": 292159488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007365295887662989, + "loss": 1.6944, + "theoretical_loss": 4.187710948004281, + "tokens_seen": 292225024 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007365135406218656, + "loss": 1.7333, + "theoretical_loss": 4.187601051959831, + "tokens_seen": 292290560 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007364974924774323, + "loss": 1.7556, + "theoretical_loss": 4.187491187450527, + "tokens_seen": 292356096 + }, + { + "epoch": 0.09, + "learning_rate": 0.000736481444332999, + "loss": 1.8586, + "theoretical_loss": 4.187381354460255, + "tokens_seen": 292421632 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007364653961885657, + "loss": 1.7408, + "theoretical_loss": 4.187271552972911, + "tokens_seen": 292487168 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007364493480441324, + "loss": 1.7306, + "theoretical_loss": 4.187161782972405, + "tokens_seen": 292552704 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007364332998996991, + "loss": 1.7378, + "theoretical_loss": 4.187052044442655, + "tokens_seen": 292618240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007364172517552658, + "loss": 1.8866, + "theoretical_loss": 4.1869423373675945, + "tokens_seen": 292683776 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007364012036108325, + "loss": 1.815, + "theoretical_loss": 4.1868326617311675, + "tokens_seen": 292749312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007363851554663992, + "loss": 1.699, + "theoretical_loss": 4.18672301751733, + "tokens_seen": 292814848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007363691073219659, + "loss": 1.7092, + "theoretical_loss": 4.186613404710048, + "tokens_seen": 292880384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007363530591775326, + "loss": 1.8559, + "theoretical_loss": 4.186503823293303, + "tokens_seen": 292945920 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007363370110330993, + "loss": 1.6635, + "theoretical_loss": 4.186394273251084, + "tokens_seen": 293011456 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007363209628886661, + "loss": 1.8038, + "theoretical_loss": 4.186284754567395, + "tokens_seen": 293076992 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007363049147442327, + "loss": 1.8287, + "theoretical_loss": 4.1861752672262496, + "tokens_seen": 293142528 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007362888665997995, + "loss": 1.8485, + "theoretical_loss": 4.186065811211675, + "tokens_seen": 293208064 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007362728184553661, + "loss": 1.7182, + "theoretical_loss": 4.185956386507708, + "tokens_seen": 293273600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007362567703109329, + "loss": 1.783, + "theoretical_loss": 4.185846993098398, + "tokens_seen": 293339136 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007362407221664995, + "loss": 1.7056, + "theoretical_loss": 4.185737630967808, + "tokens_seen": 293404672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007362246740220663, + "loss": 1.8482, + "theoretical_loss": 4.18562830010001, + "tokens_seen": 293470208 + }, + { + "epoch": 0.09, + "learning_rate": 0.000736208625877633, + "loss": 1.7116, + "theoretical_loss": 4.185519000479089, + "tokens_seen": 293535744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007361925777331997, + "loss": 1.7428, + "theoretical_loss": 4.18540973208914, + "tokens_seen": 293601280 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007361765295887663, + "loss": 1.7476, + "theoretical_loss": 4.185300494914273, + "tokens_seen": 293666816 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007361604814443331, + "loss": 1.8073, + "theoretical_loss": 4.185191288938606, + "tokens_seen": 293732352 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007361444332998997, + "loss": 1.5811, + "theoretical_loss": 4.185082114146271, + "tokens_seen": 293797888 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007361283851554665, + "loss": 1.746, + "theoretical_loss": 4.184972970521409, + "tokens_seen": 293863424 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007361123370110332, + "loss": 1.7414, + "theoretical_loss": 4.184863858048177, + "tokens_seen": 293928960 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007360962888665999, + "loss": 1.7694, + "theoretical_loss": 4.18475477671074, + "tokens_seen": 293994496 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007360802407221666, + "loss": 1.67, + "theoretical_loss": 4.184645726493275, + "tokens_seen": 294060032 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007360641925777333, + "loss": 1.8083, + "theoretical_loss": 4.184536707379972, + "tokens_seen": 294125568 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007360481444333, + "loss": 1.6991, + "theoretical_loss": 4.184427719355031, + "tokens_seen": 294191104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007360320962888667, + "loss": 1.845, + "theoretical_loss": 4.184318762402664, + "tokens_seen": 294256640 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007360160481444334, + "loss": 1.7422, + "theoretical_loss": 4.184209836507096, + "tokens_seen": 294322176 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007360000000000001, + "loss": 1.7148, + "theoretical_loss": 4.18410094165256, + "tokens_seen": 294387712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007359839518555668, + "loss": 1.8483, + "theoretical_loss": 4.183992077823305, + "tokens_seen": 294453248 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007359679037111335, + "loss": 1.7322, + "theoretical_loss": 4.183883245003588, + "tokens_seen": 294518784 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007359518555667002, + "loss": 1.791, + "theoretical_loss": 4.183774443177679, + "tokens_seen": 294584320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007359358074222669, + "loss": 1.931, + "theoretical_loss": 4.183665672329861, + "tokens_seen": 294649856 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007359197592778336, + "loss": 1.802, + "theoretical_loss": 4.183556932444424, + "tokens_seen": 294715392 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007359037111334003, + "loss": 2.0942, + "theoretical_loss": 4.183448223505673, + "tokens_seen": 294780928 + }, + { + "epoch": 0.09, + "learning_rate": 0.000735887662988967, + "loss": 1.8294, + "theoretical_loss": 4.183339545497924, + "tokens_seen": 294846464 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.9940253496170044, + "objective/train/avg_token_score": 0.005974633619189262, + "objective/train/docs_used": 114046, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.8631888628005981, + "objective/train/original_loss": 1.8736722469329834, + "objective/train/theoretical_loss": 4.1832308984055055, + "objective/train/tokens_used": 315372000, + "objective/train/unlikelihood_loss": 0.013550673611462116, + "theoretical_loss": 4.1832308984055055, + "tokens_seen": 294912000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007358716148445337, + "loss": 1.8431, + "theoretical_loss": 4.1832308984055055, + "tokens_seen": 294912000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007358555667001004, + "loss": 1.7599, + "theoretical_loss": 4.183122282212754, + "tokens_seen": 294977536 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007358395185556671, + "loss": 1.9602, + "theoretical_loss": 4.183013696904018, + "tokens_seen": 295043072 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007358234704112338, + "loss": 1.747, + "theoretical_loss": 4.182905142463662, + "tokens_seen": 295108608 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007358074222668004, + "loss": 1.7887, + "theoretical_loss": 4.182796618876057, + "tokens_seen": 295174144 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007357913741223672, + "loss": 1.8521, + "theoretical_loss": 4.182688126125587, + "tokens_seen": 295239680 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007357753259779338, + "loss": 1.8529, + "theoretical_loss": 4.182579664196648, + "tokens_seen": 295305216 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007357592778335006, + "loss": 1.749, + "theoretical_loss": 4.182471233073647, + "tokens_seen": 295370752 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007357432296890672, + "loss": 1.8022, + "theoretical_loss": 4.182362832741001, + "tokens_seen": 295436288 + }, + { + "epoch": 0.09, + "learning_rate": 0.000735727181544634, + "loss": 1.972, + "theoretical_loss": 4.182254463183139, + "tokens_seen": 295501824 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007357111334002006, + "loss": 1.6731, + "theoretical_loss": 4.182146124384504, + "tokens_seen": 295567360 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007356950852557674, + "loss": 1.8922, + "theoretical_loss": 4.182037816329547, + "tokens_seen": 295632896 + }, + { + "epoch": 0.09, + "learning_rate": 0.000735679037111334, + "loss": 1.8092, + "theoretical_loss": 4.181929539002731, + "tokens_seen": 295698432 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007356629889669007, + "loss": 1.8176, + "theoretical_loss": 4.1818212923885305, + "tokens_seen": 295763968 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007356469408224674, + "loss": 1.7302, + "theoretical_loss": 4.181713076471432, + "tokens_seen": 295829504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007356308926780341, + "loss": 1.7423, + "theoretical_loss": 4.181604891235933, + "tokens_seen": 295895040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007356148445336008, + "loss": 1.7356, + "theoretical_loss": 4.181496736666542, + "tokens_seen": 295960576 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007355987963891675, + "loss": 1.7221, + "theoretical_loss": 4.181388612747778, + "tokens_seen": 296026112 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007355827482447342, + "loss": 1.8275, + "theoretical_loss": 4.181280519464172, + "tokens_seen": 296091648 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007355667001003009, + "loss": 1.7491, + "theoretical_loss": 4.181172456800267, + "tokens_seen": 296157184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007355506519558676, + "loss": 1.7673, + "theoretical_loss": 4.181064424740617, + "tokens_seen": 296222720 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007355346038114343, + "loss": 1.7513, + "theoretical_loss": 4.180956423269784, + "tokens_seen": 296288256 + }, + { + "epoch": 0.09, + "learning_rate": 0.000735518555667001, + "loss": 1.7096, + "theoretical_loss": 4.180848452372347, + "tokens_seen": 296353792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007355025075225677, + "loss": 1.7834, + "theoretical_loss": 4.180740512032891, + "tokens_seen": 296419328 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007354864593781344, + "loss": 1.7452, + "theoretical_loss": 4.180632602236014, + "tokens_seen": 296484864 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007354704112337011, + "loss": 1.7853, + "theoretical_loss": 4.180524722966327, + "tokens_seen": 296550400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007354543630892678, + "loss": 1.7009, + "theoretical_loss": 4.180416874208449, + "tokens_seen": 296615936 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007354383149448345, + "loss": 1.8783, + "theoretical_loss": 4.180309055947012, + "tokens_seen": 296681472 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007354222668004012, + "loss": 1.7725, + "theoretical_loss": 4.18020126816666, + "tokens_seen": 296747008 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007354062186559679, + "loss": 1.6049, + "theoretical_loss": 4.180093510852045, + "tokens_seen": 296812544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007353901705115346, + "loss": 1.774, + "theoretical_loss": 4.179985783987834, + "tokens_seen": 296878080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007353741223671013, + "loss": 1.7479, + "theoretical_loss": 4.179878087558702, + "tokens_seen": 296943616 + }, + { + "epoch": 0.09, + "learning_rate": 0.000735358074222668, + "loss": 1.8325, + "theoretical_loss": 4.179770421549336, + "tokens_seen": 297009152 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007353420260782347, + "loss": 1.7244, + "theoretical_loss": 4.179662785944435, + "tokens_seen": 297074688 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007353259779338014, + "loss": 1.7875, + "theoretical_loss": 4.179555180728708, + "tokens_seen": 297140224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007353099297893681, + "loss": 1.6599, + "theoretical_loss": 4.179447605886876, + "tokens_seen": 297205760 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007352938816449348, + "loss": 1.5621, + "theoretical_loss": 4.1793400614036695, + "tokens_seen": 297271296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007352778335005015, + "loss": 1.6465, + "theoretical_loss": 4.179232547263831, + "tokens_seen": 297336832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007352617853560682, + "loss": 1.7405, + "theoretical_loss": 4.1791250634521155, + "tokens_seen": 297402368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007352457372116349, + "loss": 1.7445, + "theoretical_loss": 4.179017609953287, + "tokens_seen": 297467904 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007352296890672016, + "loss": 1.7304, + "theoretical_loss": 4.17891018675212, + "tokens_seen": 297533440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007352136409227683, + "loss": 1.7169, + "theoretical_loss": 4.178802793833402, + "tokens_seen": 297598976 + }, + { + "epoch": 0.09, + "learning_rate": 0.000735197592778335, + "loss": 1.7359, + "theoretical_loss": 4.17869543118193, + "tokens_seen": 297664512 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007351815446339017, + "loss": 1.676, + "theoretical_loss": 4.178588098782513, + "tokens_seen": 297730048 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007351654964894684, + "loss": 1.8075, + "theoretical_loss": 4.178480796619971, + "tokens_seen": 297795584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007351494483450351, + "loss": 1.7003, + "theoretical_loss": 4.178373524679134, + "tokens_seen": 297861120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007351334002006018, + "loss": 1.7122, + "theoretical_loss": 4.1782662829448425, + "tokens_seen": 297926656 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007351173520561686, + "loss": 1.7883, + "theoretical_loss": 4.17815907140195, + "tokens_seen": 297992192 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007351013039117352, + "loss": 1.7481, + "theoretical_loss": 4.178051890035319, + "tokens_seen": 298057728 + }, + { + "epoch": 0.09, + "learning_rate": 0.000735085255767302, + "loss": 1.6647, + "theoretical_loss": 4.177944738829825, + "tokens_seen": 298123264 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.9918665289878845, + "objective/train/avg_token_score": 0.00813344493508339, + "objective/train/docs_used": 115247, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6643564701080322, + "objective/train/original_loss": 1.6778080463409424, + "objective/train/theoretical_loss": 4.177837617770352, + "objective/train/tokens_used": 318648800, + "objective/train/unlikelihood_loss": 0.018988044932484627, + "theoretical_loss": 4.177837617770352, + "tokens_seen": 298188800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007350692076228686, + "loss": 1.7406, + "theoretical_loss": 4.177837617770352, + "tokens_seen": 298188800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007350531594784354, + "loss": 1.6381, + "theoretical_loss": 4.1777305268417955, + "tokens_seen": 298254336 + }, + { + "epoch": 0.09, + "learning_rate": 0.000735037111334002, + "loss": 1.7736, + "theoretical_loss": 4.177623466029064, + "tokens_seen": 298319872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007350210631895688, + "loss": 1.8171, + "theoretical_loss": 4.177516435317074, + "tokens_seen": 298385408 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007350050150451354, + "loss": 1.775, + "theoretical_loss": 4.177409434690754, + "tokens_seen": 298450944 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007349889669007022, + "loss": 1.6719, + "theoretical_loss": 4.177302464135044, + "tokens_seen": 298516480 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007349729187562688, + "loss": 1.7708, + "theoretical_loss": 4.177195523634895, + "tokens_seen": 298582016 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007349568706118356, + "loss": 1.7102, + "theoretical_loss": 4.1770886131752665, + "tokens_seen": 298647552 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007349408224674022, + "loss": 1.7544, + "theoretical_loss": 4.176981732741131, + "tokens_seen": 298713088 + }, + { + "epoch": 0.09, + "learning_rate": 0.000734924774322969, + "loss": 1.7732, + "theoretical_loss": 4.176874882317472, + "tokens_seen": 298778624 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007349087261785356, + "loss": 1.86, + "theoretical_loss": 4.176768061889283, + "tokens_seen": 298844160 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007348926780341024, + "loss": 1.8906, + "theoretical_loss": 4.176661271441568, + "tokens_seen": 298909696 + }, + { + "epoch": 0.09, + "learning_rate": 0.000734876629889669, + "loss": 1.709, + "theoretical_loss": 4.176554510959342, + "tokens_seen": 298975232 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007348605817452358, + "loss": 1.7043, + "theoretical_loss": 4.1764477804276305, + "tokens_seen": 299040768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007348445336008024, + "loss": 1.6672, + "theoretical_loss": 4.176341079831471, + "tokens_seen": 299106304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007348284854563692, + "loss": 1.7704, + "theoretical_loss": 4.176234409155912, + "tokens_seen": 299171840 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007348124373119358, + "loss": 1.6937, + "theoretical_loss": 4.176127768386008, + "tokens_seen": 299237376 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007347963891675026, + "loss": 1.7457, + "theoretical_loss": 4.176021157506832, + "tokens_seen": 299302912 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007347803410230692, + "loss": 1.6849, + "theoretical_loss": 4.1759145765034615, + "tokens_seen": 299368448 + }, + { + "epoch": 0.09, + "learning_rate": 0.000734764292878636, + "loss": 1.6085, + "theoretical_loss": 4.175808025360988, + "tokens_seen": 299433984 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007347482447342027, + "loss": 1.7279, + "theoretical_loss": 4.175701504064511, + "tokens_seen": 299499520 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007347321965897694, + "loss": 1.6278, + "theoretical_loss": 4.175595012599143, + "tokens_seen": 299565056 + }, + { + "epoch": 0.09, + "learning_rate": 0.000734716148445336, + "loss": 1.6855, + "theoretical_loss": 4.175488550950007, + "tokens_seen": 299630592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007347001003009028, + "loss": 1.7585, + "theoretical_loss": 4.175382119102235, + "tokens_seen": 299696128 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007346840521564695, + "loss": 1.6587, + "theoretical_loss": 4.175275717040972, + "tokens_seen": 299761664 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007346680040120362, + "loss": 1.6828, + "theoretical_loss": 4.175169344751371, + "tokens_seen": 299827200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007346519558676029, + "loss": 1.7838, + "theoretical_loss": 4.175063002218598, + "tokens_seen": 299892736 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007346359077231696, + "loss": 1.7822, + "theoretical_loss": 4.174956689427829, + "tokens_seen": 299958272 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007346198595787363, + "loss": 1.7883, + "theoretical_loss": 4.17485040636425, + "tokens_seen": 300023808 + }, + { + "epoch": 0.09, + "learning_rate": 0.000734603811434303, + "loss": 1.6953, + "theoretical_loss": 4.174744153013057, + "tokens_seen": 300089344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007345877632898697, + "loss": 1.7443, + "theoretical_loss": 4.174637929359458, + "tokens_seen": 300154880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007345717151454364, + "loss": 1.6521, + "theoretical_loss": 4.174531735388673, + "tokens_seen": 300220416 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007345556670010031, + "loss": 1.6735, + "theoretical_loss": 4.174425571085927, + "tokens_seen": 300285952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007345396188565698, + "loss": 1.7097, + "theoretical_loss": 4.174319436436462, + "tokens_seen": 300351488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007345235707121365, + "loss": 1.5888, + "theoretical_loss": 4.174213331425527, + "tokens_seen": 300417024 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007345075225677032, + "loss": 1.6496, + "theoretical_loss": 4.174107256038383, + "tokens_seen": 300482560 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007344914744232699, + "loss": 1.699, + "theoretical_loss": 4.1740012102603, + "tokens_seen": 300548096 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007344754262788366, + "loss": 1.7634, + "theoretical_loss": 4.17389519407656, + "tokens_seen": 300613632 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007344593781344033, + "loss": 1.5694, + "theoretical_loss": 4.173789207472455, + "tokens_seen": 300679168 + }, + { + "epoch": 0.09, + "learning_rate": 0.00073444332998997, + "loss": 1.8166, + "theoretical_loss": 4.173683250433287, + "tokens_seen": 300744704 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007344272818455367, + "loss": 1.7068, + "theoretical_loss": 4.17357732294437, + "tokens_seen": 300810240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007344112337011033, + "loss": 1.7019, + "theoretical_loss": 4.1734714249910265, + "tokens_seen": 300875776 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007343951855566701, + "loss": 1.8613, + "theoretical_loss": 4.173365556558591, + "tokens_seen": 300941312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007343791374122367, + "loss": 1.7061, + "theoretical_loss": 4.1732597176324076, + "tokens_seen": 301006848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007343630892678035, + "loss": 1.5984, + "theoretical_loss": 4.173153908197831, + "tokens_seen": 301072384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007343470411233701, + "loss": 1.853, + "theoretical_loss": 4.173048128240227, + "tokens_seen": 301137920 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007343309929789369, + "loss": 1.6273, + "theoretical_loss": 4.172942377744972, + "tokens_seen": 301203456 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007343149448345035, + "loss": 1.6101, + "theoretical_loss": 4.17283665669745, + "tokens_seen": 301268992 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007342988966900703, + "loss": 1.6009, + "theoretical_loss": 4.172730965083061, + "tokens_seen": 301334528 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007342828485456369, + "loss": 1.7047, + "theoretical_loss": 4.1726253028872105, + "tokens_seen": 301400064 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.9897353649139404, + "objective/train/avg_token_score": 0.010264642536640167, + "objective/train/docs_used": 116418, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5901623964309692, + "objective/train/original_loss": 1.6072778701782227, + "objective/train/theoretical_loss": 4.172519670095315, + "objective/train/tokens_used": 321925600, + "objective/train/unlikelihood_loss": 0.02213309146463871, + "theoretical_loss": 4.172519670095315, + "tokens_seen": 301465600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007342668004012036, + "loss": 1.7955, + "theoretical_loss": 4.172519670095315, + "tokens_seen": 301465600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007342507522567703, + "loss": 1.6198, + "theoretical_loss": 4.172414066692804, + "tokens_seen": 301531136 + }, + { + "epoch": 0.09, + "learning_rate": 0.000734234704112337, + "loss": 1.6693, + "theoretical_loss": 4.172308492665114, + "tokens_seen": 301596672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007342186559679037, + "loss": 1.7085, + "theoretical_loss": 4.172202947997695, + "tokens_seen": 301662208 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007342026078234704, + "loss": 1.633, + "theoretical_loss": 4.172097432676006, + "tokens_seen": 301727744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007341865596790371, + "loss": 1.8288, + "theoretical_loss": 4.171991946685516, + "tokens_seen": 301793280 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007341705115346038, + "loss": 1.6549, + "theoretical_loss": 4.171886490011705, + "tokens_seen": 301858816 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007341544633901705, + "loss": 1.6776, + "theoretical_loss": 4.171781062640063, + "tokens_seen": 301924352 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007341384152457372, + "loss": 1.6914, + "theoretical_loss": 4.17167566455609, + "tokens_seen": 301989888 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007341223671013039, + "loss": 1.6932, + "theoretical_loss": 4.171570295745297, + "tokens_seen": 302055424 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007341063189568706, + "loss": 1.8784, + "theoretical_loss": 4.171464956193205, + "tokens_seen": 302120960 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007340902708124373, + "loss": 1.7132, + "theoretical_loss": 4.171359645885346, + "tokens_seen": 302186496 + }, + { + "epoch": 0.09, + "learning_rate": 0.000734074222668004, + "loss": 1.7058, + "theoretical_loss": 4.17125436480726, + "tokens_seen": 302252032 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007340581745235707, + "loss": 1.763, + "theoretical_loss": 4.171149112944501, + "tokens_seen": 302317568 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007340421263791374, + "loss": 1.6724, + "theoretical_loss": 4.171043890282629, + "tokens_seen": 302383104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007340260782347041, + "loss": 1.714, + "theoretical_loss": 4.170938696807218, + "tokens_seen": 302448640 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007340100300902708, + "loss": 1.6408, + "theoretical_loss": 4.17083353250385, + "tokens_seen": 302514176 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007339939819458375, + "loss": 1.7141, + "theoretical_loss": 4.170728397358118, + "tokens_seen": 302579712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007339779338014042, + "loss": 1.7152, + "theoretical_loss": 4.170623291355626, + "tokens_seen": 302645248 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007339618856569709, + "loss": 1.7207, + "theoretical_loss": 4.1705182144819855, + "tokens_seen": 302710784 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007339458375125376, + "loss": 1.709, + "theoretical_loss": 4.170413166722822, + "tokens_seen": 302776320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007339297893681043, + "loss": 1.6522, + "theoretical_loss": 4.170308148063769, + "tokens_seen": 302841856 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007339137412236711, + "loss": 1.7759, + "theoretical_loss": 4.1702031584904695, + "tokens_seen": 302907392 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007338976930792377, + "loss": 1.6369, + "theoretical_loss": 4.170098197988578, + "tokens_seen": 302972928 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007338816449348045, + "loss": 1.7978, + "theoretical_loss": 4.169993266543759, + "tokens_seen": 303038464 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007338655967903711, + "loss": 1.6959, + "theoretical_loss": 4.169888364141689, + "tokens_seen": 303104000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007338495486459379, + "loss": 1.6958, + "theoretical_loss": 4.169783490768049, + "tokens_seen": 303169536 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007338335005015045, + "loss": 1.7201, + "theoretical_loss": 4.169678646408536, + "tokens_seen": 303235072 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007338174523570713, + "loss": 1.6169, + "theoretical_loss": 4.169573831048854, + "tokens_seen": 303300608 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007338014042126379, + "loss": 1.7928, + "theoretical_loss": 4.16946904467472, + "tokens_seen": 303366144 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007337853560682047, + "loss": 1.6813, + "theoretical_loss": 4.169364287271858, + "tokens_seen": 303431680 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007337693079237713, + "loss": 1.6734, + "theoretical_loss": 4.169259558826003, + "tokens_seen": 303497216 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007337532597793381, + "loss": 1.8603, + "theoretical_loss": 4.169154859322901, + "tokens_seen": 303562752 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007337372116349047, + "loss": 1.7972, + "theoretical_loss": 4.169050188748307, + "tokens_seen": 303628288 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007337211634904715, + "loss": 1.7937, + "theoretical_loss": 4.168945547087987, + "tokens_seen": 303693824 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007337051153460381, + "loss": 1.7261, + "theoretical_loss": 4.168840934327717, + "tokens_seen": 303759360 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007336890672016049, + "loss": 1.6361, + "theoretical_loss": 4.1687363504532815, + "tokens_seen": 303824896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007336730190571715, + "loss": 1.5983, + "theoretical_loss": 4.168631795450478, + "tokens_seen": 303890432 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007336569709127383, + "loss": 1.6863, + "theoretical_loss": 4.168527269305112, + "tokens_seen": 303955968 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007336409227683049, + "loss": 1.7595, + "theoretical_loss": 4.168422772002998, + "tokens_seen": 304021504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007336248746238717, + "loss": 1.7182, + "theoretical_loss": 4.168318303529963, + "tokens_seen": 304087040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007336088264794383, + "loss": 1.7289, + "theoretical_loss": 4.168213863871843, + "tokens_seen": 304152576 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007335927783350051, + "loss": 1.7237, + "theoretical_loss": 4.168109453014484, + "tokens_seen": 304218112 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007335767301905717, + "loss": 1.6706, + "theoretical_loss": 4.1680050709437415, + "tokens_seen": 304283648 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007335606820461385, + "loss": 1.6401, + "theoretical_loss": 4.167900717645482, + "tokens_seen": 304349184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007335446339017051, + "loss": 1.6426, + "theoretical_loss": 4.16779639310558, + "tokens_seen": 304414720 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007335285857572719, + "loss": 1.6982, + "theoretical_loss": 4.167692097309923, + "tokens_seen": 304480256 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007335125376128385, + "loss": 1.5454, + "theoretical_loss": 4.167587830244408, + "tokens_seen": 304545792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007334964894684053, + "loss": 1.6083, + "theoretical_loss": 4.167483591894937, + "tokens_seen": 304611328 + }, + { + "epoch": 0.09, + "learning_rate": 0.000733480441323972, + "loss": 1.8034, + "theoretical_loss": 4.167379382247429, + "tokens_seen": 304676864 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.9910046458244324, + "objective/train/avg_token_score": 0.0089954175055027, + "objective/train/docs_used": 117554, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5992234945297241, + "objective/train/original_loss": 1.614647626876831, + "objective/train/theoretical_loss": 4.167275201287808, + "objective/train/tokens_used": 325202400, + "objective/train/unlikelihood_loss": 0.018571501597762108, + "theoretical_loss": 4.167275201287808, + "tokens_seen": 304742400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007334643931795387, + "loss": 1.6062, + "theoretical_loss": 4.167275201287808, + "tokens_seen": 304742400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007334483450351053, + "loss": 1.8127, + "theoretical_loss": 4.16717104900201, + "tokens_seen": 304807936 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007334322968906721, + "loss": 1.9052, + "theoretical_loss": 4.167066925375981, + "tokens_seen": 304873472 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007334162487462387, + "loss": 1.6823, + "theoretical_loss": 4.1669628303956765, + "tokens_seen": 304939008 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007334002006018055, + "loss": 1.7124, + "theoretical_loss": 4.166858764047062, + "tokens_seen": 305004544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007333841524573721, + "loss": 1.7336, + "theoretical_loss": 4.166754726316111, + "tokens_seen": 305070080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007333681043129389, + "loss": 1.6686, + "theoretical_loss": 4.16665071718881, + "tokens_seen": 305135616 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007333520561685056, + "loss": 1.8863, + "theoretical_loss": 4.1665467366511555, + "tokens_seen": 305201152 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007333360080240723, + "loss": 1.7789, + "theoretical_loss": 4.166442784689149, + "tokens_seen": 305266688 + }, + { + "epoch": 0.09, + "learning_rate": 0.000733319959879639, + "loss": 1.7885, + "theoretical_loss": 4.166338861288808, + "tokens_seen": 305332224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007333039117352057, + "loss": 1.8485, + "theoretical_loss": 4.166234966436155, + "tokens_seen": 305397760 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007332878635907724, + "loss": 1.751, + "theoretical_loss": 4.166131100117226, + "tokens_seen": 305463296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007332718154463391, + "loss": 1.7691, + "theoretical_loss": 4.166027262318064, + "tokens_seen": 305528832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007332557673019058, + "loss": 1.8374, + "theoretical_loss": 4.165923453024724, + "tokens_seen": 305594368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007332397191574725, + "loss": 1.8034, + "theoretical_loss": 4.165819672223269, + "tokens_seen": 305659904 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007332236710130392, + "loss": 1.8114, + "theoretical_loss": 4.165715919899773, + "tokens_seen": 305725440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007332076228686059, + "loss": 1.6594, + "theoretical_loss": 4.165612196040319, + "tokens_seen": 305790976 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007331915747241726, + "loss": 1.7028, + "theoretical_loss": 4.1655085006310015, + "tokens_seen": 305856512 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007331755265797393, + "loss": 1.8361, + "theoretical_loss": 4.165404833657923, + "tokens_seen": 305922048 + }, + { + "epoch": 0.09, + "learning_rate": 0.000733159478435306, + "loss": 1.7469, + "theoretical_loss": 4.165301195107195, + "tokens_seen": 305987584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007331434302908727, + "loss": 1.785, + "theoretical_loss": 4.1651975849649405, + "tokens_seen": 306053120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007331273821464394, + "loss": 1.6883, + "theoretical_loss": 4.165094003217293, + "tokens_seen": 306118656 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007331113340020061, + "loss": 1.7022, + "theoretical_loss": 4.164990449850393, + "tokens_seen": 306184192 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007330952858575728, + "loss": 1.8655, + "theoretical_loss": 4.164886924850393, + "tokens_seen": 306249728 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007330792377131395, + "loss": 1.7419, + "theoretical_loss": 4.164783428203453, + "tokens_seen": 306315264 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007330631895687062, + "loss": 1.688, + "theoretical_loss": 4.1646799598957465, + "tokens_seen": 306380800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007330471414242729, + "loss": 1.5923, + "theoretical_loss": 4.164576519913453, + "tokens_seen": 306446336 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007330310932798396, + "loss": 1.6858, + "theoretical_loss": 4.164473108242763, + "tokens_seen": 306511872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007330150451354062, + "loss": 1.9784, + "theoretical_loss": 4.164369724869878, + "tokens_seen": 306577408 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732998996990973, + "loss": 1.5107, + "theoretical_loss": 4.164266369781005, + "tokens_seen": 306642944 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007329829488465396, + "loss": 1.7494, + "theoretical_loss": 4.164163042962366, + "tokens_seen": 306708480 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007329669007021064, + "loss": 1.6482, + "theoretical_loss": 4.1640597444001894, + "tokens_seen": 306774016 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732950852557673, + "loss": 1.7396, + "theoretical_loss": 4.163956474080715, + "tokens_seen": 306839552 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007329348044132398, + "loss": 1.7699, + "theoretical_loss": 4.16385323199019, + "tokens_seen": 306905088 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007329187562688064, + "loss": 1.7168, + "theoretical_loss": 4.1637500181148726, + "tokens_seen": 306970624 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007329027081243732, + "loss": 1.8076, + "theoretical_loss": 4.163646832441031, + "tokens_seen": 307036160 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007328866599799398, + "loss": 1.7472, + "theoretical_loss": 4.163543674954942, + "tokens_seen": 307101696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007328706118355065, + "loss": 1.8346, + "theoretical_loss": 4.163440545642894, + "tokens_seen": 307167232 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007328545636910732, + "loss": 1.7756, + "theoretical_loss": 4.163337444491183, + "tokens_seen": 307232768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007328385155466399, + "loss": 1.7881, + "theoretical_loss": 4.163234371486114, + "tokens_seen": 307298304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007328224674022066, + "loss": 1.6958, + "theoretical_loss": 4.163131326614003, + "tokens_seen": 307363840 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007328064192577733, + "loss": 1.7017, + "theoretical_loss": 4.163028309861176, + "tokens_seen": 307429376 + }, + { + "epoch": 0.09, + "learning_rate": 0.00073279037111334, + "loss": 1.8031, + "theoretical_loss": 4.162925321213968, + "tokens_seen": 307494912 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007327743229689067, + "loss": 1.6827, + "theoretical_loss": 4.162822360658723, + "tokens_seen": 307560448 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007327582748244734, + "loss": 1.7812, + "theoretical_loss": 4.162719428181794, + "tokens_seen": 307625984 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007327422266800401, + "loss": 1.5881, + "theoretical_loss": 4.162616523769547, + "tokens_seen": 307691520 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007327261785356068, + "loss": 1.7169, + "theoretical_loss": 4.1625136474083515, + "tokens_seen": 307757056 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007327101303911736, + "loss": 1.6765, + "theoretical_loss": 4.1624107990845935, + "tokens_seen": 307822592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007326940822467402, + "loss": 1.6176, + "theoretical_loss": 4.162307978784662, + "tokens_seen": 307888128 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732678034102307, + "loss": 1.7707, + "theoretical_loss": 4.162205186494962, + "tokens_seen": 307953664 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.991561770439148, + "objective/train/avg_token_score": 0.008438272401690483, + "objective/train/docs_used": 118729, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.7717139720916748, + "objective/train/original_loss": 1.7846508026123047, + "objective/train/theoretical_loss": 4.162102422201901, + "objective/train/tokens_used": 328479200, + "objective/train/unlikelihood_loss": 0.020347073674201965, + "theoretical_loss": 4.162102422201901, + "tokens_seen": 308019200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007326619859578736, + "loss": 1.7021, + "theoretical_loss": 4.162102422201901, + "tokens_seen": 308019200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007326459378134404, + "loss": 1.7896, + "theoretical_loss": 4.161999685891901, + "tokens_seen": 308084736 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732629889669007, + "loss": 1.7536, + "theoretical_loss": 4.161896977551393, + "tokens_seen": 308150272 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007326138415245738, + "loss": 1.6445, + "theoretical_loss": 4.161794297166815, + "tokens_seen": 308215808 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007325977933801404, + "loss": 1.6438, + "theoretical_loss": 4.161691644724616, + "tokens_seen": 308281344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007325817452357072, + "loss": 1.8107, + "theoretical_loss": 4.161589020211256, + "tokens_seen": 308346880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007325656970912738, + "loss": 1.6766, + "theoretical_loss": 4.1614864236132, + "tokens_seen": 308412416 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007325496489468406, + "loss": 1.7944, + "theoretical_loss": 4.161383854916926, + "tokens_seen": 308477952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007325336008024072, + "loss": 1.7196, + "theoretical_loss": 4.161281314108923, + "tokens_seen": 308543488 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732517552657974, + "loss": 1.716, + "theoretical_loss": 4.1611788011756845, + "tokens_seen": 308609024 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007325015045135406, + "loss": 1.7755, + "theoretical_loss": 4.161076316103717, + "tokens_seen": 308674560 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007324854563691074, + "loss": 1.7518, + "theoretical_loss": 4.160973858879535, + "tokens_seen": 308740096 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732469408224674, + "loss": 1.6486, + "theoretical_loss": 4.160871429489663, + "tokens_seen": 308805632 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007324533600802408, + "loss": 1.6828, + "theoretical_loss": 4.160769027920635, + "tokens_seen": 308871168 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007324373119358074, + "loss": 1.7101, + "theoretical_loss": 4.160666654158993, + "tokens_seen": 308936704 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007324212637913742, + "loss": 1.7265, + "theoretical_loss": 4.160564308191289, + "tokens_seen": 309002240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007324052156469408, + "loss": 1.8291, + "theoretical_loss": 4.1604619900040865, + "tokens_seen": 309067776 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007323891675025076, + "loss": 1.7638, + "theoretical_loss": 4.160359699583955, + "tokens_seen": 309133312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007323731193580742, + "loss": 1.701, + "theoretical_loss": 4.160257436917476, + "tokens_seen": 309198848 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732357071213641, + "loss": 1.7505, + "theoretical_loss": 4.160155201991238, + "tokens_seen": 309264384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007323410230692076, + "loss": 1.7729, + "theoretical_loss": 4.160052994791841, + "tokens_seen": 309329920 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007323249749247744, + "loss": 1.6711, + "theoretical_loss": 4.159950815305893, + "tokens_seen": 309395456 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732308926780341, + "loss": 1.6562, + "theoretical_loss": 4.1598486635200125, + "tokens_seen": 309460992 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007322928786359078, + "loss": 1.6112, + "theoretical_loss": 4.159746539420825, + "tokens_seen": 309526528 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007322768304914744, + "loss": 1.7772, + "theoretical_loss": 4.159644442994967, + "tokens_seen": 309592064 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007322607823470412, + "loss": 1.7144, + "theoretical_loss": 4.159542374229084, + "tokens_seen": 309657600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007322447342026078, + "loss": 1.6485, + "theoretical_loss": 4.159440333109832, + "tokens_seen": 309723136 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007322286860581746, + "loss": 1.6698, + "theoretical_loss": 4.159338319623874, + "tokens_seen": 309788672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007322126379137412, + "loss": 1.6082, + "theoretical_loss": 4.1592363337578835, + "tokens_seen": 309854208 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732196589769308, + "loss": 1.5997, + "theoretical_loss": 4.159134375498542, + "tokens_seen": 309919744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007321805416248746, + "loss": 1.6641, + "theoretical_loss": 4.159032444832543, + "tokens_seen": 309985280 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007321644934804414, + "loss": 1.6856, + "theoretical_loss": 4.158930541746587, + "tokens_seen": 310050816 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732148445336008, + "loss": 1.6824, + "theoretical_loss": 4.158828666227383, + "tokens_seen": 310116352 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007321323971915748, + "loss": 1.6742, + "theoretical_loss": 4.158726818261652, + "tokens_seen": 310181888 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007321163490471414, + "loss": 1.6513, + "theoretical_loss": 4.158624997836123, + "tokens_seen": 310247424 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007321003009027082, + "loss": 1.6443, + "theoretical_loss": 4.158523204937531, + "tokens_seen": 310312960 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007320842527582748, + "loss": 1.7374, + "theoretical_loss": 4.158421439552626, + "tokens_seen": 310378496 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007320682046138416, + "loss": 1.6499, + "theoretical_loss": 4.158319701668162, + "tokens_seen": 310444032 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007320521564694082, + "loss": 1.5736, + "theoretical_loss": 4.158217991270906, + "tokens_seen": 310509568 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732036108324975, + "loss": 1.7804, + "theoretical_loss": 4.158116308347632, + "tokens_seen": 310575104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007320200601805416, + "loss": 1.822, + "theoretical_loss": 4.1580146528851225, + "tokens_seen": 310640640 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007320040120361084, + "loss": 1.6623, + "theoretical_loss": 4.157913024870172, + "tokens_seen": 310706176 + }, + { + "epoch": 0.09, + "learning_rate": 0.000731987963891675, + "loss": 1.6879, + "theoretical_loss": 4.157811424289582, + "tokens_seen": 310771712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007319719157472418, + "loss": 1.6308, + "theoretical_loss": 4.1577098511301624, + "tokens_seen": 310837248 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007319558676028085, + "loss": 1.8068, + "theoretical_loss": 4.157608305378735, + "tokens_seen": 310902784 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007319398194583752, + "loss": 1.6752, + "theoretical_loss": 4.157506787022128, + "tokens_seen": 310968320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007319237713139419, + "loss": 1.8526, + "theoretical_loss": 4.15740529604718, + "tokens_seen": 311033856 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007319077231695086, + "loss": 1.7555, + "theoretical_loss": 4.15730383244074, + "tokens_seen": 311099392 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007318916750250753, + "loss": 1.782, + "theoretical_loss": 4.157202396189662, + "tokens_seen": 311164928 + }, + { + "epoch": 0.09, + "learning_rate": 0.000731875626880642, + "loss": 1.4557, + "theoretical_loss": 4.157100987280814, + "tokens_seen": 311230464 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.9924384355545044, + "objective/train/avg_token_score": 0.007561549544334412, + "objective/train/docs_used": 120076, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.8087691068649292, + "objective/train/original_loss": 1.8233898878097534, + "objective/train/theoretical_loss": 4.15699960570107, + "objective/train/tokens_used": 331756000, + "objective/train/unlikelihood_loss": 0.013250382617115974, + "theoretical_loss": 4.15699960570107, + "tokens_seen": 311296000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007318595787362087, + "loss": 1.6826, + "theoretical_loss": 4.15699960570107, + "tokens_seen": 311296000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007318435305917754, + "loss": 1.7021, + "theoretical_loss": 4.156898251437312, + "tokens_seen": 311361536 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007318274824473421, + "loss": 1.6512, + "theoretical_loss": 4.156796924476435, + "tokens_seen": 311427072 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007318114343029088, + "loss": 1.7148, + "theoretical_loss": 4.156695624805341, + "tokens_seen": 311492608 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007317953861584755, + "loss": 1.7651, + "theoretical_loss": 4.15659435241094, + "tokens_seen": 311558144 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007317793380140422, + "loss": 1.696, + "theoretical_loss": 4.15649310728015, + "tokens_seen": 311623680 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007317632898696089, + "loss": 1.6603, + "theoretical_loss": 4.156391889399904, + "tokens_seen": 311689216 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007317472417251756, + "loss": 1.5367, + "theoretical_loss": 4.156290698757139, + "tokens_seen": 311754752 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007317311935807423, + "loss": 1.7376, + "theoretical_loss": 4.156189535338799, + "tokens_seen": 311820288 + }, + { + "epoch": 0.09, + "learning_rate": 0.000731715145436309, + "loss": 1.6819, + "theoretical_loss": 4.156088399131844, + "tokens_seen": 311885824 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007316990972918757, + "loss": 1.7636, + "theoretical_loss": 4.155987290123236, + "tokens_seen": 311951360 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007316830491474424, + "loss": 1.6966, + "theoretical_loss": 4.15588620829995, + "tokens_seen": 312016896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007316670010030091, + "loss": 1.749, + "theoretical_loss": 4.15578515364897, + "tokens_seen": 312082432 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007316509528585758, + "loss": 1.7625, + "theoretical_loss": 4.155684126157286, + "tokens_seen": 312147968 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007316349047141425, + "loss": 1.802, + "theoretical_loss": 4.1555831258119005, + "tokens_seen": 312213504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007316188565697092, + "loss": 1.8664, + "theoretical_loss": 4.155482152599823, + "tokens_seen": 312279040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007316028084252759, + "loss": 1.5582, + "theoretical_loss": 4.155381206508071, + "tokens_seen": 312344576 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007315867602808425, + "loss": 1.7214, + "theoretical_loss": 4.155280287523674, + "tokens_seen": 312410112 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007315707121364093, + "loss": 1.5971, + "theoretical_loss": 4.155179395633667, + "tokens_seen": 312475648 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007315546639919759, + "loss": 1.7175, + "theoretical_loss": 4.1550785308250955, + "tokens_seen": 312541184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007315386158475427, + "loss": 1.531, + "theoretical_loss": 4.154977693085016, + "tokens_seen": 312606720 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007315225677031093, + "loss": 1.6529, + "theoretical_loss": 4.154876882400489, + "tokens_seen": 312672256 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007315065195586761, + "loss": 1.6848, + "theoretical_loss": 4.154776098758589, + "tokens_seen": 312737792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007314904714142427, + "loss": 1.6103, + "theoretical_loss": 4.154675342146397, + "tokens_seen": 312803328 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007314744232698095, + "loss": 1.7681, + "theoretical_loss": 4.154574612551001, + "tokens_seen": 312868864 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007314583751253761, + "loss": 1.6891, + "theoretical_loss": 4.154473909959503, + "tokens_seen": 312934400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007314423269809429, + "loss": 1.7937, + "theoretical_loss": 4.154373234359007, + "tokens_seen": 312999936 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007314262788365095, + "loss": 1.8323, + "theoretical_loss": 4.154272585736632, + "tokens_seen": 313065472 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007314102306920763, + "loss": 1.7509, + "theoretical_loss": 4.154171964079504, + "tokens_seen": 313131008 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007313941825476429, + "loss": 1.5254, + "theoretical_loss": 4.154071369374755, + "tokens_seen": 313196544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007313781344032097, + "loss": 1.6313, + "theoretical_loss": 4.15397080160953, + "tokens_seen": 313262080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007313620862587763, + "loss": 1.8143, + "theoretical_loss": 4.153870260770979, + "tokens_seen": 313327616 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007313460381143431, + "loss": 1.7735, + "theoretical_loss": 4.153769746846265, + "tokens_seen": 313393152 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007313299899699097, + "loss": 1.6819, + "theoretical_loss": 4.153669259822556, + "tokens_seen": 313458688 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007313139418254765, + "loss": 1.7067, + "theoretical_loss": 4.153568799687031, + "tokens_seen": 313524224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007312978936810431, + "loss": 1.7287, + "theoretical_loss": 4.153468366426877, + "tokens_seen": 313589760 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007312818455366099, + "loss": 1.6785, + "theoretical_loss": 4.153367960029288, + "tokens_seen": 313655296 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007312657973921765, + "loss": 1.693, + "theoretical_loss": 4.153267580481472, + "tokens_seen": 313720832 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007312497492477433, + "loss": 1.9243, + "theoretical_loss": 4.15316722777064, + "tokens_seen": 313786368 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007312337011033099, + "loss": 1.7228, + "theoretical_loss": 4.153066901884016, + "tokens_seen": 313851904 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007312176529588767, + "loss": 1.8097, + "theoretical_loss": 4.15296660280883, + "tokens_seen": 313917440 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007312016048144433, + "loss": 1.7347, + "theoretical_loss": 4.152866330532321, + "tokens_seen": 313982976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007311855566700101, + "loss": 1.7312, + "theoretical_loss": 4.152766085041737, + "tokens_seen": 314048512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007311695085255767, + "loss": 1.6604, + "theoretical_loss": 4.152665866324337, + "tokens_seen": 314114048 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007311534603811435, + "loss": 1.7484, + "theoretical_loss": 4.152565674367387, + "tokens_seen": 314179584 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007311374122367101, + "loss": 1.7185, + "theoretical_loss": 4.15246550915816, + "tokens_seen": 314245120 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007311213640922769, + "loss": 1.692, + "theoretical_loss": 4.15236537068394, + "tokens_seen": 314310656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007311053159478435, + "loss": 1.7648, + "theoretical_loss": 4.152265258932019, + "tokens_seen": 314376192 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007310892678034103, + "loss": 1.6932, + "theoretical_loss": 4.152165173889697, + "tokens_seen": 314441728 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007310732196589769, + "loss": 1.6059, + "theoretical_loss": 4.152065115544284, + "tokens_seen": 314507264 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.9928939342498779, + "objective/train/avg_token_score": 0.007106083910912275, + "objective/train/docs_used": 121169, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.7189292907714844, + "objective/train/original_loss": 1.7328643798828125, + "objective/train/theoretical_loss": 4.151965083883098, + "objective/train/tokens_used": 335032800, + "objective/train/unlikelihood_loss": 0.0137941287830472, + "theoretical_loss": 4.151965083883098, + "tokens_seen": 314572800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007310571715145437, + "loss": 1.6437, + "theoretical_loss": 4.151965083883098, + "tokens_seen": 314572800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007310411233701103, + "loss": 1.6844, + "theoretical_loss": 4.151865078893466, + "tokens_seen": 314638336 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007310250752256771, + "loss": 1.6474, + "theoretical_loss": 4.151765100562723, + "tokens_seen": 314703872 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007310090270812437, + "loss": 1.6734, + "theoretical_loss": 4.151665148878212, + "tokens_seen": 314769408 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007309929789368105, + "loss": 1.7932, + "theoretical_loss": 4.151565223827288, + "tokens_seen": 314834944 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007309769307923771, + "loss": 1.6534, + "theoretical_loss": 4.15146532539731, + "tokens_seen": 314900480 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007309608826479439, + "loss": 1.7135, + "theoretical_loss": 4.151365453575648, + "tokens_seen": 314966016 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007309448345035105, + "loss": 1.7031, + "theoretical_loss": 4.151265608349682, + "tokens_seen": 315031552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007309287863590773, + "loss": 1.751, + "theoretical_loss": 4.151165789706798, + "tokens_seen": 315097088 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007309127382146439, + "loss": 1.7149, + "theoretical_loss": 4.151065997634393, + "tokens_seen": 315162624 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007308966900702107, + "loss": 1.6658, + "theoretical_loss": 4.1509662321198695, + "tokens_seen": 315228160 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007308806419257773, + "loss": 1.7945, + "theoretical_loss": 4.150866493150642, + "tokens_seen": 315293696 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007308645937813441, + "loss": 1.7234, + "theoretical_loss": 4.150766780714131, + "tokens_seen": 315359232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007308485456369107, + "loss": 1.6854, + "theoretical_loss": 4.150667094797766, + "tokens_seen": 315424768 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007308324974924775, + "loss": 1.6726, + "theoretical_loss": 4.150567435388988, + "tokens_seen": 315490304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007308164493480441, + "loss": 1.6745, + "theoretical_loss": 4.150467802475241, + "tokens_seen": 315555840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007308004012036109, + "loss": 1.6833, + "theoretical_loss": 4.150368196043983, + "tokens_seen": 315621376 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007307843530591775, + "loss": 1.7065, + "theoretical_loss": 4.150268616082678, + "tokens_seen": 315686912 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007307683049147443, + "loss": 1.5885, + "theoretical_loss": 4.150169062578798, + "tokens_seen": 315752448 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007307522567703109, + "loss": 1.6819, + "theoretical_loss": 4.150069535519825, + "tokens_seen": 315817984 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007307362086258777, + "loss": 1.6535, + "theoretical_loss": 4.149970034893249, + "tokens_seen": 315883520 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007307201604814443, + "loss": 1.7075, + "theoretical_loss": 4.149870560686567, + "tokens_seen": 315949056 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007307041123370111, + "loss": 2.0313, + "theoretical_loss": 4.149771112887288, + "tokens_seen": 316014592 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007306880641925777, + "loss": 1.7475, + "theoretical_loss": 4.149671691482926, + "tokens_seen": 316080128 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007306720160481445, + "loss": 1.6612, + "theoretical_loss": 4.149572296461005, + "tokens_seen": 316145664 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007306559679037111, + "loss": 1.6449, + "theoretical_loss": 4.149472927809057, + "tokens_seen": 316211200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007306399197592779, + "loss": 1.6774, + "theoretical_loss": 4.149373585514625, + "tokens_seen": 316276736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007306238716148446, + "loss": 1.5625, + "theoretical_loss": 4.149274269565256, + "tokens_seen": 316342272 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007306078234704113, + "loss": 1.6705, + "theoretical_loss": 4.149174979948507, + "tokens_seen": 316407808 + }, + { + "epoch": 0.1, + "learning_rate": 0.000730591775325978, + "loss": 1.5533, + "theoretical_loss": 4.1490757166519465, + "tokens_seen": 316473344 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007305757271815447, + "loss": 1.7669, + "theoretical_loss": 4.148976479663148, + "tokens_seen": 316538880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007305596790371114, + "loss": 1.6759, + "theoretical_loss": 4.148877268969694, + "tokens_seen": 316604416 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007305436308926781, + "loss": 1.7514, + "theoretical_loss": 4.1487780845591775, + "tokens_seen": 316669952 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007305275827482448, + "loss": 1.6598, + "theoretical_loss": 4.148678926419197, + "tokens_seen": 316735488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007305115346038115, + "loss": 1.6772, + "theoretical_loss": 4.148579794537362, + "tokens_seen": 316801024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007304954864593782, + "loss": 1.7633, + "theoretical_loss": 4.148480688901288, + "tokens_seen": 316866560 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007304794383149449, + "loss": 1.6617, + "theoretical_loss": 4.148381609498602, + "tokens_seen": 316932096 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007304633901705116, + "loss": 1.71, + "theoretical_loss": 4.148282556316936, + "tokens_seen": 316997632 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007304473420260783, + "loss": 1.7426, + "theoretical_loss": 4.148183529343932, + "tokens_seen": 317063168 + }, + { + "epoch": 0.1, + "learning_rate": 0.000730431293881645, + "loss": 1.6482, + "theoretical_loss": 4.148084528567241, + "tokens_seen": 317128704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007304152457372117, + "loss": 1.6709, + "theoretical_loss": 4.147985553974522, + "tokens_seen": 317194240 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007303991975927784, + "loss": 1.6408, + "theoretical_loss": 4.14788660555344, + "tokens_seen": 317259776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007303831494483451, + "loss": 1.6693, + "theoretical_loss": 4.147787683291672, + "tokens_seen": 317325312 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007303671013039118, + "loss": 1.6477, + "theoretical_loss": 4.147688787176902, + "tokens_seen": 317390848 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007303510531594785, + "loss": 1.7974, + "theoretical_loss": 4.147589917196821, + "tokens_seen": 317456384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007303350050150452, + "loss": 1.6924, + "theoretical_loss": 4.14749107333913, + "tokens_seen": 317521920 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007303189568706119, + "loss": 1.5776, + "theoretical_loss": 4.147392255591538, + "tokens_seen": 317587456 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007303029087261786, + "loss": 1.6484, + "theoretical_loss": 4.147293463941762, + "tokens_seen": 317652992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007302868605817453, + "loss": 1.6291, + "theoretical_loss": 4.147194698377526, + "tokens_seen": 317718528 + }, + { + "epoch": 0.1, + "learning_rate": 0.000730270812437312, + "loss": 1.8467, + "theoretical_loss": 4.147095958886566, + "tokens_seen": 317784064 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.9945030212402344, + "objective/train/avg_token_score": 0.0054969461634755135, + "objective/train/docs_used": 122440, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.545818567276001, + "objective/train/original_loss": 1.554169774055481, + "objective/train/theoretical_loss": 4.146997245456623, + "objective/train/tokens_used": 338309600, + "objective/train/unlikelihood_loss": 0.012868283316493034, + "theoretical_loss": 4.146997245456623, + "tokens_seen": 317849600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007302547642928787, + "loss": 1.6827, + "theoretical_loss": 4.146997245456623, + "tokens_seen": 317849600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007302387161484454, + "loss": 1.758, + "theoretical_loss": 4.146898558075446, + "tokens_seen": 317915136 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007302226680040121, + "loss": 1.6643, + "theoretical_loss": 4.146799896730794, + "tokens_seen": 317980672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007302066198595788, + "loss": 1.6198, + "theoretical_loss": 4.146701261410435, + "tokens_seen": 318046208 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007301905717151454, + "loss": 1.5603, + "theoretical_loss": 4.1466026521021435, + "tokens_seen": 318111744 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007301745235707122, + "loss": 1.737, + "theoretical_loss": 4.146504068793703, + "tokens_seen": 318177280 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007301584754262788, + "loss": 1.5652, + "theoretical_loss": 4.146405511472905, + "tokens_seen": 318242816 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007301424272818456, + "loss": 1.6592, + "theoretical_loss": 4.146306980127548, + "tokens_seen": 318308352 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007301263791374122, + "loss": 1.6625, + "theoretical_loss": 4.146208474745442, + "tokens_seen": 318373888 + }, + { + "epoch": 0.1, + "learning_rate": 0.000730110330992979, + "loss": 1.6419, + "theoretical_loss": 4.1461099953144025, + "tokens_seen": 318439424 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007300942828485456, + "loss": 1.7244, + "theoretical_loss": 4.146011541822253, + "tokens_seen": 318504960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007300782347041124, + "loss": 1.7734, + "theoretical_loss": 4.145913114256829, + "tokens_seen": 318570496 + }, + { + "epoch": 0.1, + "learning_rate": 0.000730062186559679, + "loss": 1.7295, + "theoretical_loss": 4.145814712605969, + "tokens_seen": 318636032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007300461384152458, + "loss": 1.7037, + "theoretical_loss": 4.1457163368575225, + "tokens_seen": 318701568 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007300300902708124, + "loss": 1.64, + "theoretical_loss": 4.145617986999348, + "tokens_seen": 318767104 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007300140421263792, + "loss": 1.7316, + "theoretical_loss": 4.145519663019309, + "tokens_seen": 318832640 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007299979939819458, + "loss": 1.6914, + "theoretical_loss": 4.145421364905282, + "tokens_seen": 318898176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007299819458375126, + "loss": 1.8164, + "theoretical_loss": 4.145323092645146, + "tokens_seen": 318963712 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007299658976930792, + "loss": 1.644, + "theoretical_loss": 4.145224846226793, + "tokens_seen": 319029248 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729949849548646, + "loss": 1.7028, + "theoretical_loss": 4.14512662563812, + "tokens_seen": 319094784 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007299338014042126, + "loss": 1.6647, + "theoretical_loss": 4.145028430867035, + "tokens_seen": 319160320 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007299177532597794, + "loss": 1.7031, + "theoretical_loss": 4.144930261901451, + "tokens_seen": 319225856 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729901705115346, + "loss": 1.5496, + "theoretical_loss": 4.144832118729291, + "tokens_seen": 319291392 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007298856569709128, + "loss": 1.6798, + "theoretical_loss": 4.144734001338486, + "tokens_seen": 319356928 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007298696088264794, + "loss": 1.7092, + "theoretical_loss": 4.144635909716976, + "tokens_seen": 319422464 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007298535606820462, + "loss": 1.6861, + "theoretical_loss": 4.144537843852705, + "tokens_seen": 319488000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007298375125376128, + "loss": 1.8418, + "theoretical_loss": 4.144439803733631, + "tokens_seen": 319553536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007298214643931796, + "loss": 1.6973, + "theoretical_loss": 4.144341789347715, + "tokens_seen": 319619072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007298054162487462, + "loss": 1.7865, + "theoretical_loss": 4.144243800682931, + "tokens_seen": 319684608 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729789368104313, + "loss": 1.559, + "theoretical_loss": 4.1441458377272555, + "tokens_seen": 319750144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007297733199598796, + "loss": 1.595, + "theoretical_loss": 4.144047900468678, + "tokens_seen": 319815680 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007297572718154464, + "loss": 1.8507, + "theoretical_loss": 4.143949988895193, + "tokens_seen": 319881216 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729741223671013, + "loss": 1.6887, + "theoretical_loss": 4.1438521029948046, + "tokens_seen": 319946752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007297251755265798, + "loss": 1.7008, + "theoretical_loss": 4.1437542427555245, + "tokens_seen": 320012288 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007297091273821464, + "loss": 1.6923, + "theoretical_loss": 4.143656408165373, + "tokens_seen": 320077824 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007296930792377132, + "loss": 1.7964, + "theoretical_loss": 4.143558599212376, + "tokens_seen": 320143360 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007296770310932798, + "loss": 1.795, + "theoretical_loss": 4.143460815884571, + "tokens_seen": 320208896 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007296609829488466, + "loss": 1.8024, + "theoretical_loss": 4.143363058170001, + "tokens_seen": 320274432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007296449348044132, + "loss": 1.5916, + "theoretical_loss": 4.143265326056719, + "tokens_seen": 320339968 + }, + { + "epoch": 0.1, + "learning_rate": 0.00072962888665998, + "loss": 1.6628, + "theoretical_loss": 4.1431676195327825, + "tokens_seen": 320405504 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007296128385155466, + "loss": 1.5872, + "theoretical_loss": 4.143069938586263, + "tokens_seen": 320471040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007295967903711134, + "loss": 1.654, + "theoretical_loss": 4.142972283205233, + "tokens_seen": 320536576 + }, + { + "epoch": 0.1, + "learning_rate": 0.00072958074222668, + "loss": 1.7346, + "theoretical_loss": 4.142874653377778, + "tokens_seen": 320602112 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007295646940822468, + "loss": 1.77, + "theoretical_loss": 4.14277704909199, + "tokens_seen": 320667648 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007295486459378134, + "loss": 1.6925, + "theoretical_loss": 4.142679470335969, + "tokens_seen": 320733184 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007295325977933802, + "loss": 1.6564, + "theoretical_loss": 4.142581917097821, + "tokens_seen": 320798720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007295165496489468, + "loss": 1.6108, + "theoretical_loss": 4.142484389365664, + "tokens_seen": 320864256 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007295005015045136, + "loss": 1.6624, + "theoretical_loss": 4.14238688712762, + "tokens_seen": 320929792 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007294844533600802, + "loss": 1.5902, + "theoretical_loss": 4.142289410371823, + "tokens_seen": 320995328 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729468405215647, + "loss": 1.6008, + "theoretical_loss": 4.14219195908641, + "tokens_seen": 321060864 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.9888757467269897, + "objective/train/avg_token_score": 0.011124251410365105, + "objective/train/docs_used": 123582, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.68244206905365, + "objective/train/original_loss": 1.700392484664917, + "objective/train/theoretical_loss": 4.14209453325953, + "objective/train/tokens_used": 341586400, + "objective/train/unlikelihood_loss": 0.024884361773729324, + "theoretical_loss": 4.14209453325953, + "tokens_seen": 321126400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007294523570712136, + "loss": 1.6973, + "theoretical_loss": 4.14209453325953, + "tokens_seen": 321126400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007294363089267804, + "loss": 1.6871, + "theoretical_loss": 4.1419971328793395, + "tokens_seen": 321191936 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729420260782347, + "loss": 1.4758, + "theoretical_loss": 4.141899757934, + "tokens_seen": 321257472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007294042126379138, + "loss": 1.5702, + "theoretical_loss": 4.141802408411683, + "tokens_seen": 321323008 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007293881644934804, + "loss": 1.6466, + "theoretical_loss": 4.14170508430057, + "tokens_seen": 321388544 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007293721163490472, + "loss": 1.7472, + "theoretical_loss": 4.141607785588846, + "tokens_seen": 321454080 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007293560682046138, + "loss": 1.6864, + "theoretical_loss": 4.141510512264707, + "tokens_seen": 321519616 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007293400200601806, + "loss": 1.564, + "theoretical_loss": 4.141413264316356, + "tokens_seen": 321585152 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007293239719157472, + "loss": 1.7068, + "theoretical_loss": 4.141316041732004, + "tokens_seen": 321650688 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729307923771314, + "loss": 1.734, + "theoretical_loss": 4.141218844499869, + "tokens_seen": 321716224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007292918756268806, + "loss": 1.7094, + "theoretical_loss": 4.141121672608178, + "tokens_seen": 321781760 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007292758274824474, + "loss": 1.646, + "theoretical_loss": 4.141024526045166, + "tokens_seen": 321847296 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729259779338014, + "loss": 1.5821, + "theoretical_loss": 4.140927404799075, + "tokens_seen": 321912832 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007292437311935808, + "loss": 1.6792, + "theoretical_loss": 4.140830308858154, + "tokens_seen": 321978368 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007292276830491475, + "loss": 1.8076, + "theoretical_loss": 4.140733238210664, + "tokens_seen": 322043904 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007292116349047142, + "loss": 1.6709, + "theoretical_loss": 4.140636192844867, + "tokens_seen": 322109440 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007291955867602809, + "loss": 1.6155, + "theoretical_loss": 4.1405391727490395, + "tokens_seen": 322174976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007291795386158476, + "loss": 1.676, + "theoretical_loss": 4.140442177911462, + "tokens_seen": 322240512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007291634904714143, + "loss": 1.7846, + "theoretical_loss": 4.140345208320425, + "tokens_seen": 322306048 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729147442326981, + "loss": 1.501, + "theoretical_loss": 4.1402482639642235, + "tokens_seen": 322371584 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007291313941825477, + "loss": 1.6903, + "theoretical_loss": 4.140151344831164, + "tokens_seen": 322437120 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007291153460381144, + "loss": 1.6474, + "theoretical_loss": 4.14005445090956, + "tokens_seen": 322502656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007290992978936812, + "loss": 1.7194, + "theoretical_loss": 4.13995758218773, + "tokens_seen": 322568192 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007290832497492478, + "loss": 1.721, + "theoretical_loss": 4.1398607386540025, + "tokens_seen": 322633728 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007290672016048146, + "loss": 1.6455, + "theoretical_loss": 4.139763920296716, + "tokens_seen": 322699264 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007290511534603812, + "loss": 1.7967, + "theoretical_loss": 4.139667127104211, + "tokens_seen": 322764800 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729035105315948, + "loss": 1.6616, + "theoretical_loss": 4.139570359064842, + "tokens_seen": 322830336 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007290190571715146, + "loss": 1.6487, + "theoretical_loss": 4.139473616166967, + "tokens_seen": 322895872 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007290030090270814, + "loss": 1.5787, + "theoretical_loss": 4.139376898398953, + "tokens_seen": 322961408 + }, + { + "epoch": 0.1, + "learning_rate": 0.000728986960882648, + "loss": 1.6309, + "theoretical_loss": 4.139280205749175, + "tokens_seen": 323026944 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007289709127382148, + "loss": 1.5857, + "theoretical_loss": 4.139183538206017, + "tokens_seen": 323092480 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007289548645937814, + "loss": 1.6029, + "theoretical_loss": 4.139086895757868, + "tokens_seen": 323158016 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007289388164493481, + "loss": 1.6982, + "theoretical_loss": 4.138990278393125, + "tokens_seen": 323223552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007289227683049148, + "loss": 1.5809, + "theoretical_loss": 4.138893686100196, + "tokens_seen": 323289088 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007289067201604815, + "loss": 1.7943, + "theoretical_loss": 4.138797118867493, + "tokens_seen": 323354624 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007288906720160482, + "loss": 1.5618, + "theoretical_loss": 4.138700576683437, + "tokens_seen": 323420160 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007288746238716149, + "loss": 1.6345, + "theoretical_loss": 4.138604059536458, + "tokens_seen": 323485696 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007288585757271816, + "loss": 1.6509, + "theoretical_loss": 4.138507567414993, + "tokens_seen": 323551232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007288425275827483, + "loss": 1.7643, + "theoretical_loss": 4.138411100307485, + "tokens_seen": 323616768 + }, + { + "epoch": 0.1, + "learning_rate": 0.000728826479438315, + "loss": 1.5158, + "theoretical_loss": 4.138314658202385, + "tokens_seen": 323682304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007288104312938817, + "loss": 1.6753, + "theoretical_loss": 4.138218241088156, + "tokens_seen": 323747840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007287943831494483, + "loss": 1.5479, + "theoretical_loss": 4.138121848953261, + "tokens_seen": 323813376 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007287783350050151, + "loss": 1.614, + "theoretical_loss": 4.138025481786178, + "tokens_seen": 323878912 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007287622868605817, + "loss": 1.6433, + "theoretical_loss": 4.137929139575387, + "tokens_seen": 323944448 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007287462387161485, + "loss": 1.6475, + "theoretical_loss": 4.137832822309382, + "tokens_seen": 324009984 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007287301905717151, + "loss": 1.7056, + "theoretical_loss": 4.1377365299766575, + "tokens_seen": 324075520 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007287141424272819, + "loss": 1.5505, + "theoretical_loss": 4.13764026256572, + "tokens_seen": 324141056 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007286980942828485, + "loss": 1.6432, + "theoretical_loss": 4.137544020065082, + "tokens_seen": 324206592 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007286820461384153, + "loss": 1.5571, + "theoretical_loss": 4.1374478024632655, + "tokens_seen": 324272128 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007286659979939819, + "loss": 1.6591, + "theoretical_loss": 4.137351609748798, + "tokens_seen": 324337664 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.995570719242096, + "objective/train/avg_token_score": 0.004429239314049482, + "objective/train/docs_used": 124782, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5955121517181396, + "objective/train/original_loss": 1.6038167476654053, + "objective/train/theoretical_loss": 4.137255441910215, + "objective/train/tokens_used": 344863200, + "objective/train/unlikelihood_loss": 0.009204545058310032, + "theoretical_loss": 4.137255441910215, + "tokens_seen": 324403200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007286499498495487, + "loss": 1.6317, + "theoretical_loss": 4.137255441910215, + "tokens_seen": 324403200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007286339017051153, + "loss": 1.6259, + "theoretical_loss": 4.13715929893606, + "tokens_seen": 324468736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007286178535606821, + "loss": 1.6819, + "theoretical_loss": 4.137063180814886, + "tokens_seen": 324534272 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007286018054162487, + "loss": 1.7197, + "theoretical_loss": 4.136967087535249, + "tokens_seen": 324599808 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007285857572718155, + "loss": 1.6048, + "theoretical_loss": 4.136871019085717, + "tokens_seen": 324665344 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007285697091273821, + "loss": 1.631, + "theoretical_loss": 4.136774975454863, + "tokens_seen": 324730880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007285536609829489, + "loss": 1.7708, + "theoretical_loss": 4.136678956631269, + "tokens_seen": 324796416 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007285376128385155, + "loss": 1.6467, + "theoretical_loss": 4.136582962603523, + "tokens_seen": 324861952 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007285215646940823, + "loss": 1.6158, + "theoretical_loss": 4.136486993360222, + "tokens_seen": 324927488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007285055165496489, + "loss": 1.6901, + "theoretical_loss": 4.136391048889971, + "tokens_seen": 324993024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007284894684052157, + "loss": 1.6009, + "theoretical_loss": 4.136295129181381, + "tokens_seen": 325058560 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007284734202607823, + "loss": 1.5675, + "theoretical_loss": 4.136199234223069, + "tokens_seen": 325124096 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007284573721163491, + "loss": 1.7291, + "theoretical_loss": 4.136103364003665, + "tokens_seen": 325189632 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007284413239719157, + "loss": 1.6137, + "theoretical_loss": 4.1360075185118, + "tokens_seen": 325255168 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007284252758274825, + "loss": 1.6817, + "theoretical_loss": 4.135911697736118, + "tokens_seen": 325320704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007284092276830491, + "loss": 1.6688, + "theoretical_loss": 4.135815901665267, + "tokens_seen": 325386240 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007283931795386159, + "loss": 1.6158, + "theoretical_loss": 4.135720130287904, + "tokens_seen": 325451776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007283771313941825, + "loss": 1.6699, + "theoretical_loss": 4.135624383592694, + "tokens_seen": 325517312 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007283610832497493, + "loss": 1.6034, + "theoretical_loss": 4.135528661568307, + "tokens_seen": 325582848 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007283450351053159, + "loss": 1.7347, + "theoretical_loss": 4.135432964203422, + "tokens_seen": 325648384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007283289869608827, + "loss": 1.6318, + "theoretical_loss": 4.135337291486727, + "tokens_seen": 325713920 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007283129388164493, + "loss": 1.6508, + "theoretical_loss": 4.135241643406915, + "tokens_seen": 325779456 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007282968906720161, + "loss": 1.7001, + "theoretical_loss": 4.135146019952688, + "tokens_seen": 325844992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007282808425275827, + "loss": 1.5765, + "theoretical_loss": 4.135050421112755, + "tokens_seen": 325910528 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007282647943831495, + "loss": 1.7209, + "theoretical_loss": 4.134954846875832, + "tokens_seen": 325976064 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007282487462387161, + "loss": 1.751, + "theoretical_loss": 4.134859297230644, + "tokens_seen": 326041600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007282326980942829, + "loss": 1.568, + "theoretical_loss": 4.13476377216592, + "tokens_seen": 326107136 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007282166499498495, + "loss": 1.7058, + "theoretical_loss": 4.134668271670401, + "tokens_seen": 326172672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007282006018054163, + "loss": 1.6569, + "theoretical_loss": 4.134572795732832, + "tokens_seen": 326238208 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007281845536609829, + "loss": 1.5632, + "theoretical_loss": 4.134477344341967, + "tokens_seen": 326303744 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007281685055165497, + "loss": 1.6837, + "theoretical_loss": 4.134381917486565, + "tokens_seen": 326369280 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007281524573721163, + "loss": 1.6918, + "theoretical_loss": 4.134286515155398, + "tokens_seen": 326434816 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007281364092276831, + "loss": 1.5725, + "theoretical_loss": 4.134191137337238, + "tokens_seen": 326500352 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007281203610832497, + "loss": 1.6762, + "theoretical_loss": 4.134095784020871, + "tokens_seen": 326565888 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007281043129388165, + "loss": 1.7294, + "theoretical_loss": 4.1340004551950855, + "tokens_seen": 326631424 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007280882647943831, + "loss": 1.5545, + "theoretical_loss": 4.133905150848681, + "tokens_seen": 326696960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007280722166499499, + "loss": 1.6067, + "theoretical_loss": 4.133809870970461, + "tokens_seen": 326762496 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007280561685055165, + "loss": 1.6151, + "theoretical_loss": 4.1337146155492395, + "tokens_seen": 326828032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007280401203610833, + "loss": 1.5339, + "theoretical_loss": 4.133619384573836, + "tokens_seen": 326893568 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007280240722166499, + "loss": 1.5488, + "theoretical_loss": 4.1335241780330785, + "tokens_seen": 326959104 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007280080240722167, + "loss": 1.6349, + "theoretical_loss": 4.1334289959158, + "tokens_seen": 327024640 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007279919759277833, + "loss": 1.6421, + "theoretical_loss": 4.133333838210844, + "tokens_seen": 327090176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007279759277833501, + "loss": 1.7595, + "theoretical_loss": 4.133238704907059, + "tokens_seen": 327155712 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007279598796389167, + "loss": 1.895, + "theoretical_loss": 4.133143595993303, + "tokens_seen": 327221248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007279438314944835, + "loss": 1.7031, + "theoretical_loss": 4.133048511458438, + "tokens_seen": 327286784 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007279277833500503, + "loss": 1.6871, + "theoretical_loss": 4.132953451291337, + "tokens_seen": 327352320 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007279117352056169, + "loss": 1.6859, + "theoretical_loss": 4.132858415480877, + "tokens_seen": 327417856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007278956870611837, + "loss": 1.7771, + "theoretical_loss": 4.132763404015946, + "tokens_seen": 327483392 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007278796389167503, + "loss": 1.6531, + "theoretical_loss": 4.132668416885435, + "tokens_seen": 327548928 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007278635907723171, + "loss": 1.6754, + "theoretical_loss": 4.132573454078246, + "tokens_seen": 327614464 + }, + { + "debugging/Compilability": 1.0, + "debugging/distinct-1-grams": 0.7246169962973871, + "debugging/entropy-1-grams": 5.13190351820591, + "debugging/length": 472.8, + "debugging/num_segments": 10, + "debugging/score": 0.0033104506818385365, + "debugging/score_std": 0.0017105448100298692, + "epoch": 0.1, + "objective/train/aligned_token_count": 0.9913387894630432, + "objective/train/avg_token_score": 0.008661167696118355, + "objective/train/docs_used": 126013, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.742929220199585, + "objective/train/original_loss": 1.7563227415084839, + "objective/train/theoretical_loss": 4.132478515583286, + "objective/train/tokens_used": 348140000, + "objective/train/unlikelihood_loss": 0.019969847053289413, + "theoretical_loss": 4.132478515583286, + "tokens_seen": 327680000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007278475426278837, + "loss": 1.765, + "theoretical_loss": 4.132478515583286, + "tokens_seen": 327680000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007278314944834505, + "loss": 1.6639, + "theoretical_loss": 4.132383601389471, + "tokens_seen": 327745536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007278154463390171, + "loss": 1.7519, + "theoretical_loss": 4.132288711485721, + "tokens_seen": 327811072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007277993981945839, + "loss": 1.7125, + "theoretical_loss": 4.132193845860969, + "tokens_seen": 327876608 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007277833500501505, + "loss": 1.6475, + "theoretical_loss": 4.132099004504149, + "tokens_seen": 327942144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007277673019057173, + "loss": 1.767, + "theoretical_loss": 4.132004187404206, + "tokens_seen": 328007680 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007277512537612839, + "loss": 1.632, + "theoretical_loss": 4.131909394550092, + "tokens_seen": 328073216 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007277352056168507, + "loss": 1.756, + "theoretical_loss": 4.131814625930764, + "tokens_seen": 328138752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007277191574724173, + "loss": 1.6257, + "theoretical_loss": 4.13171988153519, + "tokens_seen": 328204288 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007277031093279841, + "loss": 1.5658, + "theoretical_loss": 4.131625161352342, + "tokens_seen": 328269824 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007276870611835507, + "loss": 1.8823, + "theoretical_loss": 4.131530465371199, + "tokens_seen": 328335360 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007276710130391175, + "loss": 1.6591, + "theoretical_loss": 4.131435793580749, + "tokens_seen": 328400896 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007276549648946841, + "loss": 1.5744, + "theoretical_loss": 4.131341145969989, + "tokens_seen": 328466432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007276389167502509, + "loss": 1.5238, + "theoretical_loss": 4.131246522527918, + "tokens_seen": 328531968 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007276228686058175, + "loss": 1.689, + "theoretical_loss": 4.1311519232435465, + "tokens_seen": 328597504 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007276068204613843, + "loss": 1.8063, + "theoretical_loss": 4.131057348105889, + "tokens_seen": 328663040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007275907723169509, + "loss": 1.5197, + "theoretical_loss": 4.1309627971039715, + "tokens_seen": 328728576 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007275747241725177, + "loss": 1.6698, + "theoretical_loss": 4.130868270226824, + "tokens_seen": 328794112 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007275586760280843, + "loss": 1.5666, + "theoretical_loss": 4.130773767463483, + "tokens_seen": 328859648 + }, + { + "epoch": 0.1, + "learning_rate": 0.000727542627883651, + "loss": 1.5676, + "theoretical_loss": 4.130679288802993, + "tokens_seen": 328925184 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007275265797392177, + "loss": 1.6668, + "theoretical_loss": 4.1305848342344085, + "tokens_seen": 328990720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007275105315947844, + "loss": 1.7349, + "theoretical_loss": 4.130490403746787, + "tokens_seen": 329056256 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007274944834503511, + "loss": 1.6256, + "theoretical_loss": 4.130395997329196, + "tokens_seen": 329121792 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007274784353059178, + "loss": 1.6814, + "theoretical_loss": 4.1303016149707075, + "tokens_seen": 329187328 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007274623871614845, + "loss": 1.6212, + "theoretical_loss": 4.130207256660404, + "tokens_seen": 329252864 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007274463390170512, + "loss": 1.6743, + "theoretical_loss": 4.130112922387372, + "tokens_seen": 329318400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007274302908726179, + "loss": 1.64, + "theoretical_loss": 4.130018612140707, + "tokens_seen": 329383936 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007274142427281846, + "loss": 1.6417, + "theoretical_loss": 4.12992432590951, + "tokens_seen": 329449472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007273981945837512, + "loss": 1.6384, + "theoretical_loss": 4.129830063682892, + "tokens_seen": 329515008 + }, + { + "epoch": 0.1, + "learning_rate": 0.000727382146439318, + "loss": 1.5966, + "theoretical_loss": 4.129735825449968, + "tokens_seen": 329580544 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007273660982948846, + "loss": 1.6143, + "theoretical_loss": 4.129641611199862, + "tokens_seen": 329646080 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007273500501504514, + "loss": 1.7686, + "theoretical_loss": 4.1295474209217025, + "tokens_seen": 329711616 + }, + { + "epoch": 0.1, + "learning_rate": 0.000727334002006018, + "loss": 1.6169, + "theoretical_loss": 4.12945325460463, + "tokens_seen": 329777152 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007273179538615848, + "loss": 1.6518, + "theoretical_loss": 4.129359112237787, + "tokens_seen": 329842688 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007273019057171514, + "loss": 1.6345, + "theoretical_loss": 4.1292649938103265, + "tokens_seen": 329908224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007272858575727182, + "loss": 1.6646, + "theoretical_loss": 4.129170899311406, + "tokens_seen": 329973760 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007272698094282848, + "loss": 1.786, + "theoretical_loss": 4.129076828730192, + "tokens_seen": 330039296 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007272537612838516, + "loss": 1.6834, + "theoretical_loss": 4.128982782055858, + "tokens_seen": 330104832 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007272377131394182, + "loss": 1.6108, + "theoretical_loss": 4.128888759277583, + "tokens_seen": 330170368 + }, + { + "epoch": 0.1, + "learning_rate": 0.000727221664994985, + "loss": 1.6569, + "theoretical_loss": 4.1287947603845545, + "tokens_seen": 330235904 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007272056168505516, + "loss": 1.733, + "theoretical_loss": 4.128700785365965, + "tokens_seen": 330301440 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007271895687061184, + "loss": 1.7304, + "theoretical_loss": 4.128606834211018, + "tokens_seen": 330366976 + }, + { + "epoch": 0.1, + "learning_rate": 0.000727173520561685, + "loss": 1.5784, + "theoretical_loss": 4.128512906908919, + "tokens_seen": 330432512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007271574724172518, + "loss": 1.6138, + "theoretical_loss": 4.128419003448886, + "tokens_seen": 330498048 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007271414242728184, + "loss": 1.7288, + "theoretical_loss": 4.128325123820138, + "tokens_seen": 330563584 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007271253761283852, + "loss": 1.7347, + "theoretical_loss": 4.1282312680119055, + "tokens_seen": 330629120 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007271093279839518, + "loss": 1.7161, + "theoretical_loss": 4.128137436013425, + "tokens_seen": 330694656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007270932798395186, + "loss": 1.656, + "theoretical_loss": 4.128043627813939, + "tokens_seen": 330760192 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007270772316950852, + "loss": 1.6242, + "theoretical_loss": 4.127949843402699, + "tokens_seen": 330825728 + }, + { + "epoch": 0.1, + "learning_rate": 0.000727061183550652, + "loss": 1.7901, + "theoretical_loss": 4.12785608276896, + "tokens_seen": 330891264 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.9939263463020325, + "objective/train/avg_token_score": 0.0060736555606126785, + "objective/train/docs_used": 127130, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.7217990159988403, + "objective/train/original_loss": 1.7331526279449463, + "objective/train/theoretical_loss": 4.127762345901987, + "objective/train/tokens_used": 351416800, + "objective/train/unlikelihood_loss": 0.011295117437839508, + "theoretical_loss": 4.127762345901987, + "tokens_seen": 330956800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007270451354062186, + "loss": 1.7491, + "theoretical_loss": 4.127762345901987, + "tokens_seen": 330956800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007270290872617854, + "loss": 1.6208, + "theoretical_loss": 4.1276686327910515, + "tokens_seen": 331022336 + }, + { + "epoch": 0.1, + "learning_rate": 0.000727013039117352, + "loss": 1.5759, + "theoretical_loss": 4.127574943425429, + "tokens_seen": 331087872 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007269969909729188, + "loss": 1.8123, + "theoretical_loss": 4.127481277794408, + "tokens_seen": 331153408 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007269809428284854, + "loss": 1.7407, + "theoretical_loss": 4.1273876358872785, + "tokens_seen": 331218944 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007269648946840522, + "loss": 1.6106, + "theoretical_loss": 4.127294017693339, + "tokens_seen": 331284480 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007269488465396188, + "loss": 1.738, + "theoretical_loss": 4.127200423201897, + "tokens_seen": 331350016 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007269327983951856, + "loss": 1.6437, + "theoretical_loss": 4.127106852402264, + "tokens_seen": 331415552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007269167502507522, + "loss": 1.6279, + "theoretical_loss": 4.127013305283759, + "tokens_seen": 331481088 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726900702106319, + "loss": 1.6002, + "theoretical_loss": 4.12691978183571, + "tokens_seen": 331546624 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007268846539618856, + "loss": 1.5514, + "theoretical_loss": 4.126826282047449, + "tokens_seen": 331612160 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007268686058174524, + "loss": 1.6626, + "theoretical_loss": 4.1267328059083175, + "tokens_seen": 331677696 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726852557673019, + "loss": 1.6483, + "theoretical_loss": 4.126639353407663, + "tokens_seen": 331743232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007268365095285858, + "loss": 1.6556, + "theoretical_loss": 4.12654592453484, + "tokens_seen": 331808768 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007268204613841524, + "loss": 1.6978, + "theoretical_loss": 4.126452519279208, + "tokens_seen": 331874304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007268044132397192, + "loss": 1.6486, + "theoretical_loss": 4.126359137630136, + "tokens_seen": 331939840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007267883650952858, + "loss": 1.7489, + "theoretical_loss": 4.126265779576999, + "tokens_seen": 332005376 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007267723169508526, + "loss": 1.6806, + "theoretical_loss": 4.126172445109178, + "tokens_seen": 332070912 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007267562688064192, + "loss": 1.7395, + "theoretical_loss": 4.126079134216063, + "tokens_seen": 332136448 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726740220661986, + "loss": 1.5865, + "theoretical_loss": 4.125985846887049, + "tokens_seen": 332201984 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007267241725175528, + "loss": 1.5451, + "theoretical_loss": 4.125892583111538, + "tokens_seen": 332267520 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007267081243731194, + "loss": 1.537, + "theoretical_loss": 4.125799342878939, + "tokens_seen": 332333056 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007266920762286862, + "loss": 1.609, + "theoretical_loss": 4.1257061261786685, + "tokens_seen": 332398592 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007266760280842528, + "loss": 1.5716, + "theoretical_loss": 4.12561293300015, + "tokens_seen": 332464128 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007266599799398196, + "loss": 1.5856, + "theoretical_loss": 4.125519763332813, + "tokens_seen": 332529664 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007266439317953862, + "loss": 1.7653, + "theoretical_loss": 4.1254266171660925, + "tokens_seen": 332595200 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726627883650953, + "loss": 1.7007, + "theoretical_loss": 4.125333494489434, + "tokens_seen": 332660736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007266118355065196, + "loss": 1.7324, + "theoretical_loss": 4.125240395292286, + "tokens_seen": 332726272 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007265957873620864, + "loss": 1.7195, + "theoretical_loss": 4.125147319564107, + "tokens_seen": 332791808 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726579739217653, + "loss": 1.7432, + "theoretical_loss": 4.12505426729436, + "tokens_seen": 332857344 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007265636910732198, + "loss": 1.5473, + "theoretical_loss": 4.124961238472517, + "tokens_seen": 332922880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007265476429287864, + "loss": 1.6277, + "theoretical_loss": 4.124868233088054, + "tokens_seen": 332988416 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007265315947843532, + "loss": 1.6628, + "theoretical_loss": 4.124775251130455, + "tokens_seen": 333053952 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007265155466399198, + "loss": 1.6502, + "theoretical_loss": 4.124682292589212, + "tokens_seen": 333119488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007264994984954866, + "loss": 1.5286, + "theoretical_loss": 4.124589357453823, + "tokens_seen": 333185024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007264834503510532, + "loss": 1.6911, + "theoretical_loss": 4.124496445713792, + "tokens_seen": 333250560 + }, + { + "epoch": 0.1, + "learning_rate": 0.00072646740220662, + "loss": 1.601, + "theoretical_loss": 4.12440355735863, + "tokens_seen": 333316096 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007264513540621866, + "loss": 1.7226, + "theoretical_loss": 4.124310692377857, + "tokens_seen": 333381632 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007264353059177534, + "loss": 1.7132, + "theoretical_loss": 4.124217850760996, + "tokens_seen": 333447168 + }, + { + "epoch": 0.1, + "learning_rate": 0.00072641925777332, + "loss": 1.6178, + "theoretical_loss": 4.124125032497578, + "tokens_seen": 333512704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007264032096288868, + "loss": 1.573, + "theoretical_loss": 4.124032237577143, + "tokens_seen": 333578240 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007263871614844534, + "loss": 1.5512, + "theoretical_loss": 4.1239394659892366, + "tokens_seen": 333643776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007263711133400202, + "loss": 1.6984, + "theoretical_loss": 4.12384671772341, + "tokens_seen": 333709312 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007263550651955868, + "loss": 1.6455, + "theoretical_loss": 4.12375399276922, + "tokens_seen": 333774848 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007263390170511536, + "loss": 1.7328, + "theoretical_loss": 4.1236612911162345, + "tokens_seen": 333840384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007263229689067202, + "loss": 1.5987, + "theoretical_loss": 4.123568612754024, + "tokens_seen": 333905920 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726306920762287, + "loss": 1.5611, + "theoretical_loss": 4.123475957672168, + "tokens_seen": 333971456 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007262908726178536, + "loss": 1.7623, + "theoretical_loss": 4.123383325860252, + "tokens_seen": 334036992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007262748244734204, + "loss": 1.7431, + "theoretical_loss": 4.123290717307867, + "tokens_seen": 334102528 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726258776328987, + "loss": 1.6746, + "theoretical_loss": 4.123198132004614, + "tokens_seen": 334168064 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.9906439185142517, + "objective/train/avg_token_score": 0.009356092661619186, + "objective/train/docs_used": 128373, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.8171687126159668, + "objective/train/original_loss": 1.8356540203094482, + "objective/train/theoretical_loss": 4.123105569940097, + "objective/train/tokens_used": 354693600, + "objective/train/unlikelihood_loss": 0.018787503242492676, + "theoretical_loss": 4.123105569940097, + "tokens_seen": 334233600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007262427281845538, + "loss": 1.719, + "theoretical_loss": 4.123105569940097, + "tokens_seen": 334233600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007262266800401204, + "loss": 1.5443, + "theoretical_loss": 4.123013031103927, + "tokens_seen": 334299136 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007262106318956872, + "loss": 1.7106, + "theoretical_loss": 4.122920515485726, + "tokens_seen": 334364672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007261945837512538, + "loss": 1.5865, + "theoretical_loss": 4.122828023075117, + "tokens_seen": 334430208 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007261785356068206, + "loss": 1.5791, + "theoretical_loss": 4.122735553861733, + "tokens_seen": 334495744 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007261624874623872, + "loss": 1.7326, + "theoretical_loss": 4.122643107835213, + "tokens_seen": 334561280 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726146439317954, + "loss": 1.5782, + "theoretical_loss": 4.122550684985203, + "tokens_seen": 334626816 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007261303911735206, + "loss": 1.7173, + "theoretical_loss": 4.122458285301355, + "tokens_seen": 334692352 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007261143430290873, + "loss": 1.7604, + "theoretical_loss": 4.122365908773327, + "tokens_seen": 334757888 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726098294884654, + "loss": 1.6638, + "theoretical_loss": 4.1222735553907865, + "tokens_seen": 334823424 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007260822467402207, + "loss": 1.61, + "theoretical_loss": 4.122181225143404, + "tokens_seen": 334888960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007260661985957874, + "loss": 1.7865, + "theoretical_loss": 4.122088918020857, + "tokens_seen": 334954496 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007260501504513541, + "loss": 1.7152, + "theoretical_loss": 4.121996634012834, + "tokens_seen": 335020032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007260341023069208, + "loss": 1.5929, + "theoretical_loss": 4.121904373109024, + "tokens_seen": 335085568 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007260180541624875, + "loss": 1.7853, + "theoretical_loss": 4.121812135299129, + "tokens_seen": 335151104 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007260020060180541, + "loss": 1.6139, + "theoretical_loss": 4.1217199205728505, + "tokens_seen": 335216640 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007259859578736209, + "loss": 1.6839, + "theoretical_loss": 4.121627728919903, + "tokens_seen": 335282176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007259699097291875, + "loss": 1.7393, + "theoretical_loss": 4.121535560330004, + "tokens_seen": 335347712 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007259538615847543, + "loss": 1.5827, + "theoretical_loss": 4.1214434147928785, + "tokens_seen": 335413248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007259378134403209, + "loss": 1.6592, + "theoretical_loss": 4.121351292298257, + "tokens_seen": 335478784 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007259217652958877, + "loss": 1.6997, + "theoretical_loss": 4.12125919283588, + "tokens_seen": 335544320 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007259057171514543, + "loss": 1.6677, + "theoretical_loss": 4.12116711639549, + "tokens_seen": 335609856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007258896690070211, + "loss": 1.7099, + "theoretical_loss": 4.12107506296684, + "tokens_seen": 335675392 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007258736208625877, + "loss": 1.5692, + "theoretical_loss": 4.120983032539687, + "tokens_seen": 335740928 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007258575727181545, + "loss": 1.6912, + "theoretical_loss": 4.120891025103796, + "tokens_seen": 335806464 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007258415245737211, + "loss": 1.6437, + "theoretical_loss": 4.1207990406489365, + "tokens_seen": 335872000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007258254764292879, + "loss": 1.7268, + "theoretical_loss": 4.120707079164887, + "tokens_seen": 335937536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007258094282848545, + "loss": 1.6488, + "theoretical_loss": 4.120615140641433, + "tokens_seen": 336003072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007257933801404213, + "loss": 1.6255, + "theoretical_loss": 4.1205232250683625, + "tokens_seen": 336068608 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007257773319959879, + "loss": 1.6519, + "theoretical_loss": 4.120431332435475, + "tokens_seen": 336134144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007257612838515547, + "loss": 1.6052, + "theoretical_loss": 4.120339462732572, + "tokens_seen": 336199680 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007257452357071213, + "loss": 1.691, + "theoretical_loss": 4.120247615949465, + "tokens_seen": 336265216 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007257291875626881, + "loss": 1.6897, + "theoretical_loss": 4.12015579207597, + "tokens_seen": 336330752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007257131394182547, + "loss": 1.6889, + "theoretical_loss": 4.12006399110191, + "tokens_seen": 336396288 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007256970912738215, + "loss": 1.7457, + "theoretical_loss": 4.119972213017116, + "tokens_seen": 336461824 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007256810431293881, + "loss": 1.6087, + "theoretical_loss": 4.119880457811424, + "tokens_seen": 336527360 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007256649949849549, + "loss": 1.603, + "theoretical_loss": 4.119788725474675, + "tokens_seen": 336592896 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007256489468405215, + "loss": 1.7368, + "theoretical_loss": 4.119697015996719, + "tokens_seen": 336658432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007256328986960883, + "loss": 1.6591, + "theoretical_loss": 4.119605329367413, + "tokens_seen": 336723968 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007256168505516549, + "loss": 1.6482, + "theoretical_loss": 4.119513665576617, + "tokens_seen": 336789504 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007256008024072217, + "loss": 1.6951, + "theoretical_loss": 4.119422024614201, + "tokens_seen": 336855040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007255847542627883, + "loss": 1.6297, + "theoretical_loss": 4.11933040647004, + "tokens_seen": 336920576 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007255687061183551, + "loss": 1.6488, + "theoretical_loss": 4.119238811134014, + "tokens_seen": 336986112 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007255526579739217, + "loss": 1.6105, + "theoretical_loss": 4.119147238596014, + "tokens_seen": 337051648 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007255366098294885, + "loss": 1.5948, + "theoretical_loss": 4.119055688845931, + "tokens_seen": 337117184 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007255205616850552, + "loss": 1.7467, + "theoretical_loss": 4.118964161873667, + "tokens_seen": 337182720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007255045135406219, + "loss": 1.6469, + "theoretical_loss": 4.118872657669131, + "tokens_seen": 337248256 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007254884653961886, + "loss": 1.7226, + "theoretical_loss": 4.118781176222235, + "tokens_seen": 337313792 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007254724172517553, + "loss": 1.5574, + "theoretical_loss": 4.1186897175229, + "tokens_seen": 337379328 + }, + { + "epoch": 0.1, + "learning_rate": 0.000725456369107322, + "loss": 1.7397, + "theoretical_loss": 4.1185982815610505, + "tokens_seen": 337444864 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.9917698502540588, + "objective/train/avg_token_score": 0.008230128325521946, + "objective/train/docs_used": 129675, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5771913528442383, + "objective/train/original_loss": 1.5902857780456543, + "objective/train/theoretical_loss": 4.118506868326623, + "objective/train/tokens_used": 357970400, + "objective/train/unlikelihood_loss": 0.019237209111452103, + "theoretical_loss": 4.118506868326623, + "tokens_seen": 337510400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007254403209628887, + "loss": 1.5526, + "theoretical_loss": 4.118506868326623, + "tokens_seen": 337510400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007254242728184554, + "loss": 1.678, + "theoretical_loss": 4.118415477809553, + "tokens_seen": 337575936 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007254082246740221, + "loss": 1.5762, + "theoretical_loss": 4.11832410999979, + "tokens_seen": 337641472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007253921765295888, + "loss": 1.7142, + "theoretical_loss": 4.118232764887285, + "tokens_seen": 337707008 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007253761283851555, + "loss": 1.5556, + "theoretical_loss": 4.118141442461995, + "tokens_seen": 337772544 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007253600802407223, + "loss": 1.5878, + "theoretical_loss": 4.118050142713886, + "tokens_seen": 337838080 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007253440320962889, + "loss": 1.6417, + "theoretical_loss": 4.1179588656329305, + "tokens_seen": 337903616 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007253279839518557, + "loss": 1.7179, + "theoretical_loss": 4.117867611209104, + "tokens_seen": 337969152 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007253119358074223, + "loss": 1.6912, + "theoretical_loss": 4.117776379432392, + "tokens_seen": 338034688 + }, + { + "epoch": 0.1, + "learning_rate": 0.000725295887662989, + "loss": 1.5582, + "theoretical_loss": 4.117685170292786, + "tokens_seen": 338100224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007252798395185557, + "loss": 1.7102, + "theoretical_loss": 4.1175939837802815, + "tokens_seen": 338165760 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007252637913741225, + "loss": 1.5707, + "theoretical_loss": 4.117502819884882, + "tokens_seen": 338231296 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007252477432296891, + "loss": 1.6791, + "theoretical_loss": 4.117411678596596, + "tokens_seen": 338296832 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007252316950852559, + "loss": 1.7933, + "theoretical_loss": 4.117320559905441, + "tokens_seen": 338362368 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007252156469408225, + "loss": 1.7488, + "theoretical_loss": 4.117229463801439, + "tokens_seen": 338427904 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007251995987963893, + "loss": 1.7807, + "theoretical_loss": 4.117138390274619, + "tokens_seen": 338493440 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007251835506519559, + "loss": 1.6548, + "theoretical_loss": 4.117047339315014, + "tokens_seen": 338558976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007251675025075227, + "loss": 1.7897, + "theoretical_loss": 4.116956310912666, + "tokens_seen": 338624512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007251514543630893, + "loss": 1.8266, + "theoretical_loss": 4.116865305057625, + "tokens_seen": 338690048 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007251354062186561, + "loss": 1.7593, + "theoretical_loss": 4.116774321739942, + "tokens_seen": 338755584 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007251193580742227, + "loss": 1.611, + "theoretical_loss": 4.116683360949678, + "tokens_seen": 338821120 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007251033099297895, + "loss": 1.6042, + "theoretical_loss": 4.1165924226769, + "tokens_seen": 338886656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007250872617853561, + "loss": 1.7258, + "theoretical_loss": 4.11650150691168, + "tokens_seen": 338952192 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007250712136409229, + "loss": 1.6804, + "theoretical_loss": 4.116410613644098, + "tokens_seen": 339017728 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007250551654964895, + "loss": 1.6194, + "theoretical_loss": 4.116319742864239, + "tokens_seen": 339083264 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007250391173520563, + "loss": 1.6718, + "theoretical_loss": 4.1162288945621945, + "tokens_seen": 339148800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007250230692076229, + "loss": 1.7282, + "theoretical_loss": 4.116138068728063, + "tokens_seen": 339214336 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007250070210631897, + "loss": 1.5866, + "theoretical_loss": 4.1160472653519475, + "tokens_seen": 339279872 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007249909729187563, + "loss": 1.4775, + "theoretical_loss": 4.11595648442396, + "tokens_seen": 339345408 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007249749247743231, + "loss": 1.5448, + "theoretical_loss": 4.115865725934217, + "tokens_seen": 339410944 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007249588766298897, + "loss": 1.6772, + "theoretical_loss": 4.11577498987284, + "tokens_seen": 339476480 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007249428284854565, + "loss": 1.6908, + "theoretical_loss": 4.11568427622996, + "tokens_seen": 339542016 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007249267803410231, + "loss": 1.5952, + "theoretical_loss": 4.115593584995712, + "tokens_seen": 339607552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007249107321965899, + "loss": 1.5901, + "theoretical_loss": 4.115502916160237, + "tokens_seen": 339673088 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007248946840521565, + "loss": 1.6288, + "theoretical_loss": 4.115412269713684, + "tokens_seen": 339738624 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007248786359077233, + "loss": 1.7957, + "theoretical_loss": 4.115321645646207, + "tokens_seen": 339804160 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007248625877632899, + "loss": 1.5972, + "theoretical_loss": 4.115231043947968, + "tokens_seen": 339869696 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007248465396188567, + "loss": 1.5844, + "theoretical_loss": 4.11514046460913, + "tokens_seen": 339935232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007248304914744233, + "loss": 1.6382, + "theoretical_loss": 4.115049907619868, + "tokens_seen": 340000768 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007248144433299901, + "loss": 1.7103, + "theoretical_loss": 4.114959372970362, + "tokens_seen": 340066304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007247983951855567, + "loss": 1.6055, + "theoretical_loss": 4.114868860650796, + "tokens_seen": 340131840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007247823470411235, + "loss": 1.7636, + "theoretical_loss": 4.114778370651361, + "tokens_seen": 340197376 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007247662988966901, + "loss": 1.6462, + "theoretical_loss": 4.114687902962258, + "tokens_seen": 340262912 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007247502507522569, + "loss": 1.6576, + "theoretical_loss": 4.114597457573687, + "tokens_seen": 340328448 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007247342026078235, + "loss": 1.6916, + "theoretical_loss": 4.11450703447586, + "tokens_seen": 340393984 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007247181544633902, + "loss": 1.6389, + "theoretical_loss": 4.114416633658992, + "tokens_seen": 340459520 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007247021063189569, + "loss": 1.6954, + "theoretical_loss": 4.114326255113308, + "tokens_seen": 340525056 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007246860581745236, + "loss": 1.5177, + "theoretical_loss": 4.114235898829034, + "tokens_seen": 340590592 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007246700100300903, + "loss": 1.7174, + "theoretical_loss": 4.114145564796406, + "tokens_seen": 340656128 + }, + { + "epoch": 0.1, + "learning_rate": 0.000724653961885657, + "loss": 1.7077, + "theoretical_loss": 4.114055253005665, + "tokens_seen": 340721664 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.9936263561248779, + "objective/train/avg_token_score": 0.006373669020831585, + "objective/train/docs_used": 130755, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.7068885564804077, + "objective/train/original_loss": 1.7184754610061646, + "objective/train/theoretical_loss": 4.113964963447058, + "objective/train/tokens_used": 361247200, + "objective/train/unlikelihood_loss": 0.012738223187625408, + "theoretical_loss": 4.113964963447058, + "tokens_seen": 340787200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007246379137412237, + "loss": 1.611, + "theoretical_loss": 4.113964963447058, + "tokens_seen": 340787200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007246218655967904, + "loss": 1.6056, + "theoretical_loss": 4.113874696110838, + "tokens_seen": 340852736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007246058174523571, + "loss": 1.7366, + "theoretical_loss": 4.113784450987264, + "tokens_seen": 340918272 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007245897693079238, + "loss": 1.5039, + "theoretical_loss": 4.113694228066604, + "tokens_seen": 340983808 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007245737211634904, + "loss": 1.7275, + "theoretical_loss": 4.113604027339126, + "tokens_seen": 341049344 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007245576730190572, + "loss": 1.6055, + "theoretical_loss": 4.11351384879511, + "tokens_seen": 341114880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007245416248746238, + "loss": 1.5577, + "theoretical_loss": 4.11342369242484, + "tokens_seen": 341180416 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007245255767301906, + "loss": 1.7679, + "theoretical_loss": 4.1133335582186055, + "tokens_seen": 341245952 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007245095285857572, + "loss": 1.5886, + "theoretical_loss": 4.1132434461667025, + "tokens_seen": 341311488 + }, + { + "epoch": 0.1, + "learning_rate": 0.000724493480441324, + "loss": 1.7833, + "theoretical_loss": 4.113153356259433, + "tokens_seen": 341377024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007244774322968906, + "loss": 1.7076, + "theoretical_loss": 4.1130632884871074, + "tokens_seen": 341442560 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007244613841524574, + "loss": 1.5666, + "theoretical_loss": 4.112973242840038, + "tokens_seen": 341508096 + }, + { + "epoch": 0.1, + "learning_rate": 0.000724445336008024, + "loss": 1.6531, + "theoretical_loss": 4.1128832193085465, + "tokens_seen": 341573632 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007244292878635908, + "loss": 1.6652, + "theoretical_loss": 4.11279321788296, + "tokens_seen": 341639168 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007244132397191574, + "loss": 1.6105, + "theoretical_loss": 4.112703238553609, + "tokens_seen": 341704704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007243971915747242, + "loss": 1.6601, + "theoretical_loss": 4.112613281310835, + "tokens_seen": 341770240 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007243811434302908, + "loss": 1.5395, + "theoretical_loss": 4.1125233461449815, + "tokens_seen": 341835776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007243650952858576, + "loss": 1.6787, + "theoretical_loss": 4.1124334330464, + "tokens_seen": 341901312 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007243490471414242, + "loss": 1.6897, + "theoretical_loss": 4.112343542005447, + "tokens_seen": 341966848 + }, + { + "epoch": 0.1, + "learning_rate": 0.000724332998996991, + "loss": 1.468, + "theoretical_loss": 4.112253673012486, + "tokens_seen": 342032384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007243169508525577, + "loss": 1.6368, + "theoretical_loss": 4.112163826057886, + "tokens_seen": 342097920 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007243009027081244, + "loss": 1.6023, + "theoretical_loss": 4.112074001132023, + "tokens_seen": 342163456 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007242848545636911, + "loss": 1.6837, + "theoretical_loss": 4.111984198225278, + "tokens_seen": 342228992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007242688064192578, + "loss": 1.6867, + "theoretical_loss": 4.1118944173280365, + "tokens_seen": 342294528 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007242527582748245, + "loss": 1.6401, + "theoretical_loss": 4.111804658430694, + "tokens_seen": 342360064 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007242367101303912, + "loss": 1.5119, + "theoretical_loss": 4.11171492152365, + "tokens_seen": 342425600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007242206619859579, + "loss": 1.6363, + "theoretical_loss": 4.111625206597309, + "tokens_seen": 342491136 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007242046138415246, + "loss": 1.7237, + "theoretical_loss": 4.111535513642082, + "tokens_seen": 342556672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007241885656970913, + "loss": 1.574, + "theoretical_loss": 4.111445842648387, + "tokens_seen": 342622208 + }, + { + "epoch": 0.1, + "learning_rate": 0.000724172517552658, + "loss": 1.6314, + "theoretical_loss": 4.111356193606648, + "tokens_seen": 342687744 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007241564694082247, + "loss": 1.6496, + "theoretical_loss": 4.111266566507294, + "tokens_seen": 342753280 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007241404212637914, + "loss": 1.5834, + "theoretical_loss": 4.11117696134076, + "tokens_seen": 342818816 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007241243731193581, + "loss": 1.5453, + "theoretical_loss": 4.111087378097488, + "tokens_seen": 342884352 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007241083249749248, + "loss": 1.7256, + "theoretical_loss": 4.1109978167679255, + "tokens_seen": 342949888 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007240922768304915, + "loss": 1.6142, + "theoretical_loss": 4.110908277342526, + "tokens_seen": 343015424 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007240762286860582, + "loss": 1.6809, + "theoretical_loss": 4.110818759811748, + "tokens_seen": 343080960 + }, + { + "epoch": 0.1, + "learning_rate": 0.000724060180541625, + "loss": 1.5784, + "theoretical_loss": 4.110729264166058, + "tokens_seen": 343146496 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007240441323971916, + "loss": 1.7665, + "theoretical_loss": 4.110639790395926, + "tokens_seen": 343212032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007240280842527583, + "loss": 1.5787, + "theoretical_loss": 4.110550338491832, + "tokens_seen": 343277568 + }, + { + "epoch": 0.1, + "learning_rate": 0.000724012036108325, + "loss": 1.7715, + "theoretical_loss": 4.1104609084442565, + "tokens_seen": 343343104 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007239959879638917, + "loss": 1.6228, + "theoretical_loss": 4.11037150024369, + "tokens_seen": 343408640 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007239799398194584, + "loss": 1.6582, + "theoretical_loss": 4.110282113880627, + "tokens_seen": 343474176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007239638916750252, + "loss": 1.6149, + "theoretical_loss": 4.11019274934557, + "tokens_seen": 343539712 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007239478435305918, + "loss": 1.6573, + "theoretical_loss": 4.110103406629026, + "tokens_seen": 343605248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007239317953861586, + "loss": 1.5709, + "theoretical_loss": 4.110014085721506, + "tokens_seen": 343670784 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007239157472417252, + "loss": 1.6929, + "theoretical_loss": 4.109924786613531, + "tokens_seen": 343736320 + }, + { + "epoch": 0.1, + "learning_rate": 0.000723899699097292, + "loss": 1.6224, + "theoretical_loss": 4.109835509295625, + "tokens_seen": 343801856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007238836509528586, + "loss": 1.5963, + "theoretical_loss": 4.10974625375832, + "tokens_seen": 343867392 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007238676028084254, + "loss": 1.6613, + "theoretical_loss": 4.1096570199921505, + "tokens_seen": 343932928 + }, + { + "epoch": 0.1, + "learning_rate": 0.000723851554663992, + "loss": 1.6313, + "theoretical_loss": 4.109567807987661, + "tokens_seen": 343998464 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.9933397769927979, + "objective/train/avg_token_score": 0.006660213228315115, + "objective/train/docs_used": 131936, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6914114952087402, + "objective/train/original_loss": 1.7036280632019043, + "objective/train/theoretical_loss": 4.109478617735399, + "objective/train/tokens_used": 364524000, + "objective/train/unlikelihood_loss": 0.01321543101221323, + "theoretical_loss": 4.109478617735399, + "tokens_seen": 344064000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007238355065195588, + "loss": 1.8014, + "theoretical_loss": 4.109478617735399, + "tokens_seen": 344064000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007238194583751254, + "loss": 1.5648, + "theoretical_loss": 4.10938944922592, + "tokens_seen": 344129536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007238034102306922, + "loss": 1.6436, + "theoretical_loss": 4.109300302449784, + "tokens_seen": 344195072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007237873620862588, + "loss": 1.5355, + "theoretical_loss": 4.109211177397556, + "tokens_seen": 344260608 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007237713139418256, + "loss": 1.5564, + "theoretical_loss": 4.109122074059811, + "tokens_seen": 344326144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007237552657973922, + "loss": 1.5611, + "theoretical_loss": 4.1090329924271245, + "tokens_seen": 344391680 + }, + { + "epoch": 0.1, + "learning_rate": 0.000723739217652959, + "loss": 1.5757, + "theoretical_loss": 4.108943932490082, + "tokens_seen": 344457216 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007237231695085256, + "loss": 1.7027, + "theoretical_loss": 4.108854894239271, + "tokens_seen": 344522752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007237071213640924, + "loss": 1.6139, + "theoretical_loss": 4.10876587766529, + "tokens_seen": 344588288 + }, + { + "epoch": 0.1, + "learning_rate": 0.000723691073219659, + "loss": 1.6032, + "theoretical_loss": 4.108676882758738, + "tokens_seen": 344653824 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007236750250752258, + "loss": 1.6457, + "theoretical_loss": 4.1085879095102245, + "tokens_seen": 344719360 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007236589769307924, + "loss": 1.7607, + "theoretical_loss": 4.108498957910362, + "tokens_seen": 344784896 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007236429287863592, + "loss": 1.6694, + "theoretical_loss": 4.108410027949769, + "tokens_seen": 344850432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007236268806419258, + "loss": 1.6538, + "theoretical_loss": 4.10832111961907, + "tokens_seen": 344915968 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007236108324974926, + "loss": 1.6783, + "theoretical_loss": 4.1082322329088985, + "tokens_seen": 344981504 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007235947843530592, + "loss": 1.7998, + "theoretical_loss": 4.108143367809888, + "tokens_seen": 345047040 + }, + { + "epoch": 0.1, + "learning_rate": 0.000723578736208626, + "loss": 1.5985, + "theoretical_loss": 4.108054524312683, + "tokens_seen": 345112576 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007235626880641926, + "loss": 1.5937, + "theoretical_loss": 4.107965702407931, + "tokens_seen": 345178112 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007235466399197594, + "loss": 1.6015, + "theoretical_loss": 4.107876902086286, + "tokens_seen": 345243648 + }, + { + "epoch": 0.1, + "learning_rate": 0.000723530591775326, + "loss": 1.6466, + "theoretical_loss": 4.107788123338408, + "tokens_seen": 345309184 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007235145436308928, + "loss": 1.6771, + "theoretical_loss": 4.1076993661549634, + "tokens_seen": 345374720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007234984954864594, + "loss": 1.7047, + "theoretical_loss": 4.107610630526622, + "tokens_seen": 345440256 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007234824473420262, + "loss": 1.8753, + "theoretical_loss": 4.107521916444064, + "tokens_seen": 345505792 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007234663991975928, + "loss": 1.5723, + "theoretical_loss": 4.10743322389797, + "tokens_seen": 345571328 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007234503510531596, + "loss": 1.5592, + "theoretical_loss": 4.107344552879029, + "tokens_seen": 345636864 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007234343029087262, + "loss": 1.6771, + "theoretical_loss": 4.107255903377938, + "tokens_seen": 345702400 + }, + { + "epoch": 0.1, + "learning_rate": 0.000723418254764293, + "loss": 1.7078, + "theoretical_loss": 4.107167275385395, + "tokens_seen": 345767936 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007234022066198596, + "loss": 1.764, + "theoretical_loss": 4.107078668892108, + "tokens_seen": 345833472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007233861584754264, + "loss": 1.6223, + "theoretical_loss": 4.106990083888789, + "tokens_seen": 345899008 + }, + { + "epoch": 0.1, + "learning_rate": 0.000723370110330993, + "loss": 1.6973, + "theoretical_loss": 4.106901520366154, + "tokens_seen": 345964544 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007233540621865598, + "loss": 1.6277, + "theoretical_loss": 4.106812978314929, + "tokens_seen": 346030080 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007233380140421264, + "loss": 1.6953, + "theoretical_loss": 4.106724457725842, + "tokens_seen": 346095616 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007233219658976931, + "loss": 1.7184, + "theoretical_loss": 4.106635958589628, + "tokens_seen": 346161152 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007233059177532598, + "loss": 1.6136, + "theoretical_loss": 4.106547480897029, + "tokens_seen": 346226688 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007232898696088265, + "loss": 1.5376, + "theoretical_loss": 4.106459024638791, + "tokens_seen": 346292224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007232738214643932, + "loss": 1.5745, + "theoretical_loss": 4.106370589805666, + "tokens_seen": 346357760 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007232577733199599, + "loss": 1.7335, + "theoretical_loss": 4.1062821763884125, + "tokens_seen": 346423296 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007232417251755266, + "loss": 1.7744, + "theoretical_loss": 4.106193784377794, + "tokens_seen": 346488832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007232256770310933, + "loss": 1.6605, + "theoretical_loss": 4.106105413764581, + "tokens_seen": 346554368 + }, + { + "epoch": 0.11, + "learning_rate": 0.00072320962888666, + "loss": 1.5521, + "theoretical_loss": 4.106017064539548, + "tokens_seen": 346619904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007231935807422267, + "loss": 1.7688, + "theoretical_loss": 4.1059287366934765, + "tokens_seen": 346685440 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007231775325977933, + "loss": 1.5951, + "theoretical_loss": 4.105840430217153, + "tokens_seen": 346750976 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007231614844533601, + "loss": 1.6736, + "theoretical_loss": 4.10575214510137, + "tokens_seen": 346816512 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007231454363089267, + "loss": 1.5842, + "theoretical_loss": 4.105663881336925, + "tokens_seen": 346882048 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007231293881644935, + "loss": 1.6191, + "theoretical_loss": 4.105575638914623, + "tokens_seen": 346947584 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007231133400200602, + "loss": 1.701, + "theoretical_loss": 4.105487417825273, + "tokens_seen": 347013120 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007230972918756269, + "loss": 1.6604, + "theoretical_loss": 4.10539921805969, + "tokens_seen": 347078656 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007230812437311936, + "loss": 1.5289, + "theoretical_loss": 4.105311039608695, + "tokens_seen": 347144192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007230651955867603, + "loss": 1.5364, + "theoretical_loss": 4.105222882463115, + "tokens_seen": 347209728 + }, + { + "epoch": 0.11, + "learning_rate": 0.000723049147442327, + "loss": 1.6423, + "theoretical_loss": 4.105134746613783, + "tokens_seen": 347275264 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.9942477941513062, + "objective/train/avg_token_score": 0.00575221935287118, + "objective/train/docs_used": 133174, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6207184791564941, + "objective/train/original_loss": 1.6313303709030151, + "objective/train/theoretical_loss": 4.105046632051535, + "objective/train/tokens_used": 367800800, + "objective/train/unlikelihood_loss": 0.011034071445465088, + "theoretical_loss": 4.105046632051535, + "tokens_seen": 347340800 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007230330992978937, + "loss": 1.5735, + "theoretical_loss": 4.105046632051535, + "tokens_seen": 347340800 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007230170511534604, + "loss": 1.6152, + "theoretical_loss": 4.1049585387672165, + "tokens_seen": 347406336 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007230010030090271, + "loss": 1.6539, + "theoretical_loss": 4.1048704667516756, + "tokens_seen": 347471872 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007229849548645938, + "loss": 1.6325, + "theoretical_loss": 4.104782415995768, + "tokens_seen": 347537408 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007229689067201605, + "loss": 1.6816, + "theoretical_loss": 4.104694386490354, + "tokens_seen": 347602944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007229528585757272, + "loss": 1.5863, + "theoretical_loss": 4.1046063782263, + "tokens_seen": 347668480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007229368104312939, + "loss": 1.7033, + "theoretical_loss": 4.104518391194477, + "tokens_seen": 347734016 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007229207622868606, + "loss": 1.5691, + "theoretical_loss": 4.104430425385765, + "tokens_seen": 347799552 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007229047141424273, + "loss": 1.6074, + "theoretical_loss": 4.104342480791044, + "tokens_seen": 347865088 + }, + { + "epoch": 0.11, + "learning_rate": 0.000722888665997994, + "loss": 1.5627, + "theoretical_loss": 4.104254557401205, + "tokens_seen": 347930624 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007228726178535607, + "loss": 1.6057, + "theoretical_loss": 4.104166655207142, + "tokens_seen": 347996160 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007228565697091274, + "loss": 1.6548, + "theoretical_loss": 4.104078774199754, + "tokens_seen": 348061696 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007228405215646941, + "loss": 1.6916, + "theoretical_loss": 4.10399091436995, + "tokens_seen": 348127232 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007228244734202608, + "loss": 1.6396, + "theoretical_loss": 4.103903075708637, + "tokens_seen": 348192768 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007228084252758275, + "loss": 1.5463, + "theoretical_loss": 4.103815258206735, + "tokens_seen": 348258304 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007227923771313942, + "loss": 1.6182, + "theoretical_loss": 4.103727461855165, + "tokens_seen": 348323840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007227763289869609, + "loss": 1.507, + "theoretical_loss": 4.103639686644856, + "tokens_seen": 348389376 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007227602808425276, + "loss": 1.713, + "theoretical_loss": 4.103551932566741, + "tokens_seen": 348454912 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007227442326980943, + "loss": 1.714, + "theoretical_loss": 4.103464199611759, + "tokens_seen": 348520448 + }, + { + "epoch": 0.11, + "learning_rate": 0.000722728184553661, + "loss": 1.6215, + "theoretical_loss": 4.1033764877708565, + "tokens_seen": 348585984 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007227121364092277, + "loss": 1.8557, + "theoretical_loss": 4.103288797034983, + "tokens_seen": 348651520 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007226960882647944, + "loss": 1.7431, + "theoretical_loss": 4.103201127395095, + "tokens_seen": 348717056 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007226800401203611, + "loss": 1.6382, + "theoretical_loss": 4.103113478842153, + "tokens_seen": 348782592 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007226639919759278, + "loss": 1.5462, + "theoretical_loss": 4.103025851367125, + "tokens_seen": 348848128 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007226479438314945, + "loss": 1.6161, + "theoretical_loss": 4.102938244960985, + "tokens_seen": 348913664 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007226318956870612, + "loss": 1.5435, + "theoretical_loss": 4.102850659614709, + "tokens_seen": 348979200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007226158475426279, + "loss": 1.6242, + "theoretical_loss": 4.102763095319282, + "tokens_seen": 349044736 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007225997993981947, + "loss": 1.6807, + "theoretical_loss": 4.102675552065694, + "tokens_seen": 349110272 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007225837512537613, + "loss": 1.6421, + "theoretical_loss": 4.10258802984494, + "tokens_seen": 349175808 + }, + { + "epoch": 0.11, + "learning_rate": 0.000722567703109328, + "loss": 1.5646, + "theoretical_loss": 4.102500528648019, + "tokens_seen": 349241344 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007225516549648947, + "loss": 1.6842, + "theoretical_loss": 4.102413048465938, + "tokens_seen": 349306880 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007225356068204615, + "loss": 1.5723, + "theoretical_loss": 4.10232558928971, + "tokens_seen": 349372416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007225195586760281, + "loss": 1.5626, + "theoretical_loss": 4.10223815111035, + "tokens_seen": 349437952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007225035105315949, + "loss": 1.6289, + "theoretical_loss": 4.102150733918882, + "tokens_seen": 349503488 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007224874623871615, + "loss": 1.7853, + "theoretical_loss": 4.102063337706333, + "tokens_seen": 349569024 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007224714142427283, + "loss": 1.571, + "theoretical_loss": 4.101975962463738, + "tokens_seen": 349634560 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007224553660982949, + "loss": 1.6432, + "theoretical_loss": 4.101888608182136, + "tokens_seen": 349700096 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007224393179538617, + "loss": 1.6618, + "theoretical_loss": 4.101801274852571, + "tokens_seen": 349765632 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007224232698094283, + "loss": 1.6247, + "theoretical_loss": 4.1017139624660945, + "tokens_seen": 349831168 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007224072216649951, + "loss": 1.47, + "theoretical_loss": 4.1016266710137606, + "tokens_seen": 349896704 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007223911735205617, + "loss": 1.6068, + "theoretical_loss": 4.101539400486631, + "tokens_seen": 349962240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007223751253761285, + "loss": 1.533, + "theoretical_loss": 4.101452150875773, + "tokens_seen": 350027776 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007223590772316951, + "loss": 1.7029, + "theoretical_loss": 4.101364922172259, + "tokens_seen": 350093312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007223430290872619, + "loss": 1.6089, + "theoretical_loss": 4.101277714367166, + "tokens_seen": 350158848 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007223269809428285, + "loss": 1.5563, + "theoretical_loss": 4.1011905274515765, + "tokens_seen": 350224384 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007223109327983953, + "loss": 1.6932, + "theoretical_loss": 4.101103361416581, + "tokens_seen": 350289920 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007222948846539619, + "loss": 1.5234, + "theoretical_loss": 4.101016216253273, + "tokens_seen": 350355456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007222788365095287, + "loss": 1.6422, + "theoretical_loss": 4.10092909195275, + "tokens_seen": 350420992 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007222627883650953, + "loss": 1.6414, + "theoretical_loss": 4.10084198850612, + "tokens_seen": 350486528 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007222467402206621, + "loss": 1.6543, + "theoretical_loss": 4.100754905904491, + "tokens_seen": 350552064 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.9893074631690979, + "objective/train/avg_token_score": 0.010692501440644264, + "objective/train/docs_used": 134382, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6653629541397095, + "objective/train/original_loss": 1.6826835870742798, + "objective/train/theoretical_loss": 4.100667844138981, + "objective/train/tokens_used": 371077600, + "objective/train/unlikelihood_loss": 0.02337191253900528, + "theoretical_loss": 4.100667844138981, + "tokens_seen": 350617600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007222306920762287, + "loss": 1.6885, + "theoretical_loss": 4.100667844138981, + "tokens_seen": 350617600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007222146439317955, + "loss": 1.6078, + "theoretical_loss": 4.10058080320071, + "tokens_seen": 350683136 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007221985957873621, + "loss": 1.6922, + "theoretical_loss": 4.100493783080806, + "tokens_seen": 350748672 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007221825476429289, + "loss": 1.7885, + "theoretical_loss": 4.1004067837704, + "tokens_seen": 350814208 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007221664994984955, + "loss": 1.6508, + "theoretical_loss": 4.1003198052606304, + "tokens_seen": 350879744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007221504513540623, + "loss": 1.6417, + "theoretical_loss": 4.10023284754264, + "tokens_seen": 350945280 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007221344032096289, + "loss": 1.5685, + "theoretical_loss": 4.100145910607576, + "tokens_seen": 351010816 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007221183550651957, + "loss": 1.5899, + "theoretical_loss": 4.100058994446596, + "tokens_seen": 351076352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007221023069207623, + "loss": 1.6945, + "theoretical_loss": 4.099972099050856, + "tokens_seen": 351141888 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007220862587763291, + "loss": 1.7378, + "theoretical_loss": 4.099885224411522, + "tokens_seen": 351207424 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007220702106318957, + "loss": 1.5492, + "theoretical_loss": 4.099798370519764, + "tokens_seen": 351272960 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007220541624874625, + "loss": 1.5821, + "theoretical_loss": 4.099711537366758, + "tokens_seen": 351338496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007220381143430291, + "loss": 1.6617, + "theoretical_loss": 4.099624724943685, + "tokens_seen": 351404032 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007220220661985959, + "loss": 1.5972, + "theoretical_loss": 4.099537933241731, + "tokens_seen": 351469568 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007220060180541625, + "loss": 1.6502, + "theoretical_loss": 4.0994511622520875, + "tokens_seen": 351535104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007219899699097293, + "loss": 1.5628, + "theoretical_loss": 4.099364411965952, + "tokens_seen": 351600640 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007219739217652959, + "loss": 1.762, + "theoretical_loss": 4.0992776823745265, + "tokens_seen": 351666176 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007219578736208627, + "loss": 1.6017, + "theoretical_loss": 4.09919097346902, + "tokens_seen": 351731712 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007219418254764293, + "loss": 1.6293, + "theoretical_loss": 4.099104285240646, + "tokens_seen": 351797248 + }, + { + "epoch": 0.11, + "learning_rate": 0.000721925777331996, + "loss": 1.7028, + "theoretical_loss": 4.099017617680621, + "tokens_seen": 351862784 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007219097291875627, + "loss": 1.6688, + "theoretical_loss": 4.098930970780172, + "tokens_seen": 351928320 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007218936810431294, + "loss": 1.632, + "theoretical_loss": 4.098844344530526, + "tokens_seen": 351993856 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007218776328986961, + "loss": 1.6881, + "theoretical_loss": 4.098757738922918, + "tokens_seen": 352059392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007218615847542628, + "loss": 1.6528, + "theoretical_loss": 4.09867115394859, + "tokens_seen": 352124928 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007218455366098295, + "loss": 1.6146, + "theoretical_loss": 4.098584589598786, + "tokens_seen": 352190464 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007218294884653962, + "loss": 1.5544, + "theoretical_loss": 4.098498045864756, + "tokens_seen": 352256000 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007218134403209629, + "loss": 1.6463, + "theoretical_loss": 4.0984115227377576, + "tokens_seen": 352321536 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007217973921765296, + "loss": 1.5968, + "theoretical_loss": 4.098325020209051, + "tokens_seen": 352387072 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007217813440320963, + "loss": 1.7144, + "theoretical_loss": 4.098238538269904, + "tokens_seen": 352452608 + }, + { + "epoch": 0.11, + "learning_rate": 0.000721765295887663, + "loss": 1.6005, + "theoretical_loss": 4.098152076911588, + "tokens_seen": 352518144 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007217492477432297, + "loss": 1.6323, + "theoretical_loss": 4.098065636125381, + "tokens_seen": 352583680 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007217331995987964, + "loss": 1.6311, + "theoretical_loss": 4.097979215902566, + "tokens_seen": 352649216 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007217171514543631, + "loss": 1.6593, + "theoretical_loss": 4.097892816234429, + "tokens_seen": 352714752 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007217011033099298, + "loss": 1.5214, + "theoretical_loss": 4.097806437112267, + "tokens_seen": 352780288 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007216850551654965, + "loss": 1.53, + "theoretical_loss": 4.0977200785273755, + "tokens_seen": 352845824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007216690070210632, + "loss": 1.5197, + "theoretical_loss": 4.09763374047106, + "tokens_seen": 352911360 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007216529588766299, + "loss": 1.6708, + "theoretical_loss": 4.097547422934629, + "tokens_seen": 352976896 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007216369107321966, + "loss": 1.6026, + "theoretical_loss": 4.097461125909398, + "tokens_seen": 353042432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007216208625877633, + "loss": 1.6878, + "theoretical_loss": 4.097374849386685, + "tokens_seen": 353107968 + }, + { + "epoch": 0.11, + "learning_rate": 0.00072160481444333, + "loss": 1.6953, + "theoretical_loss": 4.097288593357817, + "tokens_seen": 353173504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007215887662988967, + "loss": 1.4305, + "theoretical_loss": 4.097202357814124, + "tokens_seen": 353239040 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007215727181544634, + "loss": 1.7325, + "theoretical_loss": 4.097116142746941, + "tokens_seen": 353304576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007215566700100301, + "loss": 1.6251, + "theoretical_loss": 4.09702994814761, + "tokens_seen": 353370112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007215406218655968, + "loss": 1.672, + "theoretical_loss": 4.096943774007476, + "tokens_seen": 353435648 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007215245737211635, + "loss": 1.7027, + "theoretical_loss": 4.096857620317892, + "tokens_seen": 353501184 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007215085255767302, + "loss": 1.6082, + "theoretical_loss": 4.096771487070214, + "tokens_seen": 353566720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007214924774322969, + "loss": 1.6045, + "theoretical_loss": 4.096685374255803, + "tokens_seen": 353632256 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007214764292878636, + "loss": 1.624, + "theoretical_loss": 4.096599281866028, + "tokens_seen": 353697792 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007214603811434303, + "loss": 1.7009, + "theoretical_loss": 4.09651320989226, + "tokens_seen": 353763328 + }, + { + "epoch": 0.11, + "learning_rate": 0.000721444332998997, + "loss": 1.5938, + "theoretical_loss": 4.096427158325877, + "tokens_seen": 353828864 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.9912610054016113, + "objective/train/avg_token_score": 0.008739026263356209, + "objective/train/docs_used": 135614, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.421583890914917, + "objective/train/original_loss": 1.4329161643981934, + "objective/train/theoretical_loss": 4.096341127158263, + "objective/train/tokens_used": 374354400, + "objective/train/unlikelihood_loss": 0.03006824292242527, + "theoretical_loss": 4.096341127158263, + "tokens_seen": 353894400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007214282848545637, + "loss": 1.5759, + "theoretical_loss": 4.096341127158263, + "tokens_seen": 353894400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007214122367101304, + "loss": 1.6625, + "theoretical_loss": 4.096255116380806, + "tokens_seen": 353959936 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007213961885656971, + "loss": 1.65, + "theoretical_loss": 4.096169125984899, + "tokens_seen": 354025472 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007213801404212638, + "loss": 1.6362, + "theoretical_loss": 4.0960831559619395, + "tokens_seen": 354091008 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007213640922768305, + "loss": 1.6523, + "theoretical_loss": 4.095997206303333, + "tokens_seen": 354156544 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007213480441323972, + "loss": 1.6247, + "theoretical_loss": 4.095911277000488, + "tokens_seen": 354222080 + }, + { + "epoch": 0.11, + "learning_rate": 0.000721331995987964, + "loss": 1.6167, + "theoretical_loss": 4.095825368044819, + "tokens_seen": 354287616 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007213159478435306, + "loss": 1.6812, + "theoretical_loss": 4.0957394794277455, + "tokens_seen": 354353152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007212998996990973, + "loss": 1.6009, + "theoretical_loss": 4.095653611140692, + "tokens_seen": 354418688 + }, + { + "epoch": 0.11, + "learning_rate": 0.000721283851554664, + "loss": 1.6775, + "theoretical_loss": 4.09556776317509, + "tokens_seen": 354484224 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007212678034102307, + "loss": 1.6786, + "theoretical_loss": 4.095481935522372, + "tokens_seen": 354549760 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007212517552657974, + "loss": 1.6192, + "theoretical_loss": 4.09539612817398, + "tokens_seen": 354615296 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007212357071213642, + "loss": 1.6509, + "theoretical_loss": 4.095310341121358, + "tokens_seen": 354680832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007212196589769308, + "loss": 1.6851, + "theoretical_loss": 4.095224574355958, + "tokens_seen": 354746368 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007212036108324976, + "loss": 1.7002, + "theoretical_loss": 4.095138827869237, + "tokens_seen": 354811904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007211875626880642, + "loss": 1.6408, + "theoretical_loss": 4.0950531016526535, + "tokens_seen": 354877440 + }, + { + "epoch": 0.11, + "learning_rate": 0.000721171514543631, + "loss": 1.6127, + "theoretical_loss": 4.094967395697675, + "tokens_seen": 354942976 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007211554663991976, + "loss": 1.6227, + "theoretical_loss": 4.094881709995773, + "tokens_seen": 355008512 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007211394182547644, + "loss": 1.7089, + "theoretical_loss": 4.094796044538423, + "tokens_seen": 355074048 + }, + { + "epoch": 0.11, + "learning_rate": 0.000721123370110331, + "loss": 1.5812, + "theoretical_loss": 4.094710399317109, + "tokens_seen": 355139584 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007211073219658978, + "loss": 1.6479, + "theoretical_loss": 4.094624774323315, + "tokens_seen": 355205120 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007210912738214644, + "loss": 1.6448, + "theoretical_loss": 4.094539169548535, + "tokens_seen": 355270656 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007210752256770312, + "loss": 1.5803, + "theoretical_loss": 4.094453584984265, + "tokens_seen": 355336192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007210591775325978, + "loss": 1.61, + "theoretical_loss": 4.094368020622008, + "tokens_seen": 355401728 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007210431293881646, + "loss": 1.7177, + "theoretical_loss": 4.094282476453272, + "tokens_seen": 355467264 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007210270812437312, + "loss": 1.6735, + "theoretical_loss": 4.094196952469568, + "tokens_seen": 355532800 + }, + { + "epoch": 0.11, + "learning_rate": 0.000721011033099298, + "loss": 1.6923, + "theoretical_loss": 4.0941114486624155, + "tokens_seen": 355598336 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007209949849548646, + "loss": 1.6407, + "theoretical_loss": 4.094025965023335, + "tokens_seen": 355663872 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007209789368104314, + "loss": 1.5244, + "theoretical_loss": 4.093940501543857, + "tokens_seen": 355729408 + }, + { + "epoch": 0.11, + "learning_rate": 0.000720962888665998, + "loss": 1.5378, + "theoretical_loss": 4.093855058215513, + "tokens_seen": 355794944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007209468405215648, + "loss": 1.7013, + "theoretical_loss": 4.093769635029841, + "tokens_seen": 355860480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007209307923771314, + "loss": 1.6444, + "theoretical_loss": 4.0936842319783855, + "tokens_seen": 355926016 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007209147442326982, + "loss": 1.567, + "theoretical_loss": 4.093598849052694, + "tokens_seen": 355991552 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007208986960882648, + "loss": 1.7021, + "theoretical_loss": 4.093513486244321, + "tokens_seen": 356057088 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007208826479438316, + "loss": 1.5534, + "theoretical_loss": 4.093428143544823, + "tokens_seen": 356122624 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007208665997993982, + "loss": 1.6628, + "theoretical_loss": 4.093342820945766, + "tokens_seen": 356188160 + }, + { + "epoch": 0.11, + "learning_rate": 0.000720850551654965, + "loss": 1.5592, + "theoretical_loss": 4.093257518438718, + "tokens_seen": 356253696 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007208345035105316, + "loss": 1.6177, + "theoretical_loss": 4.093172236015253, + "tokens_seen": 356319232 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007208184553660984, + "loss": 1.6115, + "theoretical_loss": 4.093086973666949, + "tokens_seen": 356384768 + }, + { + "epoch": 0.11, + "learning_rate": 0.000720802407221665, + "loss": 1.6136, + "theoretical_loss": 4.0930017313853915, + "tokens_seen": 356450304 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007207863590772318, + "loss": 1.5544, + "theoretical_loss": 4.092916509162169, + "tokens_seen": 356515840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007207703109327984, + "loss": 1.6674, + "theoretical_loss": 4.092831306988875, + "tokens_seen": 356581376 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007207542627883652, + "loss": 1.6184, + "theoretical_loss": 4.0927461248571095, + "tokens_seen": 356646912 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007207382146439318, + "loss": 1.6125, + "theoretical_loss": 4.092660962758477, + "tokens_seen": 356712448 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007207221664994986, + "loss": 1.615, + "theoretical_loss": 4.092575820684586, + "tokens_seen": 356777984 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007207061183550653, + "loss": 1.5191, + "theoretical_loss": 4.092490698627052, + "tokens_seen": 356843520 + }, + { + "epoch": 0.11, + "learning_rate": 0.000720690070210632, + "loss": 1.6094, + "theoretical_loss": 4.0924055965774935, + "tokens_seen": 356909056 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007206740220661987, + "loss": 1.6726, + "theoretical_loss": 4.092320514527535, + "tokens_seen": 356974592 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007206579739217654, + "loss": 1.5941, + "theoretical_loss": 4.092235452468808, + "tokens_seen": 357040128 + }, + { + "epoch": 0.11, + "learning_rate": 0.000720641925777332, + "loss": 1.6075, + "theoretical_loss": 4.092150410392945, + "tokens_seen": 357105664 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.9929530024528503, + "objective/train/avg_token_score": 0.00704704225063324, + "objective/train/docs_used": 136331, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.7091403007507324, + "objective/train/original_loss": 1.7227659225463867, + "objective/train/theoretical_loss": 4.092065388291585, + "objective/train/tokens_used": 377631200, + "objective/train/unlikelihood_loss": 0.014746198430657387, + "theoretical_loss": 4.092065388291585, + "tokens_seen": 357171200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007206258776328988, + "loss": 1.7089, + "theoretical_loss": 4.092065388291585, + "tokens_seen": 357171200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007206098294884654, + "loss": 1.7033, + "theoretical_loss": 4.091980386156376, + "tokens_seen": 357236736 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007205937813440322, + "loss": 1.7072, + "theoretical_loss": 4.091895403978965, + "tokens_seen": 357302272 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007205777331995988, + "loss": 1.6296, + "theoretical_loss": 4.091810441751007, + "tokens_seen": 357367808 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007205616850551656, + "loss": 1.7279, + "theoretical_loss": 4.091725499464163, + "tokens_seen": 357433344 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007205456369107322, + "loss": 1.6717, + "theoretical_loss": 4.091640577110096, + "tokens_seen": 357498880 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007205295887662989, + "loss": 1.5792, + "theoretical_loss": 4.091555674680476, + "tokens_seen": 357564416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007205135406218656, + "loss": 1.5734, + "theoretical_loss": 4.09147079216698, + "tokens_seen": 357629952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007204974924774323, + "loss": 1.4202, + "theoretical_loss": 4.0913859295612856, + "tokens_seen": 357695488 + }, + { + "epoch": 0.11, + "learning_rate": 0.000720481444332999, + "loss": 1.6061, + "theoretical_loss": 4.091301086855078, + "tokens_seen": 357761024 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007204653961885657, + "loss": 1.6187, + "theoretical_loss": 4.091216264040048, + "tokens_seen": 357826560 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007204493480441324, + "loss": 1.4596, + "theoretical_loss": 4.091131461107889, + "tokens_seen": 357892096 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007204332998996991, + "loss": 1.5589, + "theoretical_loss": 4.091046678050301, + "tokens_seen": 357957632 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007204172517552658, + "loss": 1.597, + "theoretical_loss": 4.090961914858989, + "tokens_seen": 358023168 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007204012036108325, + "loss": 1.4985, + "theoretical_loss": 4.090877171525663, + "tokens_seen": 358088704 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007203851554663992, + "loss": 1.7261, + "theoretical_loss": 4.090792448042038, + "tokens_seen": 358154240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007203691073219659, + "loss": 1.5841, + "theoretical_loss": 4.0907077443998325, + "tokens_seen": 358219776 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007203530591775326, + "loss": 1.7095, + "theoretical_loss": 4.090623060590772, + "tokens_seen": 358285312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007203370110330993, + "loss": 1.654, + "theoretical_loss": 4.090538396606585, + "tokens_seen": 358350848 + }, + { + "epoch": 0.11, + "learning_rate": 0.000720320962888666, + "loss": 1.5941, + "theoretical_loss": 4.090453752439008, + "tokens_seen": 358416384 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007203049147442327, + "loss": 1.6289, + "theoretical_loss": 4.09036912807978, + "tokens_seen": 358481920 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007202888665997994, + "loss": 1.5938, + "theoretical_loss": 4.090284523520644, + "tokens_seen": 358547456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007202728184553661, + "loss": 1.4756, + "theoretical_loss": 4.09019993875335, + "tokens_seen": 358612992 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007202567703109328, + "loss": 1.6491, + "theoretical_loss": 4.090115373769653, + "tokens_seen": 358678528 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007202407221664995, + "loss": 1.5498, + "theoretical_loss": 4.0900308285613125, + "tokens_seen": 358744064 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007202246740220662, + "loss": 1.7097, + "theoretical_loss": 4.089946303120092, + "tokens_seen": 358809600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007202086258776329, + "loss": 1.5807, + "theoretical_loss": 4.089861797437761, + "tokens_seen": 358875136 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007201925777331996, + "loss": 1.6434, + "theoretical_loss": 4.089777311506094, + "tokens_seen": 358940672 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007201765295887663, + "loss": 1.6564, + "theoretical_loss": 4.089692845316868, + "tokens_seen": 359006208 + }, + { + "epoch": 0.11, + "learning_rate": 0.000720160481444333, + "loss": 1.5372, + "theoretical_loss": 4.08960839886187, + "tokens_seen": 359071744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007201444332998997, + "loss": 1.6017, + "theoretical_loss": 4.089523972132886, + "tokens_seen": 359137280 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007201283851554664, + "loss": 1.6885, + "theoretical_loss": 4.089439565121712, + "tokens_seen": 359202816 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007201123370110331, + "loss": 1.6977, + "theoretical_loss": 4.089355177820146, + "tokens_seen": 359268352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007200962888665998, + "loss": 1.5686, + "theoretical_loss": 4.0892708102199915, + "tokens_seen": 359333888 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007200802407221665, + "loss": 1.6653, + "theoretical_loss": 4.089186462313056, + "tokens_seen": 359399424 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007200641925777332, + "loss": 1.565, + "theoretical_loss": 4.089102134091155, + "tokens_seen": 359464960 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007200481444332999, + "loss": 1.6244, + "theoretical_loss": 4.0890178255461045, + "tokens_seen": 359530496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007200320962888666, + "loss": 1.6397, + "theoretical_loss": 4.08893353666973, + "tokens_seen": 359596032 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007200160481444333, + "loss": 1.5675, + "theoretical_loss": 4.088849267453858, + "tokens_seen": 359661568 + }, + { + "epoch": 0.11, + "learning_rate": 0.00072, + "loss": 1.5554, + "theoretical_loss": 4.088765017890322, + "tokens_seen": 359727104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007199839518555667, + "loss": 1.5745, + "theoretical_loss": 4.08868078797096, + "tokens_seen": 359792640 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007199679037111334, + "loss": 1.5396, + "theoretical_loss": 4.088596577687614, + "tokens_seen": 359858176 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007199518555667001, + "loss": 1.7328, + "theoretical_loss": 4.088512387032132, + "tokens_seen": 359923712 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007199358074222668, + "loss": 1.5632, + "theoretical_loss": 4.088428215996366, + "tokens_seen": 359989248 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007199197592778335, + "loss": 1.6366, + "theoretical_loss": 4.0883440645721745, + "tokens_seen": 360054784 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007199037111334002, + "loss": 1.6683, + "theoretical_loss": 4.088259932751419, + "tokens_seen": 360120320 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007198876629889669, + "loss": 1.5347, + "theoretical_loss": 4.088175820525966, + "tokens_seen": 360185856 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007198716148445337, + "loss": 1.6259, + "theoretical_loss": 4.088091727887688, + "tokens_seen": 360251392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007198555667001003, + "loss": 1.7176, + "theoretical_loss": 4.088007654828463, + "tokens_seen": 360316928 + }, + { + "epoch": 0.11, + "learning_rate": 0.000719839518555667, + "loss": 1.671, + "theoretical_loss": 4.087923601340169, + "tokens_seen": 360382464 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.991811215877533, + "objective/train/avg_token_score": 0.008188797160983086, + "objective/train/docs_used": 137426, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.7539641857147217, + "objective/train/original_loss": 1.770206332206726, + "objective/train/theoretical_loss": 4.0878395674146955, + "objective/train/tokens_used": 380908000, + "objective/train/unlikelihood_loss": 0.014311683364212513, + "theoretical_loss": 4.0878395674146955, + "tokens_seen": 360448000 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007198234704112337, + "loss": 1.7446, + "theoretical_loss": 4.0878395674146955, + "tokens_seen": 360448000 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007198074222668005, + "loss": 1.6169, + "theoretical_loss": 4.087755553043934, + "tokens_seen": 360513536 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007197913741223671, + "loss": 1.5536, + "theoretical_loss": 4.087671558219778, + "tokens_seen": 360579072 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007197753259779339, + "loss": 1.5165, + "theoretical_loss": 4.087587582934131, + "tokens_seen": 360644608 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007197592778335005, + "loss": 1.7031, + "theoretical_loss": 4.087503627178897, + "tokens_seen": 360710144 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007197432296890673, + "loss": 1.6981, + "theoretical_loss": 4.087419690945988, + "tokens_seen": 360775680 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007197271815446339, + "loss": 1.5713, + "theoretical_loss": 4.087335774227318, + "tokens_seen": 360841216 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007197111334002007, + "loss": 1.4843, + "theoretical_loss": 4.0872518770148085, + "tokens_seen": 360906752 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007196950852557673, + "loss": 1.591, + "theoretical_loss": 4.087167999300384, + "tokens_seen": 360972288 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007196790371113341, + "loss": 1.6564, + "theoretical_loss": 4.087084141075974, + "tokens_seen": 361037824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007196629889669007, + "loss": 1.6776, + "theoretical_loss": 4.087000302333513, + "tokens_seen": 361103360 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007196469408224675, + "loss": 1.7472, + "theoretical_loss": 4.086916483064941, + "tokens_seen": 361168896 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007196308926780341, + "loss": 1.6474, + "theoretical_loss": 4.086832683262203, + "tokens_seen": 361234432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007196148445336009, + "loss": 1.6351, + "theoretical_loss": 4.086748902917246, + "tokens_seen": 361299968 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007195987963891675, + "loss": 1.5866, + "theoretical_loss": 4.086665142022025, + "tokens_seen": 361365504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007195827482447343, + "loss": 1.6168, + "theoretical_loss": 4.086581400568499, + "tokens_seen": 361431040 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007195667001003009, + "loss": 1.6035, + "theoretical_loss": 4.086497678548631, + "tokens_seen": 361496576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007195506519558677, + "loss": 1.7653, + "theoretical_loss": 4.086413975954389, + "tokens_seen": 361562112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007195346038114343, + "loss": 1.7166, + "theoretical_loss": 4.086330292777745, + "tokens_seen": 361627648 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007195185556670011, + "loss": 1.6117, + "theoretical_loss": 4.086246629010679, + "tokens_seen": 361693184 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007195025075225678, + "loss": 1.6582, + "theoretical_loss": 4.086162984645172, + "tokens_seen": 361758720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007194864593781345, + "loss": 1.5413, + "theoretical_loss": 4.086079359673212, + "tokens_seen": 361824256 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007194704112337012, + "loss": 1.5415, + "theoretical_loss": 4.085995754086789, + "tokens_seen": 361889792 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007194543630892679, + "loss": 1.4831, + "theoretical_loss": 4.085912167877902, + "tokens_seen": 361955328 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007194383149448346, + "loss": 1.6391, + "theoretical_loss": 4.085828601038552, + "tokens_seen": 362020864 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007194222668004013, + "loss": 1.6148, + "theoretical_loss": 4.0857450535607445, + "tokens_seen": 362086400 + }, + { + "epoch": 0.11, + "learning_rate": 0.000719406218655968, + "loss": 1.7008, + "theoretical_loss": 4.085661525436492, + "tokens_seen": 362151936 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007193901705115347, + "loss": 1.7344, + "theoretical_loss": 4.0855780166578075, + "tokens_seen": 362217472 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007193741223671014, + "loss": 1.6207, + "theoretical_loss": 4.0854945272167145, + "tokens_seen": 362283008 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007193580742226681, + "loss": 1.5653, + "theoretical_loss": 4.085411057105237, + "tokens_seen": 362348544 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007193420260782348, + "loss": 1.6963, + "theoretical_loss": 4.0853276063154045, + "tokens_seen": 362414080 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007193259779338015, + "loss": 1.6041, + "theoretical_loss": 4.085244174839252, + "tokens_seen": 362479616 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007193099297893682, + "loss": 1.601, + "theoretical_loss": 4.085160762668819, + "tokens_seen": 362545152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007192938816449349, + "loss": 1.6366, + "theoretical_loss": 4.085077369796149, + "tokens_seen": 362610688 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007192778335005016, + "loss": 1.608, + "theoretical_loss": 4.084993996213292, + "tokens_seen": 362676224 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007192617853560683, + "loss": 1.6622, + "theoretical_loss": 4.084910641912301, + "tokens_seen": 362741760 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007192457372116349, + "loss": 1.6681, + "theoretical_loss": 4.084827306885234, + "tokens_seen": 362807296 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007192296890672017, + "loss": 1.6854, + "theoretical_loss": 4.084743991124155, + "tokens_seen": 362872832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007192136409227683, + "loss": 1.6212, + "theoretical_loss": 4.08466069462113, + "tokens_seen": 362938368 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007191975927783351, + "loss": 1.7928, + "theoretical_loss": 4.084577417368232, + "tokens_seen": 363003904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007191815446339017, + "loss": 1.6737, + "theoretical_loss": 4.084494159357538, + "tokens_seen": 363069440 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007191654964894685, + "loss": 1.7023, + "theoretical_loss": 4.0844109205811305, + "tokens_seen": 363134976 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007191494483450351, + "loss": 1.686, + "theoretical_loss": 4.084327701031095, + "tokens_seen": 363200512 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007191334002006019, + "loss": 1.6056, + "theoretical_loss": 4.084244500699524, + "tokens_seen": 363266048 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007191173520561685, + "loss": 1.6512, + "theoretical_loss": 4.084161319578511, + "tokens_seen": 363331584 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007191013039117352, + "loss": 1.6667, + "theoretical_loss": 4.084078157660159, + "tokens_seen": 363397120 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007190852557673019, + "loss": 1.5535, + "theoretical_loss": 4.083995014936571, + "tokens_seen": 363462656 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007190692076228686, + "loss": 1.6868, + "theoretical_loss": 4.0839118913998576, + "tokens_seen": 363528192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007190531594784353, + "loss": 1.5919, + "theoretical_loss": 4.083828787042134, + "tokens_seen": 363593728 + }, + { + "epoch": 0.11, + "learning_rate": 0.000719037111334002, + "loss": 1.6918, + "theoretical_loss": 4.083745701855518, + "tokens_seen": 363659264 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.9935913681983948, + "objective/train/avg_token_score": 0.006408608052879572, + "objective/train/docs_used": 138686, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.696382999420166, + "objective/train/original_loss": 1.7071021795272827, + "objective/train/theoretical_loss": 4.083662635832135, + "objective/train/tokens_used": 384184800, + "objective/train/unlikelihood_loss": 0.015040148049592972, + "theoretical_loss": 4.083662635832135, + "tokens_seen": 363724800 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007190210631895687, + "loss": 1.6815, + "theoretical_loss": 4.083662635832135, + "tokens_seen": 363724800 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007190050150451354, + "loss": 1.6982, + "theoretical_loss": 4.0835795889641116, + "tokens_seen": 363790336 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007189889669007021, + "loss": 1.5971, + "theoretical_loss": 4.0834965612435825, + "tokens_seen": 363855872 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007189729187562688, + "loss": 1.7173, + "theoretical_loss": 4.083413552662685, + "tokens_seen": 363921408 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007189568706118355, + "loss": 1.6753, + "theoretical_loss": 4.08333056321356, + "tokens_seen": 363986944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007189408224674022, + "loss": 1.5957, + "theoretical_loss": 4.083247592888357, + "tokens_seen": 364052480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007189247743229689, + "loss": 1.6649, + "theoretical_loss": 4.083164641679225, + "tokens_seen": 364118016 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007189087261785356, + "loss": 1.6475, + "theoretical_loss": 4.083081709578323, + "tokens_seen": 364183552 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007188926780341023, + "loss": 1.7219, + "theoretical_loss": 4.082998796577811, + "tokens_seen": 364249088 + }, + { + "epoch": 0.11, + "learning_rate": 0.000718876629889669, + "loss": 1.5865, + "theoretical_loss": 4.082915902669853, + "tokens_seen": 364314624 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007188605817452357, + "loss": 1.6946, + "theoretical_loss": 4.082833027846622, + "tokens_seen": 364380160 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007188445336008024, + "loss": 1.7294, + "theoretical_loss": 4.08275017210029, + "tokens_seen": 364445696 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007188284854563691, + "loss": 1.5839, + "theoretical_loss": 4.082667335423038, + "tokens_seen": 364511232 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007188124373119358, + "loss": 1.6611, + "theoretical_loss": 4.0825845178070495, + "tokens_seen": 364576768 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007187963891675025, + "loss": 1.5831, + "theoretical_loss": 4.0825017192445126, + "tokens_seen": 364642304 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007187803410230692, + "loss": 1.6148, + "theoretical_loss": 4.082418939727622, + "tokens_seen": 364707840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007187642928786359, + "loss": 1.5986, + "theoretical_loss": 4.082336179248574, + "tokens_seen": 364773376 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007187482447342026, + "loss": 1.5855, + "theoretical_loss": 4.082253437799572, + "tokens_seen": 364838912 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007187321965897693, + "loss": 1.6757, + "theoretical_loss": 4.082170715372823, + "tokens_seen": 364904448 + }, + { + "epoch": 0.11, + "learning_rate": 0.000718716148445336, + "loss": 1.7678, + "theoretical_loss": 4.082088011960538, + "tokens_seen": 364969984 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007187001003009027, + "loss": 1.6747, + "theoretical_loss": 4.082005327554934, + "tokens_seen": 365035520 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007186840521564694, + "loss": 1.5745, + "theoretical_loss": 4.081922662148231, + "tokens_seen": 365101056 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007186680040120361, + "loss": 1.6757, + "theoretical_loss": 4.081840015732654, + "tokens_seen": 365166592 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007186519558676028, + "loss": 1.6196, + "theoretical_loss": 4.081757388300435, + "tokens_seen": 365232128 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007186359077231695, + "loss": 1.5596, + "theoretical_loss": 4.081674779843806, + "tokens_seen": 365297664 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007186198595787362, + "loss": 1.6054, + "theoretical_loss": 4.081592190355008, + "tokens_seen": 365363200 + }, + { + "epoch": 0.11, + "learning_rate": 0.000718603811434303, + "loss": 1.5398, + "theoretical_loss": 4.081509619826283, + "tokens_seen": 365428736 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007185877632898696, + "loss": 1.5535, + "theoretical_loss": 4.081427068249881, + "tokens_seen": 365494272 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007185717151454363, + "loss": 1.6899, + "theoretical_loss": 4.081344535618053, + "tokens_seen": 365559808 + }, + { + "epoch": 0.11, + "learning_rate": 0.000718555667001003, + "loss": 1.604, + "theoretical_loss": 4.081262021923058, + "tokens_seen": 365625344 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007185396188565697, + "loss": 1.6925, + "theoretical_loss": 4.081179527157156, + "tokens_seen": 365690880 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007185235707121364, + "loss": 1.5963, + "theoretical_loss": 4.081097051312614, + "tokens_seen": 365756416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007185075225677032, + "loss": 1.5902, + "theoretical_loss": 4.081014594381705, + "tokens_seen": 365821952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007184914744232698, + "loss": 1.6133, + "theoretical_loss": 4.0809321563567025, + "tokens_seen": 365887488 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007184754262788366, + "loss": 1.5401, + "theoretical_loss": 4.080849737229887, + "tokens_seen": 365953024 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007184593781344032, + "loss": 1.6296, + "theoretical_loss": 4.080767336993542, + "tokens_seen": 366018560 + }, + { + "epoch": 0.11, + "learning_rate": 0.00071844332998997, + "loss": 1.628, + "theoretical_loss": 4.080684955639958, + "tokens_seen": 366084096 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007184272818455366, + "loss": 1.6959, + "theoretical_loss": 4.080602593161428, + "tokens_seen": 366149632 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007184112337011034, + "loss": 1.6116, + "theoretical_loss": 4.0805202495502515, + "tokens_seen": 366215168 + }, + { + "epoch": 0.11, + "learning_rate": 0.00071839518555667, + "loss": 1.6326, + "theoretical_loss": 4.080437924798729, + "tokens_seen": 366280704 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007183791374122368, + "loss": 1.5412, + "theoretical_loss": 4.080355618899169, + "tokens_seen": 366346240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007183630892678034, + "loss": 1.6189, + "theoretical_loss": 4.080273331843884, + "tokens_seen": 366411776 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007183470411233702, + "loss": 1.6923, + "theoretical_loss": 4.080191063625188, + "tokens_seen": 366477312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007183309929789368, + "loss": 1.5648, + "theoretical_loss": 4.080108814235404, + "tokens_seen": 366542848 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007183149448345036, + "loss": 1.6069, + "theoretical_loss": 4.080026583666855, + "tokens_seen": 366608384 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007182988966900703, + "loss": 1.6143, + "theoretical_loss": 4.079944371911871, + "tokens_seen": 366673920 + }, + { + "epoch": 0.11, + "learning_rate": 0.000718282848545637, + "loss": 1.7197, + "theoretical_loss": 4.07986217896279, + "tokens_seen": 366739456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007182668004012037, + "loss": 1.7419, + "theoretical_loss": 4.079780004811946, + "tokens_seen": 366804992 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007182507522567704, + "loss": 1.6147, + "theoretical_loss": 4.079697849451684, + "tokens_seen": 366870528 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007182347041123371, + "loss": 1.8676, + "theoretical_loss": 4.079615712874352, + "tokens_seen": 366936064 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.9951712489128113, + "objective/train/avg_token_score": 0.004828761797398329, + "objective/train/docs_used": 139849, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6924651861190796, + "objective/train/original_loss": 1.702858567237854, + "objective/train/theoretical_loss": 4.079533595072301, + "objective/train/tokens_used": 387461600, + "objective/train/unlikelihood_loss": 0.008062578737735748, + "theoretical_loss": 4.079533595072301, + "tokens_seen": 367001600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007182186559679038, + "loss": 1.671, + "theoretical_loss": 4.079533595072301, + "tokens_seen": 367001600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007182026078234705, + "loss": 1.6862, + "theoretical_loss": 4.0794514960378905, + "tokens_seen": 367067136 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007181865596790372, + "loss": 1.6777, + "theoretical_loss": 4.079369415763479, + "tokens_seen": 367132672 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007181705115346039, + "loss": 1.6127, + "theoretical_loss": 4.079287354241432, + "tokens_seen": 367198208 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007181544633901706, + "loss": 1.653, + "theoretical_loss": 4.07920531146412, + "tokens_seen": 367263744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007181384152457373, + "loss": 1.6427, + "theoretical_loss": 4.0791232874239185, + "tokens_seen": 367329280 + }, + { + "epoch": 0.11, + "learning_rate": 0.000718122367101304, + "loss": 1.6125, + "theoretical_loss": 4.079041282113206, + "tokens_seen": 367394816 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007181063189568707, + "loss": 1.7516, + "theoretical_loss": 4.078959295524365, + "tokens_seen": 367460352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007180902708124374, + "loss": 1.659, + "theoretical_loss": 4.078877327649785, + "tokens_seen": 367525888 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007180742226680041, + "loss": 1.5297, + "theoretical_loss": 4.078795378481858, + "tokens_seen": 367591424 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007180581745235708, + "loss": 1.5573, + "theoretical_loss": 4.07871344801298, + "tokens_seen": 367656960 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007180421263791375, + "loss": 1.7163, + "theoretical_loss": 4.0786315362355525, + "tokens_seen": 367722496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007180260782347042, + "loss": 1.6323, + "theoretical_loss": 4.078549643141981, + "tokens_seen": 367788032 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007180100300902709, + "loss": 1.6065, + "theoretical_loss": 4.078467768724677, + "tokens_seen": 367853568 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007179939819458376, + "loss": 1.5684, + "theoretical_loss": 4.078385912976053, + "tokens_seen": 367919104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007179779338014043, + "loss": 1.6355, + "theoretical_loss": 4.078304075888529, + "tokens_seen": 367984640 + }, + { + "epoch": 0.11, + "learning_rate": 0.000717961885656971, + "loss": 1.6821, + "theoretical_loss": 4.078222257454529, + "tokens_seen": 368050176 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007179458375125377, + "loss": 1.662, + "theoretical_loss": 4.07814045766648, + "tokens_seen": 368115712 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007179297893681044, + "loss": 1.6372, + "theoretical_loss": 4.0780586765168145, + "tokens_seen": 368181248 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007179137412236711, + "loss": 1.4973, + "theoretical_loss": 4.07797691399797, + "tokens_seen": 368246784 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007178976930792378, + "loss": 1.7037, + "theoretical_loss": 4.077895170102386, + "tokens_seen": 368312320 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007178816449348045, + "loss": 1.6498, + "theoretical_loss": 4.077813444822509, + "tokens_seen": 368377856 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007178655967903712, + "loss": 1.5559, + "theoretical_loss": 4.077731738150789, + "tokens_seen": 368443392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007178495486459378, + "loss": 1.5991, + "theoretical_loss": 4.077650050079681, + "tokens_seen": 368508928 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007178335005015046, + "loss": 1.5615, + "theoretical_loss": 4.077568380601642, + "tokens_seen": 368574464 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007178174523570712, + "loss": 1.5027, + "theoretical_loss": 4.077486729709136, + "tokens_seen": 368640000 + }, + { + "epoch": 0.11, + "learning_rate": 0.000717801404212638, + "loss": 1.5693, + "theoretical_loss": 4.077405097394631, + "tokens_seen": 368705536 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007177853560682046, + "loss": 1.6987, + "theoretical_loss": 4.077323483650598, + "tokens_seen": 368771072 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007177693079237714, + "loss": 1.6999, + "theoretical_loss": 4.077241888469514, + "tokens_seen": 368836608 + }, + { + "epoch": 0.11, + "learning_rate": 0.000717753259779338, + "loss": 1.6289, + "theoretical_loss": 4.077160311843858, + "tokens_seen": 368902144 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007177372116349048, + "loss": 1.6178, + "theoretical_loss": 4.077078753766118, + "tokens_seen": 368967680 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007177211634904714, + "loss": 1.7899, + "theoretical_loss": 4.076997214228782, + "tokens_seen": 369033216 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007177051153460381, + "loss": 1.6251, + "theoretical_loss": 4.076915693224343, + "tokens_seen": 369098752 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007176890672016048, + "loss": 1.7177, + "theoretical_loss": 4.0768341907453, + "tokens_seen": 369164288 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007176730190571715, + "loss": 1.6142, + "theoretical_loss": 4.0767527067841565, + "tokens_seen": 369229824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007176569709127382, + "loss": 1.5564, + "theoretical_loss": 4.076671241333417, + "tokens_seen": 369295360 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007176409227683049, + "loss": 1.6002, + "theoretical_loss": 4.076589794385596, + "tokens_seen": 369360896 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007176248746238716, + "loss": 1.5614, + "theoretical_loss": 4.076508365933206, + "tokens_seen": 369426432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007176088264794383, + "loss": 1.6753, + "theoretical_loss": 4.076426955968769, + "tokens_seen": 369491968 + }, + { + "epoch": 0.11, + "learning_rate": 0.000717592778335005, + "loss": 1.6432, + "theoretical_loss": 4.076345564484807, + "tokens_seen": 369557504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007175767301905717, + "loss": 1.5822, + "theoretical_loss": 4.076264191473852, + "tokens_seen": 369623040 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007175606820461384, + "loss": 1.6975, + "theoretical_loss": 4.076182836928435, + "tokens_seen": 369688576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007175446339017051, + "loss": 1.7147, + "theoretical_loss": 4.0761015008410935, + "tokens_seen": 369754112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007175285857572718, + "loss": 1.6723, + "theoretical_loss": 4.07602018320437, + "tokens_seen": 369819648 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007175125376128385, + "loss": 1.5604, + "theoretical_loss": 4.075938884010809, + "tokens_seen": 369885184 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007174964894684052, + "loss": 1.5921, + "theoretical_loss": 4.075857603252963, + "tokens_seen": 369950720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007174804413239719, + "loss": 1.5917, + "theoretical_loss": 4.075776340923385, + "tokens_seen": 370016256 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007174643931795386, + "loss": 1.6641, + "theoretical_loss": 4.075695097014635, + "tokens_seen": 370081792 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007174483450351053, + "loss": 1.6365, + "theoretical_loss": 4.075613871519276, + "tokens_seen": 370147328 + }, + { + "epoch": 0.11, + "learning_rate": 0.000717432296890672, + "loss": 1.6822, + "theoretical_loss": 4.0755326644298755, + "tokens_seen": 370212864 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.9896782040596008, + "objective/train/avg_token_score": 0.010321811772882938, + "objective/train/docs_used": 141078, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5975168943405151, + "objective/train/original_loss": 1.615493893623352, + "objective/train/theoretical_loss": 4.075451475739007, + "objective/train/tokens_used": 390738400, + "objective/train/unlikelihood_loss": 0.021900298073887825, + "theoretical_loss": 4.075451475739007, + "tokens_seen": 370278400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007174162487462387, + "loss": 1.55, + "theoretical_loss": 4.075451475739007, + "tokens_seen": 370278400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007174002006018054, + "loss": 1.6199, + "theoretical_loss": 4.075370305439243, + "tokens_seen": 370343936 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007173841524573721, + "loss": 1.5603, + "theoretical_loss": 4.075289153523168, + "tokens_seen": 370409472 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007173681043129388, + "loss": 1.6033, + "theoretical_loss": 4.075208019983366, + "tokens_seen": 370475008 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007173520561685055, + "loss": 1.7111, + "theoretical_loss": 4.075126904812425, + "tokens_seen": 370540544 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007173360080240722, + "loss": 1.5611, + "theoretical_loss": 4.0750458080029395, + "tokens_seen": 370606080 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007173199598796389, + "loss": 1.6559, + "theoretical_loss": 4.074964729547506, + "tokens_seen": 370671616 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007173039117352056, + "loss": 1.5453, + "theoretical_loss": 4.074883669438729, + "tokens_seen": 370737152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007172878635907723, + "loss": 1.6371, + "theoretical_loss": 4.074802627669213, + "tokens_seen": 370802688 + }, + { + "epoch": 0.11, + "learning_rate": 0.000717271815446339, + "loss": 1.6256, + "theoretical_loss": 4.07472160423157, + "tokens_seen": 370868224 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007172557673019057, + "loss": 1.5926, + "theoretical_loss": 4.074640599118413, + "tokens_seen": 370933760 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007172397191574724, + "loss": 1.6051, + "theoretical_loss": 4.074559612322363, + "tokens_seen": 370999296 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007172236710130391, + "loss": 1.6682, + "theoretical_loss": 4.074478643836043, + "tokens_seen": 371064832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007172076228686058, + "loss": 1.6466, + "theoretical_loss": 4.074397693652081, + "tokens_seen": 371130368 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007171915747241725, + "loss": 1.6507, + "theoretical_loss": 4.074316761763107, + "tokens_seen": 371195904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007171755265797392, + "loss": 1.6849, + "theoretical_loss": 4.074235848161761, + "tokens_seen": 371261440 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007171594784353059, + "loss": 1.6588, + "theoretical_loss": 4.074154952840681, + "tokens_seen": 371326976 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007171434302908726, + "loss": 1.6105, + "theoretical_loss": 4.074074075792513, + "tokens_seen": 371392512 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007171273821464393, + "loss": 1.6823, + "theoretical_loss": 4.073993217009905, + "tokens_seen": 371458048 + }, + { + "epoch": 0.11, + "learning_rate": 0.000717111334002006, + "loss": 1.5373, + "theoretical_loss": 4.073912376485511, + "tokens_seen": 371523584 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007170952858575728, + "loss": 1.6053, + "theoretical_loss": 4.073831554211989, + "tokens_seen": 371589120 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007170792377131395, + "loss": 1.5693, + "theoretical_loss": 4.0737507501820005, + "tokens_seen": 371654656 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007170631895687062, + "loss": 1.589, + "theoretical_loss": 4.073669964388212, + "tokens_seen": 371720192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007170471414242729, + "loss": 1.7747, + "theoretical_loss": 4.073589196823293, + "tokens_seen": 371785728 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007170310932798396, + "loss": 1.7192, + "theoretical_loss": 4.073508447479918, + "tokens_seen": 371851264 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007170150451354063, + "loss": 1.6624, + "theoretical_loss": 4.073427716350767, + "tokens_seen": 371916800 + }, + { + "epoch": 0.11, + "learning_rate": 0.000716998996990973, + "loss": 1.7861, + "theoretical_loss": 4.0733470034285215, + "tokens_seen": 371982336 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007169829488465397, + "loss": 1.5653, + "theoretical_loss": 4.07326630870587, + "tokens_seen": 372047872 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007169669007021064, + "loss": 1.5843, + "theoretical_loss": 4.073185632175504, + "tokens_seen": 372113408 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007169508525576731, + "loss": 1.6793, + "theoretical_loss": 4.073104973830118, + "tokens_seen": 372178944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007169348044132398, + "loss": 1.7351, + "theoretical_loss": 4.073024333662413, + "tokens_seen": 372244480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007169187562688065, + "loss": 1.6421, + "theoretical_loss": 4.072943711665093, + "tokens_seen": 372310016 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007169027081243732, + "loss": 1.4749, + "theoretical_loss": 4.072863107830866, + "tokens_seen": 372375552 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007168866599799399, + "loss": 1.6343, + "theoretical_loss": 4.072782522152444, + "tokens_seen": 372441088 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007168706118355066, + "loss": 1.6157, + "theoretical_loss": 4.072701954622546, + "tokens_seen": 372506624 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007168545636910733, + "loss": 1.7292, + "theoretical_loss": 4.072621405233891, + "tokens_seen": 372572160 + }, + { + "epoch": 0.11, + "learning_rate": 0.00071683851554664, + "loss": 1.6774, + "theoretical_loss": 4.072540873979205, + "tokens_seen": 372637696 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007168224674022067, + "loss": 1.5956, + "theoretical_loss": 4.072460360851217, + "tokens_seen": 372703232 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007168064192577734, + "loss": 1.624, + "theoretical_loss": 4.07237986584266, + "tokens_seen": 372768768 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007167903711133401, + "loss": 1.5774, + "theoretical_loss": 4.0722993889462735, + "tokens_seen": 372834304 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007167743229689068, + "loss": 1.61, + "theoretical_loss": 4.072218930154797, + "tokens_seen": 372899840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007167582748244735, + "loss": 1.5067, + "theoretical_loss": 4.072138489460979, + "tokens_seen": 372965376 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007167422266800402, + "loss": 1.5922, + "theoretical_loss": 4.0720580668575685, + "tokens_seen": 373030912 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007167261785356069, + "loss": 1.7088, + "theoretical_loss": 4.0719776623373205, + "tokens_seen": 373096448 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007167101303911736, + "loss": 1.6063, + "theoretical_loss": 4.071897275892994, + "tokens_seen": 373161984 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007166940822467403, + "loss": 1.5991, + "theoretical_loss": 4.071816907517349, + "tokens_seen": 373227520 + }, + { + "epoch": 0.11, + "learning_rate": 0.000716678034102307, + "loss": 1.5677, + "theoretical_loss": 4.071736557203157, + "tokens_seen": 373293056 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007166619859578737, + "loss": 1.5886, + "theoretical_loss": 4.071656224943186, + "tokens_seen": 373358592 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007166459378134404, + "loss": 1.6033, + "theoretical_loss": 4.071575910730212, + "tokens_seen": 373424128 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007166298896690071, + "loss": 1.4658, + "theoretical_loss": 4.071495614557016, + "tokens_seen": 373489664 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.9907689690589905, + "objective/train/avg_token_score": 0.009231050498783588, + "objective/train/docs_used": 142333, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6837348937988281, + "objective/train/original_loss": 1.7008459568023682, + "objective/train/theoretical_loss": 4.071415336416378, + "objective/train/tokens_used": 394015200, + "objective/train/unlikelihood_loss": 0.018875285983085632, + "theoretical_loss": 4.071415336416378, + "tokens_seen": 373555200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007166138415245738, + "loss": 1.6876, + "theoretical_loss": 4.071415336416378, + "tokens_seen": 373555200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007165977933801405, + "loss": 1.5099, + "theoretical_loss": 4.071335076301089, + "tokens_seen": 373620736 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007165817452357072, + "loss": 1.6767, + "theoretical_loss": 4.071254834203941, + "tokens_seen": 373686272 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007165656970912739, + "loss": 1.7183, + "theoretical_loss": 4.071174610117729, + "tokens_seen": 373751808 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007165496489468406, + "loss": 1.6287, + "theoretical_loss": 4.071094404035253, + "tokens_seen": 373817344 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007165336008024073, + "loss": 1.6288, + "theoretical_loss": 4.071014215949317, + "tokens_seen": 373882880 + }, + { + "epoch": 0.11, + "learning_rate": 0.000716517552657974, + "loss": 1.487, + "theoretical_loss": 4.070934045852731, + "tokens_seen": 373948416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007165015045135407, + "loss": 1.6415, + "theoretical_loss": 4.070853893738307, + "tokens_seen": 374013952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007164854563691074, + "loss": 1.6725, + "theoretical_loss": 4.070773759598861, + "tokens_seen": 374079488 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007164694082246741, + "loss": 1.5828, + "theoretical_loss": 4.070693643427215, + "tokens_seen": 374145024 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007164533600802407, + "loss": 1.5546, + "theoretical_loss": 4.070613545216193, + "tokens_seen": 374210560 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007164373119358075, + "loss": 1.6135, + "theoretical_loss": 4.070533464958626, + "tokens_seen": 374276096 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007164212637913741, + "loss": 1.5815, + "theoretical_loss": 4.070453402647344, + "tokens_seen": 374341632 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007164052156469409, + "loss": 1.6922, + "theoretical_loss": 4.070373358275187, + "tokens_seen": 374407168 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007163891675025075, + "loss": 1.5625, + "theoretical_loss": 4.070293331834996, + "tokens_seen": 374472704 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007163731193580743, + "loss": 1.6519, + "theoretical_loss": 4.070213323319616, + "tokens_seen": 374538240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007163570712136409, + "loss": 1.7853, + "theoretical_loss": 4.070133332721897, + "tokens_seen": 374603776 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007163410230692077, + "loss": 1.6538, + "theoretical_loss": 4.070053360034693, + "tokens_seen": 374669312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007163249749247743, + "loss": 1.6098, + "theoretical_loss": 4.069973405250862, + "tokens_seen": 374734848 + }, + { + "epoch": 0.11, + "learning_rate": 0.000716308926780341, + "loss": 1.6426, + "theoretical_loss": 4.069893468363265, + "tokens_seen": 374800384 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007162928786359077, + "loss": 1.6082, + "theoretical_loss": 4.06981354936477, + "tokens_seen": 374865920 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007162768304914744, + "loss": 1.6034, + "theoretical_loss": 4.069733648248246, + "tokens_seen": 374931456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007162607823470411, + "loss": 1.5939, + "theoretical_loss": 4.069653765006565, + "tokens_seen": 374996992 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007162447342026078, + "loss": 1.576, + "theoretical_loss": 4.06957389963261, + "tokens_seen": 375062528 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007162286860581745, + "loss": 1.6436, + "theoretical_loss": 4.069494052119261, + "tokens_seen": 375128064 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007162126379137412, + "loss": 1.6433, + "theoretical_loss": 4.069414222459403, + "tokens_seen": 375193600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007161965897693079, + "loss": 1.706, + "theoretical_loss": 4.069334410645928, + "tokens_seen": 375259136 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007161805416248746, + "loss": 1.5855, + "theoretical_loss": 4.069254616671732, + "tokens_seen": 375324672 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007161644934804413, + "loss": 1.6992, + "theoretical_loss": 4.069174840529712, + "tokens_seen": 375390208 + }, + { + "epoch": 0.11, + "learning_rate": 0.000716148445336008, + "loss": 1.7346, + "theoretical_loss": 4.069095082212772, + "tokens_seen": 375455744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007161323971915747, + "loss": 1.6354, + "theoretical_loss": 4.0690153417138175, + "tokens_seen": 375521280 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007161163490471414, + "loss": 1.7744, + "theoretical_loss": 4.06893561902576, + "tokens_seen": 375586816 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007161003009027081, + "loss": 1.6922, + "theoretical_loss": 4.068855914141515, + "tokens_seen": 375652352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007160842527582748, + "loss": 1.7186, + "theoretical_loss": 4.068776227054, + "tokens_seen": 375717888 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007160682046138415, + "loss": 1.5755, + "theoretical_loss": 4.06869655775614, + "tokens_seen": 375783424 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007160521564694082, + "loss": 1.626, + "theoretical_loss": 4.068616906240861, + "tokens_seen": 375848960 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007160361083249749, + "loss": 1.6163, + "theoretical_loss": 4.068537272501095, + "tokens_seen": 375914496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007160200601805416, + "loss": 1.5644, + "theoretical_loss": 4.068457656529775, + "tokens_seen": 375980032 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007160040120361083, + "loss": 1.7622, + "theoretical_loss": 4.068378058319843, + "tokens_seen": 376045568 + }, + { + "epoch": 0.11, + "learning_rate": 0.000715987963891675, + "loss": 1.7686, + "theoretical_loss": 4.068298477864241, + "tokens_seen": 376111104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007159719157472417, + "loss": 1.573, + "theoretical_loss": 4.068218915155917, + "tokens_seen": 376176640 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007159558676028084, + "loss": 1.6102, + "theoretical_loss": 4.068139370187821, + "tokens_seen": 376242176 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007159398194583751, + "loss": 1.6427, + "theoretical_loss": 4.06805984295291, + "tokens_seen": 376307712 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007159237713139418, + "loss": 1.5808, + "theoretical_loss": 4.067980333444142, + "tokens_seen": 376373248 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007159077231695085, + "loss": 1.6674, + "theoretical_loss": 4.067900841654481, + "tokens_seen": 376438784 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007158916750250753, + "loss": 1.6912, + "theoretical_loss": 4.067821367576894, + "tokens_seen": 376504320 + }, + { + "epoch": 0.11, + "learning_rate": 0.000715875626880642, + "loss": 1.6056, + "theoretical_loss": 4.067741911204354, + "tokens_seen": 376569856 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007158595787362087, + "loss": 1.5689, + "theoretical_loss": 4.067662472529834, + "tokens_seen": 376635392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007158435305917753, + "loss": 1.7963, + "theoretical_loss": 4.067583051546316, + "tokens_seen": 376700928 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007158274824473421, + "loss": 1.6395, + "theoretical_loss": 4.067503648246781, + "tokens_seen": 376766464 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.9925506711006165, + "objective/train/avg_token_score": 0.0074493479914963245, + "objective/train/docs_used": 143470, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6793549060821533, + "objective/train/original_loss": 1.6934773921966553, + "objective/train/theoretical_loss": 4.067424262624218, + "objective/train/tokens_used": 397292000, + "objective/train/unlikelihood_loss": 0.014788336120545864, + "theoretical_loss": 4.067424262624218, + "tokens_seen": 376832000 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007158114343029087, + "loss": 1.6745, + "theoretical_loss": 4.067424262624218, + "tokens_seen": 376832000 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007157953861584755, + "loss": 1.7235, + "theoretical_loss": 4.06734489467162, + "tokens_seen": 376897536 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007157793380140421, + "loss": 1.5962, + "theoretical_loss": 4.0672655443819785, + "tokens_seen": 376963072 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007157632898696089, + "loss": 1.5945, + "theoretical_loss": 4.067186211748297, + "tokens_seen": 377028608 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007157472417251756, + "loss": 1.6667, + "theoretical_loss": 4.067106896763575, + "tokens_seen": 377094144 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007157311935807423, + "loss": 1.5834, + "theoretical_loss": 4.067027599420824, + "tokens_seen": 377159680 + }, + { + "epoch": 0.11, + "learning_rate": 0.000715715145436309, + "loss": 1.584, + "theoretical_loss": 4.0669483197130525, + "tokens_seen": 377225216 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007156990972918757, + "loss": 1.5604, + "theoretical_loss": 4.066869057633277, + "tokens_seen": 377290752 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007156830491474424, + "loss": 1.5744, + "theoretical_loss": 4.066789813174517, + "tokens_seen": 377356288 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007156670010030091, + "loss": 1.5834, + "theoretical_loss": 4.066710586329796, + "tokens_seen": 377421824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007156509528585758, + "loss": 1.637, + "theoretical_loss": 4.066631377092141, + "tokens_seen": 377487360 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007156349047141425, + "loss": 1.6289, + "theoretical_loss": 4.066552185454582, + "tokens_seen": 377552896 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007156188565697092, + "loss": 1.5, + "theoretical_loss": 4.066473011410157, + "tokens_seen": 377618432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007156028084252759, + "loss": 1.5586, + "theoretical_loss": 4.066393854951903, + "tokens_seen": 377683968 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007155867602808426, + "loss": 1.6468, + "theoretical_loss": 4.066314716072864, + "tokens_seen": 377749504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007155707121364093, + "loss": 1.5136, + "theoretical_loss": 4.066235594766088, + "tokens_seen": 377815040 + }, + { + "epoch": 0.11, + "learning_rate": 0.000715554663991976, + "loss": 1.5705, + "theoretical_loss": 4.066156491024625, + "tokens_seen": 377880576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007155386158475427, + "loss": 1.544, + "theoretical_loss": 4.06607740484153, + "tokens_seen": 377946112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007155225677031094, + "loss": 1.7605, + "theoretical_loss": 4.065998336209863, + "tokens_seen": 378011648 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007155065195586761, + "loss": 1.6804, + "theoretical_loss": 4.065919285122686, + "tokens_seen": 378077184 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007154904714142428, + "loss": 1.5955, + "theoretical_loss": 4.065840251573066, + "tokens_seen": 378142720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007154744232698095, + "loss": 1.7858, + "theoretical_loss": 4.065761235554075, + "tokens_seen": 378208256 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007154583751253762, + "loss": 1.5176, + "theoretical_loss": 4.065682237058787, + "tokens_seen": 378273792 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007154423269809429, + "loss": 1.6189, + "theoretical_loss": 4.06560325608028, + "tokens_seen": 378339328 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007154262788365096, + "loss": 1.5269, + "theoretical_loss": 4.065524292611637, + "tokens_seen": 378404864 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007154102306920763, + "loss": 1.59, + "theoretical_loss": 4.065445346645945, + "tokens_seen": 378470400 + }, + { + "epoch": 0.11, + "learning_rate": 0.000715394182547643, + "loss": 1.6077, + "theoretical_loss": 4.0653664181762945, + "tokens_seen": 378535936 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007153781344032097, + "loss": 1.7089, + "theoretical_loss": 4.06528750719578, + "tokens_seen": 378601472 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007153620862587764, + "loss": 1.7, + "theoretical_loss": 4.0652086136975, + "tokens_seen": 378667008 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007153460381143431, + "loss": 1.5714, + "theoretical_loss": 4.065129737674556, + "tokens_seen": 378732544 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007153299899699098, + "loss": 1.4857, + "theoretical_loss": 4.065050879120054, + "tokens_seen": 378798080 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007153139418254765, + "loss": 1.5559, + "theoretical_loss": 4.064972038027106, + "tokens_seen": 378863616 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007152978936810432, + "loss": 1.6184, + "theoretical_loss": 4.064893214388825, + "tokens_seen": 378929152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007152818455366099, + "loss": 1.5684, + "theoretical_loss": 4.064814408198327, + "tokens_seen": 378994688 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007152657973921766, + "loss": 1.6581, + "theoretical_loss": 4.0647356194487365, + "tokens_seen": 379060224 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007152497492477433, + "loss": 1.5291, + "theoretical_loss": 4.064656848133178, + "tokens_seen": 379125760 + }, + { + "epoch": 0.11, + "learning_rate": 0.00071523370110331, + "loss": 1.6074, + "theoretical_loss": 4.064578094244782, + "tokens_seen": 379191296 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007152176529588767, + "loss": 1.5395, + "theoretical_loss": 4.06449935777668, + "tokens_seen": 379256832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007152016048144434, + "loss": 1.589, + "theoretical_loss": 4.0644206387220105, + "tokens_seen": 379322368 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007151855566700101, + "loss": 1.5956, + "theoretical_loss": 4.064341937073916, + "tokens_seen": 379387904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007151695085255768, + "loss": 1.6933, + "theoretical_loss": 4.0642632528255405, + "tokens_seen": 379453440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007151534603811435, + "loss": 1.7746, + "theoretical_loss": 4.064184585970033, + "tokens_seen": 379518976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007151374122367102, + "loss": 1.5399, + "theoretical_loss": 4.064105936500547, + "tokens_seen": 379584512 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007151213640922769, + "loss": 1.5361, + "theoretical_loss": 4.064027304410239, + "tokens_seen": 379650048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007151053159478436, + "loss": 1.6196, + "theoretical_loss": 4.063948689692269, + "tokens_seen": 379715584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007150892678034103, + "loss": 1.707, + "theoretical_loss": 4.063870092339803, + "tokens_seen": 379781120 + }, + { + "epoch": 0.12, + "learning_rate": 0.000715073219658977, + "loss": 1.6045, + "theoretical_loss": 4.0637915123460076, + "tokens_seen": 379846656 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007150571715145436, + "loss": 1.6171, + "theoretical_loss": 4.0637129497040565, + "tokens_seen": 379912192 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007150411233701104, + "loss": 1.5558, + "theoretical_loss": 4.063634404407126, + "tokens_seen": 379977728 + }, + { + "epoch": 0.12, + "learning_rate": 0.000715025075225677, + "loss": 1.6322, + "theoretical_loss": 4.063555876448395, + "tokens_seen": 380043264 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.9942775964736938, + "objective/train/avg_token_score": 0.005722436122596264, + "objective/train/docs_used": 144698, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5490413904190063, + "objective/train/original_loss": 1.5589340925216675, + "objective/train/theoretical_loss": 4.063477365821048, + "objective/train/tokens_used": 400568800, + "objective/train/unlikelihood_loss": 0.01109250821173191, + "theoretical_loss": 4.063477365821048, + "tokens_seen": 380108800 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007150090270812438, + "loss": 1.5249, + "theoretical_loss": 4.063477365821048, + "tokens_seen": 380108800 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007149929789368104, + "loss": 1.6464, + "theoretical_loss": 4.063398872518273, + "tokens_seen": 380174336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007149769307923772, + "loss": 1.593, + "theoretical_loss": 4.0633203965332605, + "tokens_seen": 380239872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007149608826479438, + "loss": 1.5895, + "theoretical_loss": 4.063241937859207, + "tokens_seen": 380305408 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007149448345035106, + "loss": 1.6699, + "theoretical_loss": 4.063163496489312, + "tokens_seen": 380370944 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007149287863590772, + "loss": 1.6495, + "theoretical_loss": 4.0630850724167775, + "tokens_seen": 380436480 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007149127382146439, + "loss": 1.6068, + "theoretical_loss": 4.063006665634811, + "tokens_seen": 380502016 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007148966900702106, + "loss": 1.6678, + "theoretical_loss": 4.062928276136624, + "tokens_seen": 380567552 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007148806419257773, + "loss": 1.5561, + "theoretical_loss": 4.0628499039154296, + "tokens_seen": 380633088 + }, + { + "epoch": 0.12, + "learning_rate": 0.000714864593781344, + "loss": 1.499, + "theoretical_loss": 4.062771548964447, + "tokens_seen": 380698624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007148485456369107, + "loss": 1.5426, + "theoretical_loss": 4.062693211276899, + "tokens_seen": 380764160 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007148324974924774, + "loss": 1.6439, + "theoretical_loss": 4.062614890846012, + "tokens_seen": 380829696 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007148164493480441, + "loss": 1.6402, + "theoretical_loss": 4.062536587665014, + "tokens_seen": 380895232 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007148004012036108, + "loss": 1.5591, + "theoretical_loss": 4.0624583017271405, + "tokens_seen": 380960768 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007147843530591775, + "loss": 1.7602, + "theoretical_loss": 4.062380033025629, + "tokens_seen": 381026304 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007147683049147442, + "loss": 1.6632, + "theoretical_loss": 4.062301781553719, + "tokens_seen": 381091840 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007147522567703109, + "loss": 1.5195, + "theoretical_loss": 4.062223547304658, + "tokens_seen": 381157376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007147362086258776, + "loss": 1.5865, + "theoretical_loss": 4.0621453302716946, + "tokens_seen": 381222912 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007147201604814443, + "loss": 1.5433, + "theoretical_loss": 4.062067130448081, + "tokens_seen": 381288448 + }, + { + "epoch": 0.12, + "learning_rate": 0.000714704112337011, + "loss": 1.5787, + "theoretical_loss": 4.061988947827073, + "tokens_seen": 381353984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007146880641925778, + "loss": 1.6202, + "theoretical_loss": 4.061910782401933, + "tokens_seen": 381419520 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007146720160481444, + "loss": 1.747, + "theoretical_loss": 4.061832634165923, + "tokens_seen": 381485056 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007146559679037112, + "loss": 1.5889, + "theoretical_loss": 4.061754503112313, + "tokens_seen": 381550592 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007146399197592778, + "loss": 1.5009, + "theoretical_loss": 4.061676389234375, + "tokens_seen": 381616128 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007146238716148446, + "loss": 1.544, + "theoretical_loss": 4.061598292525382, + "tokens_seen": 381681664 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007146078234704112, + "loss": 1.6985, + "theoretical_loss": 4.061520212978616, + "tokens_seen": 381747200 + }, + { + "epoch": 0.12, + "learning_rate": 0.000714591775325978, + "loss": 1.6046, + "theoretical_loss": 4.061442150587358, + "tokens_seen": 381812736 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007145757271815446, + "loss": 1.634, + "theoretical_loss": 4.0613641053448974, + "tokens_seen": 381878272 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007145596790371114, + "loss": 1.7136, + "theoretical_loss": 4.061286077244523, + "tokens_seen": 381943808 + }, + { + "epoch": 0.12, + "learning_rate": 0.000714543630892678, + "loss": 1.7293, + "theoretical_loss": 4.06120806627953, + "tokens_seen": 382009344 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007145275827482448, + "loss": 1.6545, + "theoretical_loss": 4.061130072443216, + "tokens_seen": 382074880 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007145115346038114, + "loss": 1.5877, + "theoretical_loss": 4.0610520957288845, + "tokens_seen": 382140416 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007144954864593782, + "loss": 1.6186, + "theoretical_loss": 4.06097413612984, + "tokens_seen": 382205952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007144794383149448, + "loss": 1.5848, + "theoretical_loss": 4.060896193639392, + "tokens_seen": 382271488 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007144633901705116, + "loss": 1.7079, + "theoretical_loss": 4.060818268250855, + "tokens_seen": 382337024 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007144473420260782, + "loss": 1.6729, + "theoretical_loss": 4.060740359957546, + "tokens_seen": 382402560 + }, + { + "epoch": 0.12, + "learning_rate": 0.000714431293881645, + "loss": 1.5894, + "theoretical_loss": 4.060662468752784, + "tokens_seen": 382468096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007144152457372116, + "loss": 1.7232, + "theoretical_loss": 4.060584594629895, + "tokens_seen": 382533632 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007143991975927784, + "loss": 1.5807, + "theoretical_loss": 4.060506737582208, + "tokens_seen": 382599168 + }, + { + "epoch": 0.12, + "learning_rate": 0.000714383149448345, + "loss": 1.6953, + "theoretical_loss": 4.060428897603054, + "tokens_seen": 382664704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007143671013039118, + "loss": 1.6501, + "theoretical_loss": 4.060351074685769, + "tokens_seen": 382730240 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007143510531594785, + "loss": 1.5184, + "theoretical_loss": 4.060273268823693, + "tokens_seen": 382795776 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007143350050150452, + "loss": 1.6422, + "theoretical_loss": 4.060195480010169, + "tokens_seen": 382861312 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007143189568706119, + "loss": 1.5414, + "theoretical_loss": 4.060117708238544, + "tokens_seen": 382926848 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007143029087261786, + "loss": 1.6887, + "theoretical_loss": 4.060039953502169, + "tokens_seen": 382992384 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007142868605817453, + "loss": 1.5507, + "theoretical_loss": 4.059962215794399, + "tokens_seen": 383057920 + }, + { + "epoch": 0.12, + "learning_rate": 0.000714270812437312, + "loss": 1.562, + "theoretical_loss": 4.059884495108591, + "tokens_seen": 383123456 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007142547642928787, + "loss": 1.6758, + "theoretical_loss": 4.059806791438108, + "tokens_seen": 383188992 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007142387161484454, + "loss": 1.5188, + "theoretical_loss": 4.059729104776316, + "tokens_seen": 383254528 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007142226680040121, + "loss": 1.6141, + "theoretical_loss": 4.059651435116583, + "tokens_seen": 383320064 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.9935578107833862, + "objective/train/avg_token_score": 0.006442194804549217, + "objective/train/docs_used": 145827, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6236742734909058, + "objective/train/original_loss": 1.6335692405700684, + "objective/train/theoretical_loss": 4.059573782452284, + "objective/train/tokens_used": 403845600, + "objective/train/unlikelihood_loss": 0.017021603882312775, + "theoretical_loss": 4.059573782452284, + "tokens_seen": 383385600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007142066198595788, + "loss": 1.5976, + "theoretical_loss": 4.059573782452284, + "tokens_seen": 383385600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007141905717151455, + "loss": 1.5791, + "theoretical_loss": 4.0594961467767945, + "tokens_seen": 383451136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007141745235707122, + "loss": 1.5634, + "theoretical_loss": 4.059418528083495, + "tokens_seen": 383516672 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007141584754262789, + "loss": 1.6433, + "theoretical_loss": 4.0593409263657705, + "tokens_seen": 383582208 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007141424272818456, + "loss": 1.6126, + "theoretical_loss": 4.059263341617009, + "tokens_seen": 383647744 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007141263791374123, + "loss": 1.4904, + "theoretical_loss": 4.059185773830602, + "tokens_seen": 383713280 + }, + { + "epoch": 0.12, + "learning_rate": 0.000714110330992979, + "loss": 1.6801, + "theoretical_loss": 4.059108222999944, + "tokens_seen": 383778816 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007140942828485457, + "loss": 1.6131, + "theoretical_loss": 4.059030689118436, + "tokens_seen": 383844352 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007140782347041124, + "loss": 1.5472, + "theoretical_loss": 4.058953172179479, + "tokens_seen": 383909888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007140621865596791, + "loss": 1.6772, + "theoretical_loss": 4.05887567217648, + "tokens_seen": 383975424 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007140461384152458, + "loss": 1.5748, + "theoretical_loss": 4.05879818910285, + "tokens_seen": 384040960 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007140300902708125, + "loss": 1.7064, + "theoretical_loss": 4.058720722952002, + "tokens_seen": 384106496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007140140421263792, + "loss": 1.6048, + "theoretical_loss": 4.058643273717355, + "tokens_seen": 384172032 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007139979939819459, + "loss": 1.543, + "theoretical_loss": 4.058565841392328, + "tokens_seen": 384237568 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007139819458375126, + "loss": 1.6559, + "theoretical_loss": 4.058488425970347, + "tokens_seen": 384303104 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007139658976930793, + "loss": 1.5241, + "theoretical_loss": 4.058411027444841, + "tokens_seen": 384368640 + }, + { + "epoch": 0.12, + "learning_rate": 0.000713949849548646, + "loss": 1.6177, + "theoretical_loss": 4.05833364580924, + "tokens_seen": 384434176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007139338014042127, + "loss": 1.606, + "theoretical_loss": 4.058256281056984, + "tokens_seen": 384499712 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007139177532597794, + "loss": 1.6213, + "theoretical_loss": 4.058178933181509, + "tokens_seen": 384565248 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007139017051153461, + "loss": 1.5875, + "theoretical_loss": 4.05810160217626, + "tokens_seen": 384630784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007138856569709128, + "loss": 1.6138, + "theoretical_loss": 4.058024288034684, + "tokens_seen": 384696320 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007138696088264795, + "loss": 1.5311, + "theoretical_loss": 4.057946990750231, + "tokens_seen": 384761856 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007138535606820462, + "loss": 1.7118, + "theoretical_loss": 4.057869710316355, + "tokens_seen": 384827392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007138375125376129, + "loss": 1.6286, + "theoretical_loss": 4.0577924467265145, + "tokens_seen": 384892928 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007138214643931796, + "loss": 1.7342, + "theoretical_loss": 4.057715199974171, + "tokens_seen": 384958464 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007138054162487463, + "loss": 1.5987, + "theoretical_loss": 4.05763797005279, + "tokens_seen": 385024000 + }, + { + "epoch": 0.12, + "learning_rate": 0.000713789368104313, + "loss": 1.6002, + "theoretical_loss": 4.05756075695584, + "tokens_seen": 385089536 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007137733199598797, + "loss": 1.6682, + "theoretical_loss": 4.0574835606767925, + "tokens_seen": 385155072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007137572718154464, + "loss": 1.6674, + "theoretical_loss": 4.057406381209126, + "tokens_seen": 385220608 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007137412236710131, + "loss": 1.6347, + "theoretical_loss": 4.057329218546318, + "tokens_seen": 385286144 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007137251755265798, + "loss": 1.6365, + "theoretical_loss": 4.0572520726818535, + "tokens_seen": 385351680 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007137091273821465, + "loss": 1.6227, + "theoretical_loss": 4.057174943609218, + "tokens_seen": 385417216 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007136930792377132, + "loss": 1.6094, + "theoretical_loss": 4.057097831321904, + "tokens_seen": 385482752 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007136770310932799, + "loss": 1.6032, + "theoretical_loss": 4.057020735813405, + "tokens_seen": 385548288 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007136609829488466, + "loss": 1.602, + "theoretical_loss": 4.056943657077219, + "tokens_seen": 385613824 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007136449348044133, + "loss": 1.5585, + "theoretical_loss": 4.056866595106847, + "tokens_seen": 385679360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007136288866599799, + "loss": 1.6715, + "theoretical_loss": 4.056789549895795, + "tokens_seen": 385744896 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007136128385155467, + "loss": 1.6028, + "theoretical_loss": 4.056712521437571, + "tokens_seen": 385810432 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007135967903711133, + "loss": 1.6038, + "theoretical_loss": 4.056635509725689, + "tokens_seen": 385875968 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007135807422266801, + "loss": 1.72, + "theoretical_loss": 4.056558514753664, + "tokens_seen": 385941504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007135646940822467, + "loss": 1.5257, + "theoretical_loss": 4.056481536515014, + "tokens_seen": 386007040 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007135486459378135, + "loss": 1.577, + "theoretical_loss": 4.056404575003266, + "tokens_seen": 386072576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007135325977933801, + "loss": 1.5742, + "theoretical_loss": 4.0563276302119435, + "tokens_seen": 386138112 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007135165496489468, + "loss": 1.5449, + "theoretical_loss": 4.056250702134578, + "tokens_seen": 386203648 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007135005015045135, + "loss": 1.6313, + "theoretical_loss": 4.0561737907647055, + "tokens_seen": 386269184 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007134844533600803, + "loss": 1.6166, + "theoretical_loss": 4.056096896095861, + "tokens_seen": 386334720 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007134684052156469, + "loss": 1.5541, + "theoretical_loss": 4.056020018121587, + "tokens_seen": 386400256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007134523570712137, + "loss": 1.5153, + "theoretical_loss": 4.055943156835428, + "tokens_seen": 386465792 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007134363089267803, + "loss": 1.5794, + "theoretical_loss": 4.055866312230933, + "tokens_seen": 386531328 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007134202607823471, + "loss": 1.6133, + "theoretical_loss": 4.055789484301653, + "tokens_seen": 386596864 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.9931545257568359, + "objective/train/avg_token_score": 0.006845511961728334, + "objective/train/docs_used": 146981, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.7292312383651733, + "objective/train/original_loss": 1.7425155639648438, + "objective/train/theoretical_loss": 4.055712673041144, + "objective/train/tokens_used": 407122400, + "objective/train/unlikelihood_loss": 0.012792146764695644, + "theoretical_loss": 4.055712673041144, + "tokens_seen": 386662400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007134042126379137, + "loss": 1.7388, + "theoretical_loss": 4.055712673041144, + "tokens_seen": 386662400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007133881644934805, + "loss": 1.6544, + "theoretical_loss": 4.055635878442967, + "tokens_seen": 386727936 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007133721163490471, + "loss": 1.5925, + "theoretical_loss": 4.055559100500682, + "tokens_seen": 386793472 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007133560682046139, + "loss": 1.6024, + "theoretical_loss": 4.055482339207858, + "tokens_seen": 386859008 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007133400200601805, + "loss": 1.4851, + "theoretical_loss": 4.055405594558062, + "tokens_seen": 386924544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007133239719157473, + "loss": 1.5731, + "theoretical_loss": 4.05532886654487, + "tokens_seen": 386990080 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007133079237713139, + "loss": 1.6785, + "theoretical_loss": 4.055252155161858, + "tokens_seen": 387055616 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007132918756268807, + "loss": 1.6995, + "theoretical_loss": 4.055175460402607, + "tokens_seen": 387121152 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007132758274824473, + "loss": 1.7001, + "theoretical_loss": 4.055098782260701, + "tokens_seen": 387186688 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007132597793380141, + "loss": 1.5394, + "theoretical_loss": 4.055022120729728, + "tokens_seen": 387252224 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007132437311935807, + "loss": 1.5432, + "theoretical_loss": 4.054945475803278, + "tokens_seen": 387317760 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007132276830491475, + "loss": 1.6931, + "theoretical_loss": 4.054868847474948, + "tokens_seen": 387383296 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007132116349047141, + "loss": 1.526, + "theoretical_loss": 4.054792235738335, + "tokens_seen": 387448832 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007131955867602809, + "loss": 1.5311, + "theoretical_loss": 4.054715640587042, + "tokens_seen": 387514368 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007131795386158475, + "loss": 1.6231, + "theoretical_loss": 4.054639062014673, + "tokens_seen": 387579904 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007131634904714143, + "loss": 1.6049, + "theoretical_loss": 4.054562500014839, + "tokens_seen": 387645440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007131474423269809, + "loss": 1.6353, + "theoretical_loss": 4.054485954581152, + "tokens_seen": 387710976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007131313941825477, + "loss": 1.7001, + "theoretical_loss": 4.054409425707227, + "tokens_seen": 387776512 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007131153460381143, + "loss": 1.6066, + "theoretical_loss": 4.054332913386686, + "tokens_seen": 387842048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007130992978936811, + "loss": 1.5648, + "theoretical_loss": 4.054256417613149, + "tokens_seen": 387907584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007130832497492477, + "loss": 1.5939, + "theoretical_loss": 4.054179938380246, + "tokens_seen": 387973120 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007130672016048145, + "loss": 1.563, + "theoretical_loss": 4.054103475681605, + "tokens_seen": 388038656 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007130511534603811, + "loss": 1.4278, + "theoretical_loss": 4.054027029510861, + "tokens_seen": 388104192 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007130351053159479, + "loss": 1.4907, + "theoretical_loss": 4.053950599861651, + "tokens_seen": 388169728 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007130190571715146, + "loss": 1.5822, + "theoretical_loss": 4.053874186727617, + "tokens_seen": 388235264 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007130030090270813, + "loss": 1.728, + "theoretical_loss": 4.053797790102402, + "tokens_seen": 388300800 + }, + { + "epoch": 0.12, + "learning_rate": 0.000712986960882648, + "loss": 1.6268, + "theoretical_loss": 4.053721409979654, + "tokens_seen": 388366336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007129709127382147, + "loss": 1.6671, + "theoretical_loss": 4.053645046353025, + "tokens_seen": 388431872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007129548645937814, + "loss": 1.4809, + "theoretical_loss": 4.05356869921617, + "tokens_seen": 388497408 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007129388164493481, + "loss": 1.592, + "theoretical_loss": 4.053492368562747, + "tokens_seen": 388562944 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007129227683049148, + "loss": 1.68, + "theoretical_loss": 4.0534160543864175, + "tokens_seen": 388628480 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007129067201604815, + "loss": 1.619, + "theoretical_loss": 4.053339756680849, + "tokens_seen": 388694016 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007128906720160482, + "loss": 1.613, + "theoretical_loss": 4.053263475439707, + "tokens_seen": 388759552 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007128746238716149, + "loss": 1.5249, + "theoretical_loss": 4.053187210656668, + "tokens_seen": 388825088 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007128585757271816, + "loss": 1.6725, + "theoretical_loss": 4.053110962325406, + "tokens_seen": 388890624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007128425275827483, + "loss": 1.4874, + "theoretical_loss": 4.053034730439601, + "tokens_seen": 388956160 + }, + { + "epoch": 0.12, + "learning_rate": 0.000712826479438315, + "loss": 1.6124, + "theoretical_loss": 4.052958514992934, + "tokens_seen": 389021696 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007128104312938817, + "loss": 1.6334, + "theoretical_loss": 4.052882315979094, + "tokens_seen": 389087232 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007127943831494484, + "loss": 1.4551, + "theoretical_loss": 4.052806133391769, + "tokens_seen": 389152768 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007127783350050151, + "loss": 1.6446, + "theoretical_loss": 4.052729967224654, + "tokens_seen": 389218304 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007127622868605818, + "loss": 1.5396, + "theoretical_loss": 4.0526538174714455, + "tokens_seen": 389283840 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007127462387161485, + "loss": 1.6458, + "theoretical_loss": 4.052577684125843, + "tokens_seen": 389349376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007127301905717152, + "loss": 1.6869, + "theoretical_loss": 4.052501567181552, + "tokens_seen": 389414912 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007127141424272819, + "loss": 1.6618, + "theoretical_loss": 4.052425466632278, + "tokens_seen": 389480448 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007126980942828486, + "loss": 1.6001, + "theoretical_loss": 4.052349382471734, + "tokens_seen": 389545984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007126820461384153, + "loss": 1.4442, + "theoretical_loss": 4.052273314693632, + "tokens_seen": 389611520 + }, + { + "epoch": 0.12, + "learning_rate": 0.000712665997993982, + "loss": 1.5922, + "theoretical_loss": 4.05219726329169, + "tokens_seen": 389677056 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007126499498495487, + "loss": 1.691, + "theoretical_loss": 4.052121228259631, + "tokens_seen": 389742592 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007126339017051154, + "loss": 1.6352, + "theoretical_loss": 4.052045209591179, + "tokens_seen": 389808128 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007126178535606821, + "loss": 1.5814, + "theoretical_loss": 4.05196920728006, + "tokens_seen": 389873664 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.9954656958580017, + "objective/train/avg_token_score": 0.004534339997917414, + "objective/train/docs_used": 148229, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.7231504917144775, + "objective/train/original_loss": 1.7318196296691895, + "objective/train/theoretical_loss": 4.051893221320009, + "objective/train/tokens_used": 410399200, + "objective/train/unlikelihood_loss": 0.009003332816064358, + "theoretical_loss": 4.051893221320009, + "tokens_seen": 389939200 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007126018054162488, + "loss": 1.699, + "theoretical_loss": 4.051893221320009, + "tokens_seen": 389939200 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007125857572718155, + "loss": 1.6286, + "theoretical_loss": 4.05181725170476, + "tokens_seen": 390004736 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007125697091273822, + "loss": 1.6225, + "theoretical_loss": 4.051741298428049, + "tokens_seen": 390070272 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007125536609829489, + "loss": 1.6067, + "theoretical_loss": 4.0516653614836216, + "tokens_seen": 390135808 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007125376128385156, + "loss": 1.5952, + "theoretical_loss": 4.05158944086522, + "tokens_seen": 390201344 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007125215646940823, + "loss": 1.5977, + "theoretical_loss": 4.0515135365665955, + "tokens_seen": 390266880 + }, + { + "epoch": 0.12, + "learning_rate": 0.000712505516549649, + "loss": 1.4709, + "theoretical_loss": 4.0514376485815, + "tokens_seen": 390332416 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007124894684052157, + "loss": 1.5799, + "theoretical_loss": 4.051361776903688, + "tokens_seen": 390397952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007124734202607824, + "loss": 1.655, + "theoretical_loss": 4.051285921526919, + "tokens_seen": 390463488 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007124573721163491, + "loss": 1.6359, + "theoretical_loss": 4.051210082444956, + "tokens_seen": 390529024 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007124413239719158, + "loss": 1.5441, + "theoretical_loss": 4.051134259651565, + "tokens_seen": 390594560 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007124252758274825, + "loss": 1.7045, + "theoretical_loss": 4.0510584531405165, + "tokens_seen": 390660096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007124092276830492, + "loss": 1.5245, + "theoretical_loss": 4.050982662905581, + "tokens_seen": 390725632 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007123931795386159, + "loss": 1.5817, + "theoretical_loss": 4.0509068889405375, + "tokens_seen": 390791168 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007123771313941826, + "loss": 1.7095, + "theoretical_loss": 4.050831131239164, + "tokens_seen": 390856704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007123610832497493, + "loss": 1.6925, + "theoretical_loss": 4.050755389795244, + "tokens_seen": 390922240 + }, + { + "epoch": 0.12, + "learning_rate": 0.000712345035105316, + "loss": 1.6583, + "theoretical_loss": 4.050679664602564, + "tokens_seen": 390987776 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007123289869608827, + "loss": 1.6517, + "theoretical_loss": 4.050603955654914, + "tokens_seen": 391053312 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007123129388164494, + "loss": 1.5881, + "theoretical_loss": 4.050528262946088, + "tokens_seen": 391118848 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007122968906720161, + "loss": 1.5867, + "theoretical_loss": 4.050452586469882, + "tokens_seen": 391184384 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007122808425275828, + "loss": 1.4048, + "theoretical_loss": 4.050376926220098, + "tokens_seen": 391249920 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007122647943831495, + "loss": 1.5109, + "theoretical_loss": 4.050301282190537, + "tokens_seen": 391315456 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007122487462387162, + "loss": 1.6697, + "theoretical_loss": 4.050225654375008, + "tokens_seen": 391380992 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007122326980942828, + "loss": 1.6954, + "theoretical_loss": 4.050150042767321, + "tokens_seen": 391446528 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007122166499498496, + "loss": 1.5494, + "theoretical_loss": 4.050074447361289, + "tokens_seen": 391512064 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007122006018054162, + "loss": 1.5717, + "theoretical_loss": 4.04999886815073, + "tokens_seen": 391577600 + }, + { + "epoch": 0.12, + "learning_rate": 0.000712184553660983, + "loss": 1.587, + "theoretical_loss": 4.0499233051294645, + "tokens_seen": 391643136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007121685055165496, + "loss": 1.637, + "theoretical_loss": 4.049847758291317, + "tokens_seen": 391708672 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007121524573721164, + "loss": 1.6044, + "theoretical_loss": 4.049772227630114, + "tokens_seen": 391774208 + }, + { + "epoch": 0.12, + "learning_rate": 0.000712136409227683, + "loss": 1.6029, + "theoretical_loss": 4.049696713139687, + "tokens_seen": 391839744 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007121203610832498, + "loss": 1.6209, + "theoretical_loss": 4.049621214813869, + "tokens_seen": 391905280 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007121043129388164, + "loss": 1.5717, + "theoretical_loss": 4.0495457326465, + "tokens_seen": 391970816 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007120882647943832, + "loss": 1.4655, + "theoretical_loss": 4.049470266631419, + "tokens_seen": 392036352 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007120722166499498, + "loss": 1.6702, + "theoretical_loss": 4.04939481676247, + "tokens_seen": 392101888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007120561685055166, + "loss": 1.6768, + "theoretical_loss": 4.0493193830335015, + "tokens_seen": 392167424 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007120401203610832, + "loss": 1.5595, + "theoretical_loss": 4.0492439654383645, + "tokens_seen": 392232960 + }, + { + "epoch": 0.12, + "learning_rate": 0.00071202407221665, + "loss": 1.6842, + "theoretical_loss": 4.049168563970914, + "tokens_seen": 392298496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007120080240722166, + "loss": 1.624, + "theoretical_loss": 4.049093178625007, + "tokens_seen": 392364032 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007119919759277834, + "loss": 1.5573, + "theoretical_loss": 4.049017809394506, + "tokens_seen": 392429568 + }, + { + "epoch": 0.12, + "learning_rate": 0.00071197592778335, + "loss": 1.4888, + "theoretical_loss": 4.048942456273274, + "tokens_seen": 392495104 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007119598796389168, + "loss": 1.5349, + "theoretical_loss": 4.048867119255178, + "tokens_seen": 392560640 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007119438314944834, + "loss": 1.6215, + "theoretical_loss": 4.048791798334092, + "tokens_seen": 392626176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007119277833500502, + "loss": 1.6464, + "theoretical_loss": 4.048716493503888, + "tokens_seen": 392691712 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007119117352056168, + "loss": 1.6963, + "theoretical_loss": 4.048641204758446, + "tokens_seen": 392757248 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007118956870611836, + "loss": 1.6238, + "theoretical_loss": 4.048565932091646, + "tokens_seen": 392822784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007118796389167502, + "loss": 1.6621, + "theoretical_loss": 4.048490675497373, + "tokens_seen": 392888320 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711863590772317, + "loss": 1.6267, + "theoretical_loss": 4.048415434969516, + "tokens_seen": 392953856 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007118475426278836, + "loss": 1.5856, + "theoretical_loss": 4.048340210501965, + "tokens_seen": 393019392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007118314944834504, + "loss": 1.6266, + "theoretical_loss": 4.048265002088614, + "tokens_seen": 393084928 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711815446339017, + "loss": 1.541, + "theoretical_loss": 4.048189809723364, + "tokens_seen": 393150464 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.9909151196479797, + "objective/train/avg_token_score": 0.009084894321858883, + "objective/train/docs_used": 149327, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.740559697151184, + "objective/train/original_loss": 1.7566354274749756, + "objective/train/theoretical_loss": 4.048114633400113, + "objective/train/tokens_used": 413676000, + "objective/train/unlikelihood_loss": 0.019272958859801292, + "theoretical_loss": 4.048114633400113, + "tokens_seen": 393216000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007117993981945838, + "loss": 1.6217, + "theoretical_loss": 4.048114633400113, + "tokens_seen": 393216000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007117833500501504, + "loss": 1.6428, + "theoretical_loss": 4.048039473112768, + "tokens_seen": 393281536 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007117673019057172, + "loss": 1.5653, + "theoretical_loss": 4.047964328855237, + "tokens_seen": 393347072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007117512537612838, + "loss": 1.6169, + "theoretical_loss": 4.047889200621428, + "tokens_seen": 393412608 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007117352056168506, + "loss": 1.6825, + "theoretical_loss": 4.047814088405261, + "tokens_seen": 393478144 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007117191574724172, + "loss": 1.7007, + "theoretical_loss": 4.0477389922006495, + "tokens_seen": 393543680 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711703109327984, + "loss": 1.6268, + "theoretical_loss": 4.047663912001518, + "tokens_seen": 393609216 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007116870611835506, + "loss": 1.6493, + "theoretical_loss": 4.047588847801788, + "tokens_seen": 393674752 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007116710130391174, + "loss": 1.6588, + "theoretical_loss": 4.04751379959539, + "tokens_seen": 393740288 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711654964894684, + "loss": 1.6213, + "theoretical_loss": 4.047438767376255, + "tokens_seen": 393805824 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007116389167502508, + "loss": 1.6704, + "theoretical_loss": 4.047363751138316, + "tokens_seen": 393871360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007116228686058175, + "loss": 1.5617, + "theoretical_loss": 4.047288750875512, + "tokens_seen": 393936896 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007116068204613842, + "loss": 1.6352, + "theoretical_loss": 4.047213766581785, + "tokens_seen": 394002432 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007115907723169509, + "loss": 1.4617, + "theoretical_loss": 4.047138798251078, + "tokens_seen": 394067968 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007115747241725176, + "loss": 1.6222, + "theoretical_loss": 4.047063845877339, + "tokens_seen": 394133504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007115586760280843, + "loss": 1.5945, + "theoretical_loss": 4.04698890945452, + "tokens_seen": 394199040 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711542627883651, + "loss": 1.6854, + "theoretical_loss": 4.046913988976575, + "tokens_seen": 394264576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007115265797392177, + "loss": 1.6127, + "theoretical_loss": 4.046839084437463, + "tokens_seen": 394330112 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007115105315947844, + "loss": 1.6542, + "theoretical_loss": 4.046764195831142, + "tokens_seen": 394395648 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007114944834503511, + "loss": 1.6937, + "theoretical_loss": 4.046689323151578, + "tokens_seen": 394461184 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007114784353059178, + "loss": 1.5933, + "theoretical_loss": 4.0466144663927395, + "tokens_seen": 394526720 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007114623871614845, + "loss": 1.7063, + "theoretical_loss": 4.046539625548595, + "tokens_seen": 394592256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007114463390170512, + "loss": 1.7055, + "theoretical_loss": 4.046464800613121, + "tokens_seen": 394657792 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007114302908726179, + "loss": 1.5178, + "theoretical_loss": 4.046389991580294, + "tokens_seen": 394723328 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007114142427281846, + "loss": 1.5246, + "theoretical_loss": 4.046315198444094, + "tokens_seen": 394788864 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007113981945837513, + "loss": 1.5767, + "theoretical_loss": 4.046240421198506, + "tokens_seen": 394854400 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711382146439318, + "loss": 1.6688, + "theoretical_loss": 4.046165659837517, + "tokens_seen": 394919936 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007113660982948847, + "loss": 1.6489, + "theoretical_loss": 4.046090914355117, + "tokens_seen": 394985472 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007113500501504514, + "loss": 1.6282, + "theoretical_loss": 4.046016184745302, + "tokens_seen": 395051008 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007113340020060181, + "loss": 1.6408, + "theoretical_loss": 4.045941471002066, + "tokens_seen": 395116544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007113179538615848, + "loss": 1.6631, + "theoretical_loss": 4.0458667731194105, + "tokens_seen": 395182080 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007113019057171515, + "loss": 1.6244, + "theoretical_loss": 4.045792091091339, + "tokens_seen": 395247616 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007112858575727182, + "loss": 1.6597, + "theoretical_loss": 4.04571742491186, + "tokens_seen": 395313152 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007112698094282849, + "loss": 1.5594, + "theoretical_loss": 4.045642774574981, + "tokens_seen": 395378688 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007112537612838516, + "loss": 1.6272, + "theoretical_loss": 4.045568140074717, + "tokens_seen": 395444224 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007112377131394183, + "loss": 1.5277, + "theoretical_loss": 4.045493521405085, + "tokens_seen": 395509760 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711221664994985, + "loss": 1.711, + "theoretical_loss": 4.045418918560104, + "tokens_seen": 395575296 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007112056168505517, + "loss": 1.5441, + "theoretical_loss": 4.045344331533796, + "tokens_seen": 395640832 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007111895687061184, + "loss": 1.6926, + "theoretical_loss": 4.04526976032019, + "tokens_seen": 395706368 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007111735205616851, + "loss": 1.6658, + "theoretical_loss": 4.0451952049133135, + "tokens_seen": 395771904 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007111574724172518, + "loss": 1.6812, + "theoretical_loss": 4.0451206653072, + "tokens_seen": 395837440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007111414242728185, + "loss": 1.5609, + "theoretical_loss": 4.045046141495887, + "tokens_seen": 395902976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007111253761283852, + "loss": 1.6222, + "theoretical_loss": 4.044971633473412, + "tokens_seen": 395968512 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711109327983952, + "loss": 1.539, + "theoretical_loss": 4.044897141233818, + "tokens_seen": 396034048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007110932798395186, + "loss": 1.6172, + "theoretical_loss": 4.044822664771151, + "tokens_seen": 396099584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007110772316950854, + "loss": 1.5662, + "theoretical_loss": 4.044748204079459, + "tokens_seen": 396165120 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711061183550652, + "loss": 1.6653, + "theoretical_loss": 4.044673759152797, + "tokens_seen": 396230656 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007110451354062188, + "loss": 1.7095, + "theoretical_loss": 4.0445993299852185, + "tokens_seen": 396296192 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007110290872617854, + "loss": 1.7101, + "theoretical_loss": 4.044524916570782, + "tokens_seen": 396361728 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007110130391173522, + "loss": 1.7172, + "theoretical_loss": 4.04445051890355, + "tokens_seen": 396427264 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.9924962520599365, + "objective/train/avg_token_score": 0.0075037311762571335, + "objective/train/docs_used": 150450, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5822343826293945, + "objective/train/original_loss": 1.595245122909546, + "objective/train/theoretical_loss": 4.044376136977588, + "objective/train/tokens_used": 416952800, + "objective/train/unlikelihood_loss": 0.016123725101351738, + "theoretical_loss": 4.044376136977588, + "tokens_seen": 396492800 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007109969909729188, + "loss": 1.5626, + "theoretical_loss": 4.044376136977588, + "tokens_seen": 396492800 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007109809428284855, + "loss": 1.6064, + "theoretical_loss": 4.044301770786964, + "tokens_seen": 396558336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007109648946840522, + "loss": 1.6432, + "theoretical_loss": 4.044227420325749, + "tokens_seen": 396623872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007109488465396189, + "loss": 1.7383, + "theoretical_loss": 4.044153085588018, + "tokens_seen": 396689408 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007109327983951856, + "loss": 1.5411, + "theoretical_loss": 4.044078766567851, + "tokens_seen": 396754944 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007109167502507523, + "loss": 1.668, + "theoretical_loss": 4.044004463259327, + "tokens_seen": 396820480 + }, + { + "epoch": 0.12, + "learning_rate": 0.000710900702106319, + "loss": 1.5257, + "theoretical_loss": 4.043930175656531, + "tokens_seen": 396886016 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007108846539618857, + "loss": 1.4892, + "theoretical_loss": 4.043855903753551, + "tokens_seen": 396951552 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007108686058174524, + "loss": 1.4884, + "theoretical_loss": 4.043781647544477, + "tokens_seen": 397017088 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007108525576730191, + "loss": 1.6805, + "theoretical_loss": 4.043707407023404, + "tokens_seen": 397082624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007108365095285857, + "loss": 1.6688, + "theoretical_loss": 4.043633182184428, + "tokens_seen": 397148160 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007108204613841525, + "loss": 1.554, + "theoretical_loss": 4.043558973021652, + "tokens_seen": 397213696 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007108044132397191, + "loss": 1.578, + "theoretical_loss": 4.043484779529176, + "tokens_seen": 397279232 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007107883650952859, + "loss": 1.7107, + "theoretical_loss": 4.04341060170111, + "tokens_seen": 397344768 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007107723169508525, + "loss": 1.7169, + "theoretical_loss": 4.043336439531561, + "tokens_seen": 397410304 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007107562688064193, + "loss": 1.6759, + "theoretical_loss": 4.043262293014646, + "tokens_seen": 397475840 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007107402206619859, + "loss": 1.68, + "theoretical_loss": 4.043188162144478, + "tokens_seen": 397541376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007107241725175527, + "loss": 1.689, + "theoretical_loss": 4.0431140469151785, + "tokens_seen": 397606912 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007107081243731193, + "loss": 1.6383, + "theoretical_loss": 4.043039947320869, + "tokens_seen": 397672448 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007106920762286861, + "loss": 1.7483, + "theoretical_loss": 4.0429658633556755, + "tokens_seen": 397737984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007106760280842527, + "loss": 1.6873, + "theoretical_loss": 4.042891795013729, + "tokens_seen": 397803520 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007106599799398195, + "loss": 1.6716, + "theoretical_loss": 4.042817742289159, + "tokens_seen": 397869056 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007106439317953861, + "loss": 1.6573, + "theoretical_loss": 4.042743705176102, + "tokens_seen": 397934592 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007106278836509529, + "loss": 1.604, + "theoretical_loss": 4.042669683668698, + "tokens_seen": 398000128 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007106118355065195, + "loss": 1.653, + "theoretical_loss": 4.042595677761087, + "tokens_seen": 398065664 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007105957873620863, + "loss": 1.5016, + "theoretical_loss": 4.0425216874474135, + "tokens_seen": 398131200 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007105797392176529, + "loss": 1.6169, + "theoretical_loss": 4.0424477127218275, + "tokens_seen": 398196736 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007105636910732197, + "loss": 1.7471, + "theoretical_loss": 4.042373753578479, + "tokens_seen": 398262272 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007105476429287863, + "loss": 1.7023, + "theoretical_loss": 4.042299810011523, + "tokens_seen": 398327808 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007105315947843531, + "loss": 1.7446, + "theoretical_loss": 4.042225882015115, + "tokens_seen": 398393344 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007105155466399197, + "loss": 1.6147, + "theoretical_loss": 4.0421519695834185, + "tokens_seen": 398458880 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007104994984954865, + "loss": 1.4901, + "theoretical_loss": 4.042078072710597, + "tokens_seen": 398524416 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007104834503510531, + "loss": 1.5313, + "theoretical_loss": 4.042004191390816, + "tokens_seen": 398589952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007104674022066199, + "loss": 1.6304, + "theoretical_loss": 4.041930325618246, + "tokens_seen": 398655488 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007104513540621865, + "loss": 1.5716, + "theoretical_loss": 4.041856475387061, + "tokens_seen": 398721024 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007104353059177533, + "loss": 1.6109, + "theoretical_loss": 4.041782640691437, + "tokens_seen": 398786560 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007104192577733199, + "loss": 1.6585, + "theoretical_loss": 4.041708821525554, + "tokens_seen": 398852096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007104032096288867, + "loss": 1.6485, + "theoretical_loss": 4.0416350178835945, + "tokens_seen": 398917632 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007103871614844533, + "loss": 1.5352, + "theoretical_loss": 4.041561229759743, + "tokens_seen": 398983168 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007103711133400201, + "loss": 1.6673, + "theoretical_loss": 4.0414874571481905, + "tokens_seen": 399048704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007103550651955867, + "loss": 1.5921, + "theoretical_loss": 4.041413700043128, + "tokens_seen": 399114240 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007103390170511535, + "loss": 1.6667, + "theoretical_loss": 4.041339958438751, + "tokens_seen": 399179776 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007103229689067201, + "loss": 1.7106, + "theoretical_loss": 4.041266232329257, + "tokens_seen": 399245312 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007103069207622869, + "loss": 1.703, + "theoretical_loss": 4.041192521708849, + "tokens_seen": 399310848 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007102908726178535, + "loss": 1.6191, + "theoretical_loss": 4.041118826571731, + "tokens_seen": 399376384 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007102748244734203, + "loss": 1.613, + "theoretical_loss": 4.041045146912111, + "tokens_seen": 399441920 + }, + { + "epoch": 0.12, + "learning_rate": 0.000710258776328987, + "loss": 1.5883, + "theoretical_loss": 4.040971482724199, + "tokens_seen": 399507456 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007102427281845537, + "loss": 1.5265, + "theoretical_loss": 4.040897834002209, + "tokens_seen": 399572992 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007102266800401204, + "loss": 1.6797, + "theoretical_loss": 4.040824200740358, + "tokens_seen": 399638528 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007102106318956871, + "loss": 1.592, + "theoretical_loss": 4.040750582932868, + "tokens_seen": 399704064 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.9920065999031067, + "objective/train/avg_token_score": 0.007993429899215698, + "objective/train/docs_used": 151634, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6164628267288208, + "objective/train/original_loss": 1.6298842430114746, + "objective/train/theoretical_loss": 4.04067698057396, + "objective/train/tokens_used": 420229600, + "objective/train/unlikelihood_loss": 0.01675393432378769, + "theoretical_loss": 4.04067698057396, + "tokens_seen": 399769600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007101945837512538, + "loss": 1.5224, + "theoretical_loss": 4.04067698057396, + "tokens_seen": 399769600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007101785356068205, + "loss": 1.5099, + "theoretical_loss": 4.040603393657862, + "tokens_seen": 399835136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007101624874623872, + "loss": 1.5893, + "theoretical_loss": 4.040529822178802, + "tokens_seen": 399900672 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007101464393179539, + "loss": 1.5522, + "theoretical_loss": 4.040456266131013, + "tokens_seen": 399966208 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007101303911735206, + "loss": 1.5872, + "theoretical_loss": 4.0403827255087315, + "tokens_seen": 400031744 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007101143430290873, + "loss": 1.5333, + "theoretical_loss": 4.040309200306195, + "tokens_seen": 400097280 + }, + { + "epoch": 0.12, + "learning_rate": 0.000710098294884654, + "loss": 1.6253, + "theoretical_loss": 4.0402356905176475, + "tokens_seen": 400162816 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007100822467402207, + "loss": 1.6762, + "theoretical_loss": 4.040162196137332, + "tokens_seen": 400228352 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007100661985957874, + "loss": 1.6406, + "theoretical_loss": 4.040088717159497, + "tokens_seen": 400293888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007100501504513541, + "loss": 1.5293, + "theoretical_loss": 4.040015253578394, + "tokens_seen": 400359424 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007100341023069208, + "loss": 1.5801, + "theoretical_loss": 4.039941805388277, + "tokens_seen": 400424960 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007100180541624875, + "loss": 1.5682, + "theoretical_loss": 4.0398683725834035, + "tokens_seen": 400490496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007100020060180542, + "loss": 1.5, + "theoretical_loss": 4.039794955158033, + "tokens_seen": 400556032 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007099859578736209, + "loss": 1.577, + "theoretical_loss": 4.039721553106431, + "tokens_seen": 400621568 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007099699097291876, + "loss": 1.7116, + "theoretical_loss": 4.039648166422863, + "tokens_seen": 400687104 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007099538615847543, + "loss": 1.6408, + "theoretical_loss": 4.039574795101597, + "tokens_seen": 400752640 + }, + { + "epoch": 0.12, + "learning_rate": 0.000709937813440321, + "loss": 1.6463, + "theoretical_loss": 4.039501439136908, + "tokens_seen": 400818176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007099217652958877, + "loss": 1.6798, + "theoretical_loss": 4.0394280985230715, + "tokens_seen": 400883712 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007099057171514545, + "loss": 1.6392, + "theoretical_loss": 4.039354773254365, + "tokens_seen": 400949248 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007098896690070211, + "loss": 1.5661, + "theoretical_loss": 4.0392814633250715, + "tokens_seen": 401014784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007098736208625879, + "loss": 1.6273, + "theoretical_loss": 4.039208168729475, + "tokens_seen": 401080320 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007098575727181545, + "loss": 1.6392, + "theoretical_loss": 4.039134889461866, + "tokens_seen": 401145856 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007098415245737213, + "loss": 1.458, + "theoretical_loss": 4.039061625516532, + "tokens_seen": 401211392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007098254764292879, + "loss": 1.633, + "theoretical_loss": 4.038988376887771, + "tokens_seen": 401276928 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007098094282848547, + "loss": 1.6036, + "theoretical_loss": 4.038915143569876, + "tokens_seen": 401342464 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007097933801404213, + "loss": 1.5687, + "theoretical_loss": 4.038841925557151, + "tokens_seen": 401408000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007097773319959881, + "loss": 1.5607, + "theoretical_loss": 4.038768722843898, + "tokens_seen": 401473536 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007097612838515547, + "loss": 1.5858, + "theoretical_loss": 4.038695535424422, + "tokens_seen": 401539072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007097452357071215, + "loss": 1.6248, + "theoretical_loss": 4.038622363293034, + "tokens_seen": 401604608 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007097291875626881, + "loss": 1.5369, + "theoretical_loss": 4.0385492064440465, + "tokens_seen": 401670144 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007097131394182549, + "loss": 1.5831, + "theoretical_loss": 4.038476064871775, + "tokens_seen": 401735680 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007096970912738215, + "loss": 1.6292, + "theoretical_loss": 4.0384029385705364, + "tokens_seen": 401801216 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007096810431293883, + "loss": 1.4244, + "theoretical_loss": 4.038329827534654, + "tokens_seen": 401866752 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007096649949849549, + "loss": 1.659, + "theoretical_loss": 4.038256731758453, + "tokens_seen": 401932288 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007096489468405217, + "loss": 1.547, + "theoretical_loss": 4.038183651236259, + "tokens_seen": 401997824 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007096328986960883, + "loss": 1.6632, + "theoretical_loss": 4.038110585962404, + "tokens_seen": 402063360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007096168505516551, + "loss": 1.547, + "theoretical_loss": 4.038037535931221, + "tokens_seen": 402128896 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007096008024072217, + "loss": 1.5907, + "theoretical_loss": 4.037964501137048, + "tokens_seen": 402194432 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007095847542627884, + "loss": 1.5477, + "theoretical_loss": 4.037891481574224, + "tokens_seen": 402259968 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007095687061183551, + "loss": 1.7473, + "theoretical_loss": 4.037818477237091, + "tokens_seen": 402325504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007095526579739218, + "loss": 1.575, + "theoretical_loss": 4.037745488119996, + "tokens_seen": 402391040 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007095366098294885, + "loss": 1.5751, + "theoretical_loss": 4.037672514217288, + "tokens_seen": 402456576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007095205616850552, + "loss": 1.4662, + "theoretical_loss": 4.037599555523318, + "tokens_seen": 402522112 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007095045135406219, + "loss": 1.5918, + "theoretical_loss": 4.037526612032441, + "tokens_seen": 402587648 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007094884653961886, + "loss": 1.5922, + "theoretical_loss": 4.037453683739015, + "tokens_seen": 402653184 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007094724172517553, + "loss": 1.6532, + "theoretical_loss": 4.037380770637402, + "tokens_seen": 402718720 + }, + { + "epoch": 0.12, + "learning_rate": 0.000709456369107322, + "loss": 1.5808, + "theoretical_loss": 4.037307872721964, + "tokens_seen": 402784256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007094403209628886, + "loss": 1.6381, + "theoretical_loss": 4.037234989987069, + "tokens_seen": 402849792 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007094242728184554, + "loss": 1.5572, + "theoretical_loss": 4.0371621224270875, + "tokens_seen": 402915328 + }, + { + "epoch": 0.12, + "learning_rate": 0.000709408224674022, + "loss": 1.6832, + "theoretical_loss": 4.037089270036391, + "tokens_seen": 402980864 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.988459050655365, + "objective/train/avg_token_score": 0.011540940031409264, + "objective/train/docs_used": 152845, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5413872003555298, + "objective/train/original_loss": 1.560990810394287, + "objective/train/theoretical_loss": 4.037016432809357, + "objective/train/tokens_used": 423506400, + "objective/train/unlikelihood_loss": 0.0252687931060791, + "theoretical_loss": 4.037016432809357, + "tokens_seen": 403046400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007093921765295888, + "loss": 1.6188, + "theoretical_loss": 4.037016432809357, + "tokens_seen": 403046400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007093761283851554, + "loss": 1.6814, + "theoretical_loss": 4.036943610740362, + "tokens_seen": 403111936 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007093600802407222, + "loss": 1.5976, + "theoretical_loss": 4.036870803823791, + "tokens_seen": 403177472 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007093440320962888, + "loss": 1.6183, + "theoretical_loss": 4.036798012054027, + "tokens_seen": 403243008 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007093279839518556, + "loss": 1.5425, + "theoretical_loss": 4.036725235425458, + "tokens_seen": 403308544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007093119358074222, + "loss": 1.5784, + "theoretical_loss": 4.0366524739324765, + "tokens_seen": 403374080 + }, + { + "epoch": 0.12, + "learning_rate": 0.000709295887662989, + "loss": 1.6427, + "theoretical_loss": 4.036579727569475, + "tokens_seen": 403439616 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007092798395185556, + "loss": 1.6415, + "theoretical_loss": 4.036506996330849, + "tokens_seen": 403505152 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007092637913741224, + "loss": 1.5989, + "theoretical_loss": 4.036434280211001, + "tokens_seen": 403570688 + }, + { + "epoch": 0.12, + "learning_rate": 0.000709247743229689, + "loss": 1.4413, + "theoretical_loss": 4.036361579204334, + "tokens_seen": 403636224 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007092316950852558, + "loss": 1.5787, + "theoretical_loss": 4.036288893305251, + "tokens_seen": 403701760 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007092156469408224, + "loss": 1.6055, + "theoretical_loss": 4.036216222508163, + "tokens_seen": 403767296 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007091995987963892, + "loss": 1.6231, + "theoretical_loss": 4.036143566807481, + "tokens_seen": 403832832 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007091835506519558, + "loss": 1.642, + "theoretical_loss": 4.03607092619762, + "tokens_seen": 403898368 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007091675025075226, + "loss": 1.6767, + "theoretical_loss": 4.035998300672999, + "tokens_seen": 403963904 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007091514543630892, + "loss": 1.5159, + "theoretical_loss": 4.035925690228035, + "tokens_seen": 404029440 + }, + { + "epoch": 0.12, + "learning_rate": 0.000709135406218656, + "loss": 1.5933, + "theoretical_loss": 4.035853094857156, + "tokens_seen": 404094976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007091193580742226, + "loss": 1.5571, + "theoretical_loss": 4.035780514554787, + "tokens_seen": 404160512 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007091033099297894, + "loss": 1.6459, + "theoretical_loss": 4.0357079493153565, + "tokens_seen": 404226048 + }, + { + "epoch": 0.12, + "learning_rate": 0.000709087261785356, + "loss": 1.6038, + "theoretical_loss": 4.035635399133298, + "tokens_seen": 404291584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007090712136409228, + "loss": 1.5922, + "theoretical_loss": 4.035562864003047, + "tokens_seen": 404357120 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007090551654964894, + "loss": 1.6161, + "theoretical_loss": 4.035490343919042, + "tokens_seen": 404422656 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007090391173520562, + "loss": 1.5483, + "theoretical_loss": 4.035417838875725, + "tokens_seen": 404488192 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007090230692076228, + "loss": 1.6193, + "theoretical_loss": 4.035345348867539, + "tokens_seen": 404553728 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007090070210631896, + "loss": 1.586, + "theoretical_loss": 4.035272873888932, + "tokens_seen": 404619264 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007089909729187562, + "loss": 1.7183, + "theoretical_loss": 4.035200413934355, + "tokens_seen": 404684800 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708974924774323, + "loss": 1.632, + "theoretical_loss": 4.03512796899826, + "tokens_seen": 404750336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007089588766298896, + "loss": 1.4827, + "theoretical_loss": 4.035055539075104, + "tokens_seen": 404815872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007089428284854564, + "loss": 1.5478, + "theoretical_loss": 4.034983124159346, + "tokens_seen": 404881408 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708926780341023, + "loss": 1.588, + "theoretical_loss": 4.0349107242454485, + "tokens_seen": 404946944 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007089107321965898, + "loss": 1.6253, + "theoretical_loss": 4.034838339327876, + "tokens_seen": 405012480 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007088946840521565, + "loss": 1.5804, + "theoretical_loss": 4.034765969401096, + "tokens_seen": 405078016 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007088786359077232, + "loss": 1.6041, + "theoretical_loss": 4.03469361445958, + "tokens_seen": 405143552 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007088625877632899, + "loss": 1.5496, + "theoretical_loss": 4.034621274497802, + "tokens_seen": 405209088 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007088465396188566, + "loss": 1.634, + "theoretical_loss": 4.0345489495102385, + "tokens_seen": 405274624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007088304914744233, + "loss": 1.6995, + "theoretical_loss": 4.034476639491368, + "tokens_seen": 405340160 + }, + { + "epoch": 0.12, + "learning_rate": 0.00070881444332999, + "loss": 1.6454, + "theoretical_loss": 4.034404344435676, + "tokens_seen": 405405696 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007087983951855567, + "loss": 1.6351, + "theoretical_loss": 4.0343320643376455, + "tokens_seen": 405471232 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007087823470411234, + "loss": 1.8406, + "theoretical_loss": 4.034259799191765, + "tokens_seen": 405536768 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007087662988966901, + "loss": 1.5643, + "theoretical_loss": 4.034187548992527, + "tokens_seen": 405602304 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007087502507522568, + "loss": 1.6022, + "theoretical_loss": 4.034115313734426, + "tokens_seen": 405667840 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007087342026078235, + "loss": 1.6371, + "theoretical_loss": 4.034043093411958, + "tokens_seen": 405733376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007087181544633902, + "loss": 1.5621, + "theoretical_loss": 4.033970888019624, + "tokens_seen": 405798912 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708702106318957, + "loss": 1.5323, + "theoretical_loss": 4.033898697551926, + "tokens_seen": 405864448 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007086860581745236, + "loss": 1.529, + "theoretical_loss": 4.0338265220033716, + "tokens_seen": 405929984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007086700100300904, + "loss": 1.7033, + "theoretical_loss": 4.0337543613684685, + "tokens_seen": 405995520 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708653961885657, + "loss": 1.5238, + "theoretical_loss": 4.033682215641729, + "tokens_seen": 406061056 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007086379137412238, + "loss": 1.4903, + "theoretical_loss": 4.033610084817667, + "tokens_seen": 406126592 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007086218655967904, + "loss": 1.6296, + "theoretical_loss": 4.0335379688908, + "tokens_seen": 406192128 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007086058174523572, + "loss": 1.631, + "theoretical_loss": 4.03346586785565, + "tokens_seen": 406257664 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.9958768486976624, + "objective/train/avg_token_score": 0.004123104270547628, + "objective/train/docs_used": 154148, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5073281526565552, + "objective/train/original_loss": 1.5138828754425049, + "objective/train/theoretical_loss": 4.03339378170674, + "objective/train/tokens_used": 426783200, + "objective/train/unlikelihood_loss": 0.010951988399028778, + "theoretical_loss": 4.03339378170674, + "tokens_seen": 406323200 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007085897693079238, + "loss": 1.5305, + "theoretical_loss": 4.03339378170674, + "tokens_seen": 406323200 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007085737211634906, + "loss": 1.5019, + "theoretical_loss": 4.033321710438594, + "tokens_seen": 406388736 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007085576730190572, + "loss": 1.5035, + "theoretical_loss": 4.033249654045743, + "tokens_seen": 406454272 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708541624874624, + "loss": 1.6004, + "theoretical_loss": 4.033177612522721, + "tokens_seen": 406519808 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007085255767301906, + "loss": 1.3919, + "theoretical_loss": 4.0331055858640585, + "tokens_seen": 406585344 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007085095285857574, + "loss": 1.6034, + "theoretical_loss": 4.033033574064297, + "tokens_seen": 406650880 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708493480441324, + "loss": 1.6122, + "theoretical_loss": 4.032961577117976, + "tokens_seen": 406716416 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007084774322968908, + "loss": 1.6314, + "theoretical_loss": 4.032889595019638, + "tokens_seen": 406781952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007084613841524574, + "loss": 1.7072, + "theoretical_loss": 4.032817627763832, + "tokens_seen": 406847488 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007084453360080242, + "loss": 1.6643, + "theoretical_loss": 4.0327456753451045, + "tokens_seen": 406913024 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007084292878635908, + "loss": 1.6639, + "theoretical_loss": 4.03267373775801, + "tokens_seen": 406978560 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007084132397191576, + "loss": 1.5705, + "theoretical_loss": 4.032601814997102, + "tokens_seen": 407044096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007083971915747242, + "loss": 1.5946, + "theoretical_loss": 4.03252990705694, + "tokens_seen": 407109632 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708381143430291, + "loss": 1.6522, + "theoretical_loss": 4.032458013932084, + "tokens_seen": 407175168 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007083650952858576, + "loss": 1.5499, + "theoretical_loss": 4.032386135617098, + "tokens_seen": 407240704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007083490471414244, + "loss": 1.7407, + "theoretical_loss": 4.032314272106548, + "tokens_seen": 407306240 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708332998996991, + "loss": 1.5964, + "theoretical_loss": 4.0322424233950045, + "tokens_seen": 407371776 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007083169508525578, + "loss": 1.6307, + "theoretical_loss": 4.032170589477039, + "tokens_seen": 407437312 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007083009027081244, + "loss": 1.6903, + "theoretical_loss": 4.032098770347228, + "tokens_seen": 407502848 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007082848545636912, + "loss": 1.5313, + "theoretical_loss": 4.0320269660001475, + "tokens_seen": 407568384 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007082688064192578, + "loss": 1.6796, + "theoretical_loss": 4.0319551764303805, + "tokens_seen": 407633920 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007082527582748246, + "loss": 1.5914, + "theoretical_loss": 4.03188340163251, + "tokens_seen": 407699456 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007082367101303912, + "loss": 1.6907, + "theoretical_loss": 4.031811641601122, + "tokens_seen": 407764992 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708220661985958, + "loss": 1.4356, + "theoretical_loss": 4.031739896330807, + "tokens_seen": 407830528 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007082046138415246, + "loss": 1.6209, + "theoretical_loss": 4.031668165816156, + "tokens_seen": 407896064 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007081885656970914, + "loss": 1.6475, + "theoretical_loss": 4.031596450051766, + "tokens_seen": 407961600 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708172517552658, + "loss": 1.5197, + "theoretical_loss": 4.031524749032234, + "tokens_seen": 408027136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007081564694082247, + "loss": 1.6799, + "theoretical_loss": 4.031453062752161, + "tokens_seen": 408092672 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007081404212637914, + "loss": 1.4967, + "theoretical_loss": 4.031381391206151, + "tokens_seen": 408158208 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007081243731193581, + "loss": 1.5063, + "theoretical_loss": 4.03130973438881, + "tokens_seen": 408223744 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007081083249749248, + "loss": 1.6524, + "theoretical_loss": 4.031238092294748, + "tokens_seen": 408289280 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007080922768304915, + "loss": 1.729, + "theoretical_loss": 4.031166464918576, + "tokens_seen": 408354816 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007080762286860582, + "loss": 1.5929, + "theoretical_loss": 4.0310948522549115, + "tokens_seen": 408420352 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007080601805416249, + "loss": 1.6902, + "theoretical_loss": 4.031023254298371, + "tokens_seen": 408485888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007080441323971915, + "loss": 1.5017, + "theoretical_loss": 4.030951671043574, + "tokens_seen": 408551424 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007080280842527583, + "loss": 1.6468, + "theoretical_loss": 4.030880102485147, + "tokens_seen": 408616960 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007080120361083249, + "loss": 1.5742, + "theoretical_loss": 4.030808548617714, + "tokens_seen": 408682496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007079959879638917, + "loss": 1.5427, + "theoretical_loss": 4.030737009435906, + "tokens_seen": 408748032 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007079799398194583, + "loss": 1.5722, + "theoretical_loss": 4.030665484934353, + "tokens_seen": 408813568 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007079638916750251, + "loss": 1.6218, + "theoretical_loss": 4.030593975107692, + "tokens_seen": 408879104 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007079478435305917, + "loss": 1.522, + "theoretical_loss": 4.03052247995056, + "tokens_seen": 408944640 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007079317953861585, + "loss": 1.5639, + "theoretical_loss": 4.030450999457596, + "tokens_seen": 409010176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007079157472417251, + "loss": 1.5853, + "theoretical_loss": 4.030379533623445, + "tokens_seen": 409075712 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007078996990972919, + "loss": 1.5865, + "theoretical_loss": 4.030308082442753, + "tokens_seen": 409141248 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007078836509528585, + "loss": 1.4984, + "theoretical_loss": 4.030236645910169, + "tokens_seen": 409206784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007078676028084253, + "loss": 1.6014, + "theoretical_loss": 4.030165224020345, + "tokens_seen": 409272320 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007078515546639919, + "loss": 1.5542, + "theoretical_loss": 4.030093816767934, + "tokens_seen": 409337856 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007078355065195587, + "loss": 1.7005, + "theoretical_loss": 4.030022424147595, + "tokens_seen": 409403392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007078194583751253, + "loss": 1.5522, + "theoretical_loss": 4.029951046153988, + "tokens_seen": 409468928 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007078034102306921, + "loss": 1.4546, + "theoretical_loss": 4.029879682781776, + "tokens_seen": 409534464 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.9928998947143555, + "objective/train/avg_token_score": 0.007100112736225128, + "objective/train/docs_used": 155224, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.7131178379058838, + "objective/train/original_loss": 1.728844165802002, + "objective/train/theoretical_loss": 4.029808334025624, + "objective/train/tokens_used": 430060000, + "objective/train/unlikelihood_loss": 0.011093653738498688, + "theoretical_loss": 4.029808334025624, + "tokens_seen": 409600000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007077873620862587, + "loss": 1.5951, + "theoretical_loss": 4.029808334025624, + "tokens_seen": 409600000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007077713139418255, + "loss": 1.4547, + "theoretical_loss": 4.0297369998802015, + "tokens_seen": 409665536 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007077552657973921, + "loss": 1.5042, + "theoretical_loss": 4.0296656803401785, + "tokens_seen": 409731072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007077392176529589, + "loss": 1.6637, + "theoretical_loss": 4.029594375400231, + "tokens_seen": 409796608 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007077231695085255, + "loss": 1.4037, + "theoretical_loss": 4.029523085055036, + "tokens_seen": 409862144 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007077071213640923, + "loss": 1.5532, + "theoretical_loss": 4.029451809299273, + "tokens_seen": 409927680 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007076910732196589, + "loss": 1.6882, + "theoretical_loss": 4.029380548127622, + "tokens_seen": 409993216 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007076750250752257, + "loss": 1.5992, + "theoretical_loss": 4.029309301534772, + "tokens_seen": 410058752 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007076589769307923, + "loss": 1.5309, + "theoretical_loss": 4.02923806951541, + "tokens_seen": 410124288 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007076429287863591, + "loss": 1.5593, + "theoretical_loss": 4.029166852064225, + "tokens_seen": 410189824 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007076268806419257, + "loss": 1.6825, + "theoretical_loss": 4.029095649175913, + "tokens_seen": 410255360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007076108324974925, + "loss": 1.6352, + "theoretical_loss": 4.02902446084517, + "tokens_seen": 410320896 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007075947843530591, + "loss": 1.6201, + "theoretical_loss": 4.028953287066695, + "tokens_seen": 410386432 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007075787362086259, + "loss": 1.5929, + "theoretical_loss": 4.02888212783519, + "tokens_seen": 410451968 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007075626880641925, + "loss": 1.597, + "theoretical_loss": 4.028810983145361, + "tokens_seen": 410517504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007075466399197593, + "loss": 1.6854, + "theoretical_loss": 4.0287398529919125, + "tokens_seen": 410583040 + }, + { + "epoch": 0.12, + "learning_rate": 0.000707530591775326, + "loss": 1.4422, + "theoretical_loss": 4.028668737369558, + "tokens_seen": 410648576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007075145436308927, + "loss": 1.5919, + "theoretical_loss": 4.028597636273009, + "tokens_seen": 410714112 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007074984954864595, + "loss": 1.5755, + "theoretical_loss": 4.028526549696982, + "tokens_seen": 410779648 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007074824473420261, + "loss": 1.5809, + "theoretical_loss": 4.0284554776361965, + "tokens_seen": 410845184 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007074663991975929, + "loss": 1.5659, + "theoretical_loss": 4.028384420085372, + "tokens_seen": 410910720 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007074503510531595, + "loss": 1.5871, + "theoretical_loss": 4.028313377039234, + "tokens_seen": 410976256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007074343029087263, + "loss": 1.6151, + "theoretical_loss": 4.028242348492509, + "tokens_seen": 411041792 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007074182547642929, + "loss": 1.5885, + "theoretical_loss": 4.028171334439927, + "tokens_seen": 411107328 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007074022066198597, + "loss": 1.5117, + "theoretical_loss": 4.02810033487622, + "tokens_seen": 411172864 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007073861584754263, + "loss": 1.6971, + "theoretical_loss": 4.028029349796125, + "tokens_seen": 411238400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007073701103309931, + "loss": 1.5344, + "theoretical_loss": 4.027958379194376, + "tokens_seen": 411303936 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007073540621865597, + "loss": 1.6443, + "theoretical_loss": 4.027887423065717, + "tokens_seen": 411369472 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007073380140421265, + "loss": 1.5826, + "theoretical_loss": 4.027816481404892, + "tokens_seen": 411435008 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007073219658976931, + "loss": 1.5174, + "theoretical_loss": 4.0277455542066445, + "tokens_seen": 411500544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007073059177532599, + "loss": 1.5744, + "theoretical_loss": 4.027674641465725, + "tokens_seen": 411566080 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007072898696088265, + "loss": 1.73, + "theoretical_loss": 4.027603743176885, + "tokens_seen": 411631616 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007072738214643933, + "loss": 1.5754, + "theoretical_loss": 4.02753285933488, + "tokens_seen": 411697152 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007072577733199599, + "loss": 1.6951, + "theoretical_loss": 4.027461989934466, + "tokens_seen": 411762688 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007072417251755267, + "loss": 1.5801, + "theoretical_loss": 4.027391134970402, + "tokens_seen": 411828224 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007072256770310933, + "loss": 1.4889, + "theoretical_loss": 4.027320294437452, + "tokens_seen": 411893760 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007072096288866601, + "loss": 1.7251, + "theoretical_loss": 4.027249468330381, + "tokens_seen": 411959296 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007071935807422267, + "loss": 1.7356, + "theoretical_loss": 4.027178656643957, + "tokens_seen": 412024832 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007071775325977935, + "loss": 1.6201, + "theoretical_loss": 4.0271078593729515, + "tokens_seen": 412090368 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007071614844533601, + "loss": 1.6234, + "theoretical_loss": 4.027037076512137, + "tokens_seen": 412155904 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007071454363089269, + "loss": 1.5945, + "theoretical_loss": 4.026966308056291, + "tokens_seen": 412221440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007071293881644935, + "loss": 1.5218, + "theoretical_loss": 4.026895554000192, + "tokens_seen": 412286976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007071133400200603, + "loss": 1.5893, + "theoretical_loss": 4.026824814338621, + "tokens_seen": 412352512 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007070972918756269, + "loss": 1.6435, + "theoretical_loss": 4.026754089066364, + "tokens_seen": 412418048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007070812437311937, + "loss": 1.5056, + "theoretical_loss": 4.026683378178206, + "tokens_seen": 412483584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007070651955867603, + "loss": 1.6749, + "theoretical_loss": 4.026612681668939, + "tokens_seen": 412549120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007070491474423271, + "loss": 1.5807, + "theoretical_loss": 4.026541999533355, + "tokens_seen": 412614656 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007070330992978937, + "loss": 1.5758, + "theoretical_loss": 4.026471331766249, + "tokens_seen": 412680192 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007070170511534605, + "loss": 1.7369, + "theoretical_loss": 4.026400678362418, + "tokens_seen": 412745728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007070010030090271, + "loss": 1.4983, + "theoretical_loss": 4.026330039316665, + "tokens_seen": 412811264 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.9909526109695435, + "objective/train/avg_token_score": 0.009047403000295162, + "objective/train/docs_used": 156351, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4916083812713623, + "objective/train/original_loss": 1.5076066255569458, + "objective/train/theoretical_loss": 4.026259414623792, + "objective/train/tokens_used": 433336800, + "objective/train/unlikelihood_loss": 0.018120326101779938, + "theoretical_loss": 4.026259414623792, + "tokens_seen": 412876800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007069849548645939, + "loss": 1.5036, + "theoretical_loss": 4.026259414623792, + "tokens_seen": 412876800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007069689067201605, + "loss": 1.6697, + "theoretical_loss": 4.026188804278605, + "tokens_seen": 412942336 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007069528585757273, + "loss": 1.5106, + "theoretical_loss": 4.026118208275914, + "tokens_seen": 413007872 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007069368104312939, + "loss": 1.6919, + "theoretical_loss": 4.02604762661053, + "tokens_seen": 413073408 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007069207622868607, + "loss": 1.5172, + "theoretical_loss": 4.025977059277267, + "tokens_seen": 413138944 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007069047141424273, + "loss": 1.5578, + "theoretical_loss": 4.025906506270941, + "tokens_seen": 413204480 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007068886659979941, + "loss": 1.5845, + "theoretical_loss": 4.0258359675863735, + "tokens_seen": 413270016 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007068726178535607, + "loss": 1.5542, + "theoretical_loss": 4.025765443218386, + "tokens_seen": 413335552 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007068565697091275, + "loss": 1.6809, + "theoretical_loss": 4.025694933161804, + "tokens_seen": 413401088 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007068405215646941, + "loss": 1.5403, + "theoretical_loss": 4.025624437411455, + "tokens_seen": 413466624 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007068244734202609, + "loss": 1.5149, + "theoretical_loss": 4.0255539559621685, + "tokens_seen": 413532160 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007068084252758275, + "loss": 1.5235, + "theoretical_loss": 4.025483488808779, + "tokens_seen": 413597696 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007067923771313943, + "loss": 1.5776, + "theoretical_loss": 4.025413035946121, + "tokens_seen": 413663232 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007067763289869609, + "loss": 1.6761, + "theoretical_loss": 4.025342597369034, + "tokens_seen": 413728768 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007067602808425276, + "loss": 1.5467, + "theoretical_loss": 4.025272173072359, + "tokens_seen": 413794304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007067442326980943, + "loss": 1.5124, + "theoretical_loss": 4.025201763050939, + "tokens_seen": 413859840 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706728184553661, + "loss": 1.6008, + "theoretical_loss": 4.025131367299621, + "tokens_seen": 413925376 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007067121364092277, + "loss": 1.4525, + "theoretical_loss": 4.025060985813255, + "tokens_seen": 413990912 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007066960882647944, + "loss": 1.5374, + "theoretical_loss": 4.024990618586692, + "tokens_seen": 414056448 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007066800401203611, + "loss": 1.4873, + "theoretical_loss": 4.024920265614787, + "tokens_seen": 414121984 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007066639919759278, + "loss": 1.631, + "theoretical_loss": 4.024849926892396, + "tokens_seen": 414187520 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007066479438314945, + "loss": 1.6312, + "theoretical_loss": 4.02477960241438, + "tokens_seen": 414253056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007066318956870612, + "loss": 1.5457, + "theoretical_loss": 4.024709292175602, + "tokens_seen": 414318592 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007066158475426278, + "loss": 1.5653, + "theoretical_loss": 4.024638996170927, + "tokens_seen": 414384128 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007065997993981946, + "loss": 1.625, + "theoretical_loss": 4.024568714395221, + "tokens_seen": 414449664 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007065837512537612, + "loss": 1.4849, + "theoretical_loss": 4.0244984468433564, + "tokens_seen": 414515200 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706567703109328, + "loss": 1.5914, + "theoretical_loss": 4.0244281935102055, + "tokens_seen": 414580736 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007065516549648946, + "loss": 1.6133, + "theoretical_loss": 4.024357954390645, + "tokens_seen": 414646272 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007065356068204614, + "loss": 1.5986, + "theoretical_loss": 4.0242877294795525, + "tokens_seen": 414711808 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706519558676028, + "loss": 1.5679, + "theoretical_loss": 4.0242175187718106, + "tokens_seen": 414777344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007065035105315948, + "loss": 1.5628, + "theoretical_loss": 4.024147322262301, + "tokens_seen": 414842880 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007064874623871614, + "loss": 1.7212, + "theoretical_loss": 4.024077139945912, + "tokens_seen": 414908416 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007064714142427282, + "loss": 1.6633, + "theoretical_loss": 4.024006971817532, + "tokens_seen": 414973952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007064553660982948, + "loss": 1.5534, + "theoretical_loss": 4.023936817872053, + "tokens_seen": 415039488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007064393179538616, + "loss": 1.6306, + "theoretical_loss": 4.023866678104368, + "tokens_seen": 415105024 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007064232698094282, + "loss": 1.5509, + "theoretical_loss": 4.023796552509376, + "tokens_seen": 415170560 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706407221664995, + "loss": 1.3888, + "theoretical_loss": 4.023726441081976, + "tokens_seen": 415236096 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007063911735205616, + "loss": 1.4238, + "theoretical_loss": 4.02365634381707, + "tokens_seen": 415301632 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007063751253761284, + "loss": 1.5812, + "theoretical_loss": 4.023586260709564, + "tokens_seen": 415367168 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706359077231695, + "loss": 1.5601, + "theoretical_loss": 4.023516191754363, + "tokens_seen": 415432704 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007063430290872618, + "loss": 1.5728, + "theoretical_loss": 4.02344613694638, + "tokens_seen": 415498240 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007063269809428284, + "loss": 1.5554, + "theoretical_loss": 4.023376096280527, + "tokens_seen": 415563776 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007063109327983952, + "loss": 1.4651, + "theoretical_loss": 4.02330606975172, + "tokens_seen": 415629312 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706294884653962, + "loss": 1.679, + "theoretical_loss": 4.023236057354875, + "tokens_seen": 415694848 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007062788365095286, + "loss": 1.5741, + "theoretical_loss": 4.023166059084916, + "tokens_seen": 415760384 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007062627883650954, + "loss": 1.6624, + "theoretical_loss": 4.023096074936764, + "tokens_seen": 415825920 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706246740220662, + "loss": 1.4597, + "theoretical_loss": 4.0230261049053455, + "tokens_seen": 415891456 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007062306920762288, + "loss": 1.5957, + "theoretical_loss": 4.022956148985589, + "tokens_seen": 415956992 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007062146439317954, + "loss": 1.5738, + "theoretical_loss": 4.0228862071724265, + "tokens_seen": 416022528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007061985957873622, + "loss": 1.5876, + "theoretical_loss": 4.022816279460791, + "tokens_seen": 416088064 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.99329674243927, + "objective/train/avg_token_score": 0.006703243590891361, + "objective/train/docs_used": 157543, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4498416185379028, + "objective/train/original_loss": 1.4592204093933105, + "objective/train/theoretical_loss": 4.02274636584562, + "objective/train/tokens_used": 436613600, + "objective/train/unlikelihood_loss": 0.01674993894994259, + "theoretical_loss": 4.02274636584562, + "tokens_seen": 416153600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007061825476429288, + "loss": 1.5685, + "theoretical_loss": 4.02274636584562, + "tokens_seen": 416153600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007061664994984956, + "loss": 1.5771, + "theoretical_loss": 4.022676466321853, + "tokens_seen": 416219136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007061504513540622, + "loss": 1.7036, + "theoretical_loss": 4.022606580884429, + "tokens_seen": 416284672 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706134403209629, + "loss": 1.6695, + "theoretical_loss": 4.0225367095282945, + "tokens_seen": 416350208 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007061183550651956, + "loss": 1.676, + "theoretical_loss": 4.022466852248396, + "tokens_seen": 416415744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007061023069207624, + "loss": 1.5068, + "theoretical_loss": 4.0223970090396834, + "tokens_seen": 416481280 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706086258776329, + "loss": 1.5889, + "theoretical_loss": 4.0223271798971085, + "tokens_seen": 416546816 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007060702106318958, + "loss": 1.5348, + "theoretical_loss": 4.0222573648156255, + "tokens_seen": 416612352 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007060541624874624, + "loss": 1.6814, + "theoretical_loss": 4.022187563790193, + "tokens_seen": 416677888 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007060381143430292, + "loss": 1.5415, + "theoretical_loss": 4.02211777681577, + "tokens_seen": 416743424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007060220661985958, + "loss": 1.6923, + "theoretical_loss": 4.022048003887319, + "tokens_seen": 416808960 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007060060180541626, + "loss": 1.5051, + "theoretical_loss": 4.021978244999806, + "tokens_seen": 416874496 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007059899699097292, + "loss": 1.7187, + "theoretical_loss": 4.021908500148198, + "tokens_seen": 416940032 + }, + { + "epoch": 0.13, + "learning_rate": 0.000705973921765296, + "loss": 1.6108, + "theoretical_loss": 4.0218387693274655, + "tokens_seen": 417005568 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007059578736208626, + "loss": 1.4543, + "theoretical_loss": 4.021769052532582, + "tokens_seen": 417071104 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007059418254764294, + "loss": 1.5139, + "theoretical_loss": 4.0216993497585225, + "tokens_seen": 417136640 + }, + { + "epoch": 0.13, + "learning_rate": 0.000705925777331996, + "loss": 1.6802, + "theoretical_loss": 4.021629661000265, + "tokens_seen": 417202176 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007059097291875628, + "loss": 1.6032, + "theoretical_loss": 4.02155998625279, + "tokens_seen": 417267712 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007058936810431294, + "loss": 1.6481, + "theoretical_loss": 4.021490325511082, + "tokens_seen": 417333248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007058776328986962, + "loss": 1.5952, + "theoretical_loss": 4.021420678770125, + "tokens_seen": 417398784 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007058615847542628, + "loss": 1.6328, + "theoretical_loss": 4.02135104602491, + "tokens_seen": 417464320 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007058455366098296, + "loss": 1.5582, + "theoretical_loss": 4.021281427270425, + "tokens_seen": 417529856 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007058294884653962, + "loss": 1.628, + "theoretical_loss": 4.021211822501666, + "tokens_seen": 417595392 + }, + { + "epoch": 0.13, + "learning_rate": 0.000705813440320963, + "loss": 1.547, + "theoretical_loss": 4.021142231713629, + "tokens_seen": 417660928 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007057973921765296, + "loss": 1.5838, + "theoretical_loss": 4.021072654901312, + "tokens_seen": 417726464 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007057813440320964, + "loss": 1.4701, + "theoretical_loss": 4.021003092059716, + "tokens_seen": 417792000 + }, + { + "epoch": 0.13, + "learning_rate": 0.000705765295887663, + "loss": 1.5245, + "theoretical_loss": 4.020933543183845, + "tokens_seen": 417857536 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007057492477432298, + "loss": 1.5839, + "theoretical_loss": 4.020864008268706, + "tokens_seen": 417923072 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007057331995987964, + "loss": 1.4188, + "theoretical_loss": 4.020794487309308, + "tokens_seen": 417988608 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007057171514543632, + "loss": 1.5757, + "theoretical_loss": 4.020724980300663, + "tokens_seen": 418054144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007057011033099298, + "loss": 1.674, + "theoretical_loss": 4.020655487237785, + "tokens_seen": 418119680 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007056850551654966, + "loss": 1.5637, + "theoretical_loss": 4.02058600811569, + "tokens_seen": 418185216 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007056690070210632, + "loss": 1.6266, + "theoretical_loss": 4.020516542929398, + "tokens_seen": 418250752 + }, + { + "epoch": 0.13, + "learning_rate": 0.00070565295887663, + "loss": 1.4694, + "theoretical_loss": 4.0204470916739306, + "tokens_seen": 418316288 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007056369107321966, + "loss": 1.5641, + "theoretical_loss": 4.020377654344312, + "tokens_seen": 418381824 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007056208625877634, + "loss": 1.6756, + "theoretical_loss": 4.020308230935569, + "tokens_seen": 418447360 + }, + { + "epoch": 0.13, + "learning_rate": 0.00070560481444333, + "loss": 1.7054, + "theoretical_loss": 4.020238821442733, + "tokens_seen": 418512896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007055887662988968, + "loss": 1.672, + "theoretical_loss": 4.020169425860834, + "tokens_seen": 418578432 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007055727181544634, + "loss": 1.6317, + "theoretical_loss": 4.0201000441849075, + "tokens_seen": 418643968 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007055566700100302, + "loss": 1.6883, + "theoretical_loss": 4.0200306764099905, + "tokens_seen": 418709504 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007055406218655968, + "loss": 1.6308, + "theoretical_loss": 4.019961322531122, + "tokens_seen": 418775040 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007055245737211636, + "loss": 1.5898, + "theoretical_loss": 4.019891982543346, + "tokens_seen": 418840576 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007055085255767302, + "loss": 1.5772, + "theoretical_loss": 4.019822656441706, + "tokens_seen": 418906112 + }, + { + "epoch": 0.13, + "learning_rate": 0.000705492477432297, + "loss": 1.5238, + "theoretical_loss": 4.01975334422125, + "tokens_seen": 418971648 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007054764292878636, + "loss": 1.5887, + "theoretical_loss": 4.019684045877027, + "tokens_seen": 419037184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007054603811434304, + "loss": 1.4491, + "theoretical_loss": 4.019614761404091, + "tokens_seen": 419102720 + }, + { + "epoch": 0.13, + "learning_rate": 0.000705444332998997, + "loss": 1.6207, + "theoretical_loss": 4.019545490797495, + "tokens_seen": 419168256 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007054282848545638, + "loss": 1.5661, + "theoretical_loss": 4.0194762340522985, + "tokens_seen": 419233792 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007054122367101304, + "loss": 1.5772, + "theoretical_loss": 4.01940699116356, + "tokens_seen": 419299328 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007053961885656972, + "loss": 1.6771, + "theoretical_loss": 4.019337762126343, + "tokens_seen": 419364864 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.9930273294448853, + "objective/train/avg_token_score": 0.006972755771130323, + "objective/train/docs_used": 158751, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.7292683124542236, + "objective/train/original_loss": 1.7424414157867432, + "objective/train/theoretical_loss": 4.019268546935713, + "objective/train/tokens_used": 439890400, + "objective/train/unlikelihood_loss": 0.012689784169197083, + "theoretical_loss": 4.019268546935713, + "tokens_seen": 419430400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007053801404212638, + "loss": 1.599, + "theoretical_loss": 4.019268546935713, + "tokens_seen": 419430400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007053640922768305, + "loss": 1.5872, + "theoretical_loss": 4.0191993455867365, + "tokens_seen": 419495936 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007053480441323972, + "loss": 1.7114, + "theoretical_loss": 4.019130158074484, + "tokens_seen": 419561472 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007053319959879639, + "loss": 1.6558, + "theoretical_loss": 4.019060984394028, + "tokens_seen": 419627008 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007053159478435306, + "loss": 1.6679, + "theoretical_loss": 4.018991824540445, + "tokens_seen": 419692544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007052998996990973, + "loss": 1.638, + "theoretical_loss": 4.018922678508812, + "tokens_seen": 419758080 + }, + { + "epoch": 0.13, + "learning_rate": 0.000705283851554664, + "loss": 1.6167, + "theoretical_loss": 4.018853546294208, + "tokens_seen": 419823616 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007052678034102307, + "loss": 1.5863, + "theoretical_loss": 4.018784427891718, + "tokens_seen": 419889152 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007052517552657974, + "loss": 1.6233, + "theoretical_loss": 4.018715323296425, + "tokens_seen": 419954688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007052357071213641, + "loss": 1.4828, + "theoretical_loss": 4.018646232503419, + "tokens_seen": 420020224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007052196589769307, + "loss": 1.4291, + "theoretical_loss": 4.0185771555077885, + "tokens_seen": 420085760 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007052036108324975, + "loss": 1.5775, + "theoretical_loss": 4.018508092304628, + "tokens_seen": 420151296 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007051875626880641, + "loss": 1.5989, + "theoretical_loss": 4.018439042889032, + "tokens_seen": 420216832 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007051715145436309, + "loss": 1.6399, + "theoretical_loss": 4.018370007256098, + "tokens_seen": 420282368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007051554663991975, + "loss": 1.6544, + "theoretical_loss": 4.018300985400927, + "tokens_seen": 420347904 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007051394182547643, + "loss": 1.6662, + "theoretical_loss": 4.018231977318622, + "tokens_seen": 420413440 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007051233701103309, + "loss": 1.6369, + "theoretical_loss": 4.018162983004287, + "tokens_seen": 420478976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007051073219658977, + "loss": 1.4981, + "theoretical_loss": 4.018094002453031, + "tokens_seen": 420544512 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007050912738214644, + "loss": 1.6393, + "theoretical_loss": 4.018025035659965, + "tokens_seen": 420610048 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007050752256770311, + "loss": 1.691, + "theoretical_loss": 4.017956082620201, + "tokens_seen": 420675584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007050591775325978, + "loss": 1.632, + "theoretical_loss": 4.0178871433288545, + "tokens_seen": 420741120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007050431293881645, + "loss": 1.6911, + "theoretical_loss": 4.017818217781044, + "tokens_seen": 420806656 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007050270812437312, + "loss": 1.5886, + "theoretical_loss": 4.017749305971889, + "tokens_seen": 420872192 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007050110330992979, + "loss": 1.5178, + "theoretical_loss": 4.0176804078965125, + "tokens_seen": 420937728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007049949849548647, + "loss": 1.6229, + "theoretical_loss": 4.017611523550039, + "tokens_seen": 421003264 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007049789368104313, + "loss": 1.6753, + "theoretical_loss": 4.017542652927599, + "tokens_seen": 421068800 + }, + { + "epoch": 0.13, + "learning_rate": 0.000704962888665998, + "loss": 1.6163, + "theoretical_loss": 4.01747379602432, + "tokens_seen": 421134336 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007049468405215647, + "loss": 1.6186, + "theoretical_loss": 4.017404952835336, + "tokens_seen": 421199872 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007049307923771315, + "loss": 1.5326, + "theoretical_loss": 4.017336123355783, + "tokens_seen": 421265408 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007049147442326981, + "loss": 1.6105, + "theoretical_loss": 4.017267307580798, + "tokens_seen": 421330944 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007048986960882649, + "loss": 1.6444, + "theoretical_loss": 4.017198505505521, + "tokens_seen": 421396480 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007048826479438315, + "loss": 1.639, + "theoretical_loss": 4.017129717125094, + "tokens_seen": 421462016 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007048665997993983, + "loss": 1.6601, + "theoretical_loss": 4.017060942434664, + "tokens_seen": 421527552 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007048505516549649, + "loss": 1.5663, + "theoretical_loss": 4.016992181429377, + "tokens_seen": 421593088 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007048345035105317, + "loss": 1.6955, + "theoretical_loss": 4.016923434104386, + "tokens_seen": 421658624 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007048184553660983, + "loss": 1.606, + "theoretical_loss": 4.01685470045484, + "tokens_seen": 421724160 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007048024072216651, + "loss": 1.6002, + "theoretical_loss": 4.0167859804758965, + "tokens_seen": 421789696 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007047863590772317, + "loss": 1.5554, + "theoretical_loss": 4.016717274162712, + "tokens_seen": 421855232 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007047703109327985, + "loss": 1.5247, + "theoretical_loss": 4.016648581510448, + "tokens_seen": 421920768 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007047542627883651, + "loss": 1.458, + "theoretical_loss": 4.016579902514264, + "tokens_seen": 421986304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007047382146439319, + "loss": 1.561, + "theoretical_loss": 4.016511237169329, + "tokens_seen": 422051840 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007047221664994985, + "loss": 1.6426, + "theoretical_loss": 4.016442585470807, + "tokens_seen": 422117376 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007047061183550653, + "loss": 1.5441, + "theoretical_loss": 4.01637394741387, + "tokens_seen": 422182912 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007046900702106319, + "loss": 1.6653, + "theoretical_loss": 4.016305322993689, + "tokens_seen": 422248448 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007046740220661987, + "loss": 1.513, + "theoretical_loss": 4.016236712205439, + "tokens_seen": 422313984 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007046579739217653, + "loss": 1.6114, + "theoretical_loss": 4.016168115044298, + "tokens_seen": 422379520 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007046419257773321, + "loss": 1.6503, + "theoretical_loss": 4.016099531505445, + "tokens_seen": 422445056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007046258776328987, + "loss": 1.5402, + "theoretical_loss": 4.016030961584063, + "tokens_seen": 422510592 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007046098294884655, + "loss": 1.4978, + "theoretical_loss": 4.015962405275336, + "tokens_seen": 422576128 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007045937813440321, + "loss": 1.5868, + "theoretical_loss": 4.015893862574451, + "tokens_seen": 422641664 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.9928730130195618, + "objective/train/avg_token_score": 0.0071269916370511055, + "objective/train/docs_used": 159824, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.541268229484558, + "objective/train/original_loss": 1.5540826320648193, + "objective/train/theoretical_loss": 4.015825333476598, + "objective/train/tokens_used": 443167200, + "objective/train/unlikelihood_loss": 0.014573877677321434, + "theoretical_loss": 4.015825333476598, + "tokens_seen": 422707200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007045777331995989, + "loss": 1.5854, + "theoretical_loss": 4.015825333476598, + "tokens_seen": 422707200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007045616850551655, + "loss": 1.6702, + "theoretical_loss": 4.0157568179769685, + "tokens_seen": 422772736 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007045456369107323, + "loss": 1.6146, + "theoretical_loss": 4.015688316070757, + "tokens_seen": 422838272 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007045295887662989, + "loss": 1.6695, + "theoretical_loss": 4.01561982775316, + "tokens_seen": 422903808 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007045135406218657, + "loss": 1.5823, + "theoretical_loss": 4.015551353019378, + "tokens_seen": 422969344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007044974924774323, + "loss": 1.5868, + "theoretical_loss": 4.015482891864612, + "tokens_seen": 423034880 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007044814443329991, + "loss": 1.6321, + "theoretical_loss": 4.015414444284065, + "tokens_seen": 423100416 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007044653961885657, + "loss": 1.5507, + "theoretical_loss": 4.015346010272946, + "tokens_seen": 423165952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007044493480441325, + "loss": 1.5781, + "theoretical_loss": 4.0152775898264625, + "tokens_seen": 423231488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007044332998996991, + "loss": 1.7038, + "theoretical_loss": 4.0152091829398255, + "tokens_seen": 423297024 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007044172517552659, + "loss": 1.5655, + "theoretical_loss": 4.0151407896082505, + "tokens_seen": 423362560 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007044012036108325, + "loss": 1.6805, + "theoretical_loss": 4.015072409826953, + "tokens_seen": 423428096 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007043851554663993, + "loss": 1.498, + "theoretical_loss": 4.015004043591151, + "tokens_seen": 423493632 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007043691073219659, + "loss": 1.6069, + "theoretical_loss": 4.014935690896067, + "tokens_seen": 423559168 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007043530591775327, + "loss": 1.6475, + "theoretical_loss": 4.014867351736924, + "tokens_seen": 423624704 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007043370110330993, + "loss": 1.6399, + "theoretical_loss": 4.014799026108948, + "tokens_seen": 423690240 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007043209628886661, + "loss": 1.5659, + "theoretical_loss": 4.014730714007367, + "tokens_seen": 423755776 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007043049147442327, + "loss": 1.5481, + "theoretical_loss": 4.014662415427413, + "tokens_seen": 423821312 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007042888665997995, + "loss": 1.5291, + "theoretical_loss": 4.014594130364319, + "tokens_seen": 423886848 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007042728184553661, + "loss": 1.4922, + "theoretical_loss": 4.01452585881332, + "tokens_seen": 423952384 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007042567703109329, + "loss": 1.6438, + "theoretical_loss": 4.014457600769655, + "tokens_seen": 424017920 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007042407221664995, + "loss": 1.6876, + "theoretical_loss": 4.0143893562285635, + "tokens_seen": 424083456 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007042246740220663, + "loss": 1.5677, + "theoretical_loss": 4.014321125185289, + "tokens_seen": 424148992 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007042086258776329, + "loss": 1.5988, + "theoretical_loss": 4.014252907635077, + "tokens_seen": 424214528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007041925777331997, + "loss": 1.6407, + "theoretical_loss": 4.014184703573176, + "tokens_seen": 424280064 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007041765295887663, + "loss": 1.6867, + "theoretical_loss": 4.014116512994835, + "tokens_seen": 424345600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007041604814443331, + "loss": 1.7415, + "theoretical_loss": 4.014048335895307, + "tokens_seen": 424411136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007041444332998997, + "loss": 1.6687, + "theoretical_loss": 4.0139801722698465, + "tokens_seen": 424476672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007041283851554665, + "loss": 1.681, + "theoretical_loss": 4.013912022113711, + "tokens_seen": 424542208 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007041123370110331, + "loss": 1.6312, + "theoretical_loss": 4.013843885422162, + "tokens_seen": 424607744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007040962888665999, + "loss": 1.5737, + "theoretical_loss": 4.01377576219046, + "tokens_seen": 424673280 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007040802407221665, + "loss": 1.612, + "theoretical_loss": 4.01370765241387, + "tokens_seen": 424738816 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007040641925777333, + "loss": 1.5308, + "theoretical_loss": 4.013639556087659, + "tokens_seen": 424804352 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007040481444332999, + "loss": 1.6585, + "theoretical_loss": 4.0135714732070955, + "tokens_seen": 424869888 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007040320962888667, + "loss": 1.6668, + "theoretical_loss": 4.013503403767453, + "tokens_seen": 424935424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007040160481444333, + "loss": 1.58, + "theoretical_loss": 4.013435347764005, + "tokens_seen": 425000960 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007040000000000001, + "loss": 1.4846, + "theoretical_loss": 4.013367305192027, + "tokens_seen": 425066496 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007039839518555667, + "loss": 1.557, + "theoretical_loss": 4.0132992760468, + "tokens_seen": 425132032 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007039679037111334, + "loss": 1.5259, + "theoretical_loss": 4.013231260323603, + "tokens_seen": 425197568 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007039518555667001, + "loss": 1.4363, + "theoretical_loss": 4.0131632580177214, + "tokens_seen": 425263104 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007039358074222668, + "loss": 1.6744, + "theoretical_loss": 4.01309526912444, + "tokens_seen": 425328640 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007039197592778335, + "loss": 1.7089, + "theoretical_loss": 4.013027293639049, + "tokens_seen": 425394176 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007039037111334002, + "loss": 1.5764, + "theoretical_loss": 4.0129593315568375, + "tokens_seen": 425459712 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007038876629889669, + "loss": 1.6506, + "theoretical_loss": 4.0128913828731, + "tokens_seen": 425525248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007038716148445336, + "loss": 1.6069, + "theoretical_loss": 4.012823447583132, + "tokens_seen": 425590784 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007038555667001003, + "loss": 1.725, + "theoretical_loss": 4.012755525682231, + "tokens_seen": 425656320 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703839518555667, + "loss": 1.6025, + "theoretical_loss": 4.012687617165698, + "tokens_seen": 425721856 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007038234704112337, + "loss": 1.5952, + "theoretical_loss": 4.012619722028835, + "tokens_seen": 425787392 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007038074222668004, + "loss": 1.575, + "theoretical_loss": 4.012551840266948, + "tokens_seen": 425852928 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007037913741223671, + "loss": 1.4586, + "theoretical_loss": 4.0124839718753424, + "tokens_seen": 425918464 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.9950158596038818, + "objective/train/avg_token_score": 0.0049841273576021194, + "objective/train/docs_used": 161088, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4542001485824585, + "objective/train/original_loss": 1.4624392986297607, + "objective/train/theoretical_loss": 4.01241611684933, + "objective/train/tokens_used": 446444000, + "objective/train/unlikelihood_loss": 0.011422989889979362, + "theoretical_loss": 4.01241611684933, + "tokens_seen": 425984000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007037753259779338, + "loss": 1.5468, + "theoretical_loss": 4.01241611684933, + "tokens_seen": 425984000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007037592778335005, + "loss": 1.6241, + "theoretical_loss": 4.012348275184223, + "tokens_seen": 426049536 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007037432296890672, + "loss": 1.5031, + "theoretical_loss": 4.012280446875336, + "tokens_seen": 426115072 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703727181544634, + "loss": 1.6488, + "theoretical_loss": 4.012212631917986, + "tokens_seen": 426180608 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007037111334002006, + "loss": 1.6172, + "theoretical_loss": 4.012144830307491, + "tokens_seen": 426246144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007036950852557673, + "loss": 1.5218, + "theoretical_loss": 4.012077042039174, + "tokens_seen": 426311680 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703679037111334, + "loss": 1.617, + "theoretical_loss": 4.012009267108358, + "tokens_seen": 426377216 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007036629889669007, + "loss": 1.6364, + "theoretical_loss": 4.01194150551037, + "tokens_seen": 426442752 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007036469408224674, + "loss": 1.6062, + "theoretical_loss": 4.011873757240539, + "tokens_seen": 426508288 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007036308926780342, + "loss": 1.6955, + "theoretical_loss": 4.011806022294196, + "tokens_seen": 426573824 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007036148445336008, + "loss": 1.6698, + "theoretical_loss": 4.011738300666674, + "tokens_seen": 426639360 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007035987963891676, + "loss": 1.6394, + "theoretical_loss": 4.011670592353309, + "tokens_seen": 426704896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007035827482447342, + "loss": 1.5188, + "theoretical_loss": 4.011602897349439, + "tokens_seen": 426770432 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703566700100301, + "loss": 1.5992, + "theoretical_loss": 4.011535215650405, + "tokens_seen": 426835968 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007035506519558676, + "loss": 1.514, + "theoretical_loss": 4.0114675472515495, + "tokens_seen": 426901504 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007035346038114344, + "loss": 1.604, + "theoretical_loss": 4.011399892148218, + "tokens_seen": 426967040 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703518555667001, + "loss": 1.5647, + "theoretical_loss": 4.011332250335758, + "tokens_seen": 427032576 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007035025075225678, + "loss": 1.669, + "theoretical_loss": 4.011264621809518, + "tokens_seen": 427098112 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007034864593781344, + "loss": 1.602, + "theoretical_loss": 4.011197006564853, + "tokens_seen": 427163648 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007034704112337012, + "loss": 1.5991, + "theoretical_loss": 4.011129404597114, + "tokens_seen": 427229184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007034543630892678, + "loss": 1.6992, + "theoretical_loss": 4.011061815901661, + "tokens_seen": 427294720 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007034383149448346, + "loss": 1.601, + "theoretical_loss": 4.010994240473853, + "tokens_seen": 427360256 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007034222668004012, + "loss": 1.6232, + "theoretical_loss": 4.0109266783090485, + "tokens_seen": 427425792 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703406218655968, + "loss": 1.6129, + "theoretical_loss": 4.010859129402615, + "tokens_seen": 427491328 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007033901705115346, + "loss": 1.5638, + "theoretical_loss": 4.010791593749916, + "tokens_seen": 427556864 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007033741223671014, + "loss": 1.6532, + "theoretical_loss": 4.010724071346322, + "tokens_seen": 427622400 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703358074222668, + "loss": 1.6717, + "theoretical_loss": 4.0106565621872035, + "tokens_seen": 427687936 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007033420260782348, + "loss": 1.571, + "theoretical_loss": 4.010589066267933, + "tokens_seen": 427753472 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007033259779338014, + "loss": 1.6603, + "theoretical_loss": 4.010521583583886, + "tokens_seen": 427819008 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007033099297893682, + "loss": 1.6205, + "theoretical_loss": 4.010454114130441, + "tokens_seen": 427884544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007032938816449348, + "loss": 1.584, + "theoretical_loss": 4.010386657902979, + "tokens_seen": 427950080 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007032778335005016, + "loss": 1.5567, + "theoretical_loss": 4.01031921489688, + "tokens_seen": 428015616 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007032617853560682, + "loss": 1.5823, + "theoretical_loss": 4.01025178510753, + "tokens_seen": 428081152 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703245737211635, + "loss": 1.4686, + "theoretical_loss": 4.010184368530318, + "tokens_seen": 428146688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007032296890672016, + "loss": 1.5392, + "theoretical_loss": 4.0101169651606305, + "tokens_seen": 428212224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007032136409227684, + "loss": 1.6639, + "theoretical_loss": 4.010049574993861, + "tokens_seen": 428277760 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703197592778335, + "loss": 1.6107, + "theoretical_loss": 4.009982198025404, + "tokens_seen": 428343296 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007031815446339018, + "loss": 1.6108, + "theoretical_loss": 4.0099148342506545, + "tokens_seen": 428408832 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007031654964894684, + "loss": 1.6579, + "theoretical_loss": 4.009847483665011, + "tokens_seen": 428474368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007031494483450352, + "loss": 1.7519, + "theoretical_loss": 4.009780146263877, + "tokens_seen": 428539904 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007031334002006018, + "loss": 1.5486, + "theoretical_loss": 4.009712822042652, + "tokens_seen": 428605440 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007031173520561686, + "loss": 1.5926, + "theoretical_loss": 4.009645510996744, + "tokens_seen": 428670976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007031013039117352, + "loss": 1.4848, + "theoretical_loss": 4.0095782131215625, + "tokens_seen": 428736512 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703085255767302, + "loss": 1.5454, + "theoretical_loss": 4.0095109284125146, + "tokens_seen": 428802048 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007030692076228686, + "loss": 1.6634, + "theoretical_loss": 4.009443656865013, + "tokens_seen": 428867584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007030531594784354, + "loss": 1.5501, + "theoretical_loss": 4.009376398474475, + "tokens_seen": 428933120 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703037111334002, + "loss": 1.543, + "theoretical_loss": 4.009309153236316, + "tokens_seen": 428998656 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007030210631895688, + "loss": 1.5651, + "theoretical_loss": 4.009241921145956, + "tokens_seen": 429064192 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007030050150451354, + "loss": 1.6333, + "theoretical_loss": 4.009174702198816, + "tokens_seen": 429129728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007029889669007022, + "loss": 1.5094, + "theoretical_loss": 4.009107496390321, + "tokens_seen": 429195264 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.9941730499267578, + "objective/train/avg_token_score": 0.005827016197144985, + "objective/train/docs_used": 162190, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.641357183456421, + "objective/train/original_loss": 1.6514679193496704, + "objective/train/theoretical_loss": 4.009040303715897, + "objective/train/tokens_used": 449720800, + "objective/train/unlikelihood_loss": 0.012080559507012367, + "theoretical_loss": 4.009040303715897, + "tokens_seen": 429260800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007029729187562688, + "loss": 1.66, + "theoretical_loss": 4.009040303715897, + "tokens_seen": 429260800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007029568706118356, + "loss": 1.5444, + "theoretical_loss": 4.008973124170971, + "tokens_seen": 429326336 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007029408224674022, + "loss": 1.7315, + "theoretical_loss": 4.008905957750976, + "tokens_seen": 429391872 + }, + { + "epoch": 0.13, + "learning_rate": 0.000702924774322969, + "loss": 1.6508, + "theoretical_loss": 4.0088388044513446, + "tokens_seen": 429457408 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007029087261785356, + "loss": 1.5038, + "theoretical_loss": 4.0087716642675115, + "tokens_seen": 429522944 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007028926780341024, + "loss": 1.5551, + "theoretical_loss": 4.008704537194916, + "tokens_seen": 429588480 + }, + { + "epoch": 0.13, + "learning_rate": 0.000702876629889669, + "loss": 1.6032, + "theoretical_loss": 4.008637423228996, + "tokens_seen": 429654016 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007028605817452358, + "loss": 1.6722, + "theoretical_loss": 4.008570322365195, + "tokens_seen": 429719552 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007028445336008024, + "loss": 1.5961, + "theoretical_loss": 4.008503234598958, + "tokens_seen": 429785088 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007028284854563692, + "loss": 1.6376, + "theoretical_loss": 4.008436159925731, + "tokens_seen": 429850624 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007028124373119358, + "loss": 1.6433, + "theoretical_loss": 4.008369098340963, + "tokens_seen": 429916160 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007027963891675026, + "loss": 1.5055, + "theoretical_loss": 4.008302049840106, + "tokens_seen": 429981696 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007027803410230692, + "loss": 1.6173, + "theoretical_loss": 4.008235014418614, + "tokens_seen": 430047232 + }, + { + "epoch": 0.13, + "learning_rate": 0.000702764292878636, + "loss": 1.6625, + "theoretical_loss": 4.008167992071941, + "tokens_seen": 430112768 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007027482447342026, + "loss": 1.6082, + "theoretical_loss": 4.008100982795549, + "tokens_seen": 430178304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007027321965897694, + "loss": 1.4983, + "theoretical_loss": 4.008033986584895, + "tokens_seen": 430243840 + }, + { + "epoch": 0.13, + "learning_rate": 0.000702716148445336, + "loss": 1.6731, + "theoretical_loss": 4.007967003435443, + "tokens_seen": 430309376 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007027001003009028, + "loss": 1.4449, + "theoretical_loss": 4.007900033342658, + "tokens_seen": 430374912 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007026840521564694, + "loss": 1.5955, + "theoretical_loss": 4.007833076302006, + "tokens_seen": 430440448 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007026680040120362, + "loss": 1.5474, + "theoretical_loss": 4.00776613230896, + "tokens_seen": 430505984 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007026519558676028, + "loss": 1.5126, + "theoretical_loss": 4.007699201358989, + "tokens_seen": 430571520 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007026359077231696, + "loss": 1.6932, + "theoretical_loss": 4.007632283447567, + "tokens_seen": 430637056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007026198595787362, + "loss": 1.4959, + "theoretical_loss": 4.007565378570171, + "tokens_seen": 430702592 + }, + { + "epoch": 0.13, + "learning_rate": 0.000702603811434303, + "loss": 1.6124, + "theoretical_loss": 4.00749848672228, + "tokens_seen": 430768128 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007025877632898696, + "loss": 1.5225, + "theoretical_loss": 4.007431607899375, + "tokens_seen": 430833664 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007025717151454363, + "loss": 1.4888, + "theoretical_loss": 4.007364742096938, + "tokens_seen": 430899200 + }, + { + "epoch": 0.13, + "learning_rate": 0.000702555667001003, + "loss": 1.5743, + "theoretical_loss": 4.007297889310455, + "tokens_seen": 430964736 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007025396188565697, + "loss": 1.5725, + "theoretical_loss": 4.007231049535413, + "tokens_seen": 431030272 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007025235707121364, + "loss": 1.5766, + "theoretical_loss": 4.007164222767304, + "tokens_seen": 431095808 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007025075225677031, + "loss": 1.6667, + "theoretical_loss": 4.0070974090016165, + "tokens_seen": 431161344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007024914744232698, + "loss": 1.6083, + "theoretical_loss": 4.007030608233848, + "tokens_seen": 431226880 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007024754262788365, + "loss": 1.7085, + "theoretical_loss": 4.006963820459494, + "tokens_seen": 431292416 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007024593781344032, + "loss": 1.5515, + "theoretical_loss": 4.006897045674053, + "tokens_seen": 431357952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007024433299899699, + "loss": 1.5793, + "theoretical_loss": 4.006830283873027, + "tokens_seen": 431423488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007024272818455366, + "loss": 1.5286, + "theoretical_loss": 4.006763535051918, + "tokens_seen": 431489024 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007024112337011033, + "loss": 1.5524, + "theoretical_loss": 4.006696799206233, + "tokens_seen": 431554560 + }, + { + "epoch": 0.13, + "learning_rate": 0.00070239518555667, + "loss": 1.5808, + "theoretical_loss": 4.006630076331478, + "tokens_seen": 431620096 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007023791374122367, + "loss": 1.6034, + "theoretical_loss": 4.0065633664231655, + "tokens_seen": 431685632 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007023630892678034, + "loss": 1.5856, + "theoretical_loss": 4.006496669476805, + "tokens_seen": 431751168 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007023470411233701, + "loss": 1.5892, + "theoretical_loss": 4.006429985487913, + "tokens_seen": 431816704 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007023309929789368, + "loss": 1.5442, + "theoretical_loss": 4.006363314452007, + "tokens_seen": 431882240 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007023149448345035, + "loss": 1.6144, + "theoretical_loss": 4.006296656364603, + "tokens_seen": 431947776 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007022988966900702, + "loss": 1.6347, + "theoretical_loss": 4.006230011221225, + "tokens_seen": 432013312 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007022828485456369, + "loss": 1.6977, + "theoretical_loss": 4.006163379017394, + "tokens_seen": 432078848 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007022668004012037, + "loss": 1.5944, + "theoretical_loss": 4.0060967597486385, + "tokens_seen": 432144384 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007022507522567703, + "loss": 1.5891, + "theoretical_loss": 4.006030153410483, + "tokens_seen": 432209920 + }, + { + "epoch": 0.13, + "learning_rate": 0.000702234704112337, + "loss": 1.6384, + "theoretical_loss": 4.005963559998461, + "tokens_seen": 432275456 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007022186559679037, + "loss": 1.5623, + "theoretical_loss": 4.005896979508103, + "tokens_seen": 432340992 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007022026078234705, + "loss": 1.6389, + "theoretical_loss": 4.005830411934944, + "tokens_seen": 432406528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007021865596790371, + "loss": 1.5923, + "theoretical_loss": 4.005763857274521, + "tokens_seen": 432472064 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.9946593046188354, + "objective/train/avg_token_score": 0.005340618547052145, + "objective/train/docs_used": 163281, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5582692623138428, + "objective/train/original_loss": 1.5680944919586182, + "objective/train/theoretical_loss": 4.005697315522371, + "objective/train/tokens_used": 452997600, + "objective/train/unlikelihood_loss": 0.009947868064045906, + "theoretical_loss": 4.005697315522371, + "tokens_seen": 432537600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007021705115346039, + "loss": 1.6171, + "theoretical_loss": 4.005697315522371, + "tokens_seen": 432537600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007021544633901705, + "loss": 1.6088, + "theoretical_loss": 4.005630786674039, + "tokens_seen": 432603136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007021384152457373, + "loss": 1.6255, + "theoretical_loss": 4.005564270725065, + "tokens_seen": 432668672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007021223671013039, + "loss": 1.606, + "theoretical_loss": 4.005497767670996, + "tokens_seen": 432734208 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007021063189568707, + "loss": 1.4742, + "theoretical_loss": 4.0054312775073795, + "tokens_seen": 432799744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007020902708124373, + "loss": 1.5275, + "theoretical_loss": 4.005364800229766, + "tokens_seen": 432865280 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007020742226680041, + "loss": 1.6377, + "theoretical_loss": 4.005298335833708, + "tokens_seen": 432930816 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007020581745235707, + "loss": 1.5719, + "theoretical_loss": 4.005231884314759, + "tokens_seen": 432996352 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007020421263791375, + "loss": 1.5448, + "theoretical_loss": 4.005165445668477, + "tokens_seen": 433061888 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007020260782347041, + "loss": 1.5293, + "theoretical_loss": 4.0050990198904195, + "tokens_seen": 433127424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007020100300902709, + "loss": 1.6347, + "theoretical_loss": 4.005032606976149, + "tokens_seen": 433192960 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007019939819458375, + "loss": 1.6098, + "theoretical_loss": 4.004966206921227, + "tokens_seen": 433258496 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007019779338014043, + "loss": 1.5519, + "theoretical_loss": 4.004899819721221, + "tokens_seen": 433324032 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007019618856569709, + "loss": 1.5861, + "theoretical_loss": 4.004833445371697, + "tokens_seen": 433389568 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007019458375125377, + "loss": 1.6894, + "theoretical_loss": 4.004767083868226, + "tokens_seen": 433455104 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007019297893681043, + "loss": 1.7075, + "theoretical_loss": 4.004700735206381, + "tokens_seen": 433520640 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007019137412236711, + "loss": 1.6457, + "theoretical_loss": 4.004634399381733, + "tokens_seen": 433586176 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007018976930792377, + "loss": 1.7092, + "theoretical_loss": 4.0045680763898615, + "tokens_seen": 433651712 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007018816449348045, + "loss": 1.5363, + "theoretical_loss": 4.004501766226344, + "tokens_seen": 433717248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007018655967903711, + "loss": 1.5422, + "theoretical_loss": 4.004435468886761, + "tokens_seen": 433782784 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007018495486459379, + "loss": 1.6868, + "theoretical_loss": 4.0043691843666975, + "tokens_seen": 433848320 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007018335005015045, + "loss": 1.4996, + "theoretical_loss": 4.004302912661736, + "tokens_seen": 433913856 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007018174523570713, + "loss": 1.607, + "theoretical_loss": 4.0042366537674665, + "tokens_seen": 433979392 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007018014042126379, + "loss": 1.5846, + "theoretical_loss": 4.004170407679476, + "tokens_seen": 434044928 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007017853560682047, + "loss": 1.6055, + "theoretical_loss": 4.004104174393359, + "tokens_seen": 434110464 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007017693079237713, + "loss": 1.6219, + "theoretical_loss": 4.004037953904708, + "tokens_seen": 434176000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007017532597793381, + "loss": 1.5613, + "theoretical_loss": 4.003971746209119, + "tokens_seen": 434241536 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007017372116349047, + "loss": 1.6966, + "theoretical_loss": 4.003905551302191, + "tokens_seen": 434307072 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007017211634904715, + "loss": 1.507, + "theoretical_loss": 4.0038393691795235, + "tokens_seen": 434372608 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007017051153460381, + "loss": 1.6575, + "theoretical_loss": 4.003773199836721, + "tokens_seen": 434438144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007016890672016049, + "loss": 1.5005, + "theoretical_loss": 4.003707043269387, + "tokens_seen": 434503680 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007016730190571715, + "loss": 1.7366, + "theoretical_loss": 4.003640899473128, + "tokens_seen": 434569216 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007016569709127383, + "loss": 1.6265, + "theoretical_loss": 4.0035747684435545, + "tokens_seen": 434634752 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007016409227683049, + "loss": 1.6539, + "theoretical_loss": 4.003508650176278, + "tokens_seen": 434700288 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007016248746238717, + "loss": 1.6634, + "theoretical_loss": 4.003442544666911, + "tokens_seen": 434765824 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007016088264794383, + "loss": 1.6993, + "theoretical_loss": 4.003376451911071, + "tokens_seen": 434831360 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007015927783350051, + "loss": 1.6168, + "theoretical_loss": 4.003310371904373, + "tokens_seen": 434896896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007015767301905717, + "loss": 1.6248, + "theoretical_loss": 4.00324430464244, + "tokens_seen": 434962432 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007015606820461385, + "loss": 1.6357, + "theoretical_loss": 4.003178250120892, + "tokens_seen": 435027968 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007015446339017051, + "loss": 1.7282, + "theoretical_loss": 4.003112208335355, + "tokens_seen": 435093504 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007015285857572719, + "loss": 1.5453, + "theoretical_loss": 4.003046179281455, + "tokens_seen": 435159040 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007015125376128385, + "loss": 1.6833, + "theoretical_loss": 4.00298016295482, + "tokens_seen": 435224576 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007014964894684053, + "loss": 1.4322, + "theoretical_loss": 4.002914159351082, + "tokens_seen": 435290112 + }, + { + "epoch": 0.13, + "learning_rate": 0.000701480441323972, + "loss": 1.597, + "theoretical_loss": 4.0028481684658725, + "tokens_seen": 435355648 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007014643931795387, + "loss": 1.6286, + "theoretical_loss": 4.002782190294829, + "tokens_seen": 435421184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007014483450351054, + "loss": 1.6963, + "theoretical_loss": 4.002716224833588, + "tokens_seen": 435486720 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007014322968906721, + "loss": 1.4867, + "theoretical_loss": 4.002650272077787, + "tokens_seen": 435552256 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007014162487462388, + "loss": 1.5558, + "theoretical_loss": 4.00258433202307, + "tokens_seen": 435617792 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007014002006018055, + "loss": 1.6126, + "theoretical_loss": 4.0025184046650795, + "tokens_seen": 435683328 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007013841524573722, + "loss": 1.6952, + "theoretical_loss": 4.002452489999463, + "tokens_seen": 435748864 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.9932430386543274, + "objective/train/avg_token_score": 0.006756909657269716, + "objective/train/docs_used": 164592, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5195950269699097, + "objective/train/original_loss": 1.5316119194030762, + "objective/train/theoretical_loss": 4.002386588021867, + "objective/train/tokens_used": 456274400, + "objective/train/unlikelihood_loss": 0.013415131717920303, + "theoretical_loss": 4.002386588021867, + "tokens_seen": 435814400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007013681043129389, + "loss": 1.6376, + "theoretical_loss": 4.002386588021867, + "tokens_seen": 435814400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007013520561685056, + "loss": 1.5511, + "theoretical_loss": 4.002320698727942, + "tokens_seen": 435879936 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007013360080240723, + "loss": 1.584, + "theoretical_loss": 4.002254822113341, + "tokens_seen": 435945472 + }, + { + "epoch": 0.13, + "learning_rate": 0.000701319959879639, + "loss": 1.6416, + "theoretical_loss": 4.0021889581737184, + "tokens_seen": 436011008 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007013039117352057, + "loss": 1.6716, + "theoretical_loss": 4.00212310690473, + "tokens_seen": 436076544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007012878635907723, + "loss": 1.5335, + "theoretical_loss": 4.0020572683020355, + "tokens_seen": 436142080 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007012718154463391, + "loss": 1.624, + "theoretical_loss": 4.0019914423612954, + "tokens_seen": 436207616 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007012557673019057, + "loss": 1.5565, + "theoretical_loss": 4.0019256290781735, + "tokens_seen": 436273152 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007012397191574725, + "loss": 1.6612, + "theoretical_loss": 4.001859828448334, + "tokens_seen": 436338688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007012236710130391, + "loss": 1.591, + "theoretical_loss": 4.001794040467446, + "tokens_seen": 436404224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007012076228686059, + "loss": 1.5931, + "theoretical_loss": 4.0017282651311765, + "tokens_seen": 436469760 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007011915747241725, + "loss": 1.6991, + "theoretical_loss": 4.001662502435199, + "tokens_seen": 436535296 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007011755265797393, + "loss": 1.456, + "theoretical_loss": 4.001596752375186, + "tokens_seen": 436600832 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007011594784353059, + "loss": 1.5804, + "theoretical_loss": 4.001531014946815, + "tokens_seen": 436666368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007011434302908726, + "loss": 1.6424, + "theoretical_loss": 4.001465290145763, + "tokens_seen": 436731904 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007011273821464393, + "loss": 1.6615, + "theoretical_loss": 4.00139957796771, + "tokens_seen": 436797440 + }, + { + "epoch": 0.13, + "learning_rate": 0.000701111334002006, + "loss": 1.7086, + "theoretical_loss": 4.001333878408339, + "tokens_seen": 436862976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007010952858575727, + "loss": 1.5177, + "theoretical_loss": 4.001268191463334, + "tokens_seen": 436928512 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007010792377131394, + "loss": 1.5185, + "theoretical_loss": 4.00120251712838, + "tokens_seen": 436994048 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007010631895687061, + "loss": 1.6498, + "theoretical_loss": 4.001136855399168, + "tokens_seen": 437059584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007010471414242728, + "loss": 1.6528, + "theoretical_loss": 4.001071206271388, + "tokens_seen": 437125120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007010310932798395, + "loss": 1.601, + "theoretical_loss": 4.001005569740732, + "tokens_seen": 437190656 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007010150451354062, + "loss": 1.4693, + "theoretical_loss": 4.000939945802896, + "tokens_seen": 437256192 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700998996990973, + "loss": 1.5316, + "theoretical_loss": 4.000874334453577, + "tokens_seen": 437321728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007009829488465396, + "loss": 1.5593, + "theoretical_loss": 4.000808735688474, + "tokens_seen": 437387264 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007009669007021063, + "loss": 1.6285, + "theoretical_loss": 4.000743149503288, + "tokens_seen": 437452800 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700950852557673, + "loss": 1.5931, + "theoretical_loss": 4.000677575893723, + "tokens_seen": 437518336 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007009348044132397, + "loss": 1.4324, + "theoretical_loss": 4.000612014855484, + "tokens_seen": 437583872 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007009187562688064, + "loss": 1.7216, + "theoretical_loss": 4.00054646638428, + "tokens_seen": 437649408 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007009027081243731, + "loss": 1.5321, + "theoretical_loss": 4.000480930475819, + "tokens_seen": 437714944 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007008866599799398, + "loss": 1.648, + "theoretical_loss": 4.000415407125814, + "tokens_seen": 437780480 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007008706118355066, + "loss": 1.6639, + "theoretical_loss": 4.000349896329979, + "tokens_seen": 437846016 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007008545636910732, + "loss": 1.6362, + "theoretical_loss": 4.0002843980840295, + "tokens_seen": 437911552 + }, + { + "epoch": 0.13, + "learning_rate": 0.00070083851554664, + "loss": 1.5794, + "theoretical_loss": 4.0002189123836835, + "tokens_seen": 437977088 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007008224674022066, + "loss": 1.5861, + "theoretical_loss": 4.000153439224662, + "tokens_seen": 438042624 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007008064192577734, + "loss": 1.4523, + "theoretical_loss": 4.0000879786026875, + "tokens_seen": 438108160 + }, + { + "epoch": 0.13, + "learning_rate": 0.00070079037111334, + "loss": 1.6, + "theoretical_loss": 4.000022530513484, + "tokens_seen": 438173696 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007007743229689068, + "loss": 1.6346, + "theoretical_loss": 3.999957094952779, + "tokens_seen": 438239232 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007007582748244734, + "loss": 1.628, + "theoretical_loss": 3.9998916719163002, + "tokens_seen": 438304768 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007007422266800402, + "loss": 1.5837, + "theoretical_loss": 3.999826261399778, + "tokens_seen": 438370304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007007261785356068, + "loss": 1.4818, + "theoretical_loss": 3.9997608633989463, + "tokens_seen": 438435840 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007007101303911736, + "loss": 1.6041, + "theoretical_loss": 3.9996954779095395, + "tokens_seen": 438501376 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007006940822467402, + "loss": 1.6485, + "theoretical_loss": 3.999630104927295, + "tokens_seen": 438566912 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700678034102307, + "loss": 1.6403, + "theoretical_loss": 3.999564744447952, + "tokens_seen": 438632448 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007006619859578736, + "loss": 1.6877, + "theoretical_loss": 3.9994993964672516, + "tokens_seen": 438697984 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007006459378134404, + "loss": 1.5782, + "theoretical_loss": 3.9994340609809367, + "tokens_seen": 438763520 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700629889669007, + "loss": 1.6181, + "theoretical_loss": 3.999368737984754, + "tokens_seen": 438829056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007006138415245738, + "loss": 1.5312, + "theoretical_loss": 3.999303427474449, + "tokens_seen": 438894592 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007005977933801404, + "loss": 1.6104, + "theoretical_loss": 3.9992381294457733, + "tokens_seen": 438960128 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007005817452357072, + "loss": 1.5725, + "theoretical_loss": 3.9991728438944767, + "tokens_seen": 439025664 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.9903278350830078, + "objective/train/avg_token_score": 0.009672233834862709, + "objective/train/docs_used": 165818, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5900578498840332, + "objective/train/original_loss": 1.604996919631958, + "objective/train/theoretical_loss": 3.9991075708163146, + "objective/train/tokens_used": 459551200, + "objective/train/unlikelihood_loss": 0.025144649669528008, + "theoretical_loss": 3.9991075708163146, + "tokens_seen": 439091200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007005656970912738, + "loss": 1.6062, + "theoretical_loss": 3.9991075708163146, + "tokens_seen": 439091200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007005496489468406, + "loss": 1.6702, + "theoretical_loss": 3.9990423102070416, + "tokens_seen": 439156736 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007005336008024072, + "loss": 1.6261, + "theoretical_loss": 3.9989770620624165, + "tokens_seen": 439222272 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700517552657974, + "loss": 1.7226, + "theoretical_loss": 3.9989118263781984, + "tokens_seen": 439287808 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007005015045135406, + "loss": 1.6119, + "theoretical_loss": 3.99884660315015, + "tokens_seen": 439353344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007004854563691074, + "loss": 1.686, + "theoretical_loss": 3.9987813923740356, + "tokens_seen": 439418880 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700469408224674, + "loss": 1.569, + "theoretical_loss": 3.9987161940456204, + "tokens_seen": 439484416 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007004533600802408, + "loss": 1.5419, + "theoretical_loss": 3.9986510081606736, + "tokens_seen": 439549952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007004373119358074, + "loss": 1.6477, + "theoretical_loss": 3.998585834714965, + "tokens_seen": 439615488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007004212637913742, + "loss": 1.7114, + "theoretical_loss": 3.9985206737042676, + "tokens_seen": 439681024 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007004052156469408, + "loss": 1.6272, + "theoretical_loss": 3.998455525124355, + "tokens_seen": 439746560 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007003891675025076, + "loss": 1.5818, + "theoretical_loss": 3.9983903889710044, + "tokens_seen": 439812096 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007003731193580742, + "loss": 1.6123, + "theoretical_loss": 3.998325265239994, + "tokens_seen": 439877632 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700357071213641, + "loss": 1.5999, + "theoretical_loss": 3.998260153927105, + "tokens_seen": 439943168 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007003410230692076, + "loss": 1.5329, + "theoretical_loss": 3.998195055028119, + "tokens_seen": 440008704 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007003249749247744, + "loss": 1.5762, + "theoretical_loss": 3.998129968538822, + "tokens_seen": 440074240 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700308926780341, + "loss": 1.5824, + "theoretical_loss": 3.998064894455, + "tokens_seen": 440139776 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007002928786359078, + "loss": 1.6441, + "theoretical_loss": 3.997999832772443, + "tokens_seen": 440205312 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007002768304914745, + "loss": 1.6, + "theoretical_loss": 3.997934783486941, + "tokens_seen": 440270848 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007002607823470412, + "loss": 1.5766, + "theoretical_loss": 3.997869746594287, + "tokens_seen": 440336384 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007002447342026079, + "loss": 1.6637, + "theoretical_loss": 3.9978047220902764, + "tokens_seen": 440401920 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007002286860581746, + "loss": 1.6036, + "theoretical_loss": 3.997739709970706, + "tokens_seen": 440467456 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007002126379137413, + "loss": 1.5132, + "theoretical_loss": 3.997674710231375, + "tokens_seen": 440532992 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700196589769308, + "loss": 1.7316, + "theoretical_loss": 3.997609722868085, + "tokens_seen": 440598528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007001805416248747, + "loss": 1.5027, + "theoretical_loss": 3.9975447478766393, + "tokens_seen": 440664064 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007001644934804414, + "loss": 1.5908, + "theoretical_loss": 3.997479785252842, + "tokens_seen": 440729600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007001484453360081, + "loss": 1.6435, + "theoretical_loss": 3.997414834992502, + "tokens_seen": 440795136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007001323971915748, + "loss": 1.5649, + "theoretical_loss": 3.9973498970914285, + "tokens_seen": 440860672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007001163490471415, + "loss": 1.6216, + "theoretical_loss": 3.9972849715454317, + "tokens_seen": 440926208 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007001003009027082, + "loss": 1.551, + "theoretical_loss": 3.9972200583503263, + "tokens_seen": 440991744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007000842527582749, + "loss": 1.6208, + "theoretical_loss": 3.9971551575019273, + "tokens_seen": 441057280 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007000682046138416, + "loss": 1.6762, + "theoretical_loss": 3.997090268996052, + "tokens_seen": 441122816 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007000521564694083, + "loss": 1.5524, + "theoretical_loss": 3.9970253928285215, + "tokens_seen": 441188352 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700036108324975, + "loss": 1.576, + "theoretical_loss": 3.996960528995155, + "tokens_seen": 441253888 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007000200601805417, + "loss": 1.6954, + "theoretical_loss": 3.996895677491778, + "tokens_seen": 441319424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007000040120361084, + "loss": 1.6025, + "theoretical_loss": 3.996830838314216, + "tokens_seen": 441384960 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006999879638916751, + "loss": 1.5282, + "theoretical_loss": 3.996766011458296, + "tokens_seen": 441450496 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006999719157472418, + "loss": 1.5898, + "theoretical_loss": 3.9967011969198483, + "tokens_seen": 441516032 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006999558676028085, + "loss": 1.5136, + "theoretical_loss": 3.996636394694705, + "tokens_seen": 441581568 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006999398194583752, + "loss": 1.5707, + "theoretical_loss": 3.9965716047786985, + "tokens_seen": 441647104 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006999237713139419, + "loss": 1.3797, + "theoretical_loss": 3.996506827167667, + "tokens_seen": 441712640 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006999077231695086, + "loss": 1.6302, + "theoretical_loss": 3.9964420618574454, + "tokens_seen": 441778176 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006998916750250752, + "loss": 1.6492, + "theoretical_loss": 3.9963773088438765, + "tokens_seen": 441843712 + }, + { + "epoch": 0.13, + "learning_rate": 0.000699875626880642, + "loss": 1.5843, + "theoretical_loss": 3.996312568122801, + "tokens_seen": 441909248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006998595787362086, + "loss": 1.5605, + "theoretical_loss": 3.9962478396900627, + "tokens_seen": 441974784 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006998435305917754, + "loss": 1.5552, + "theoretical_loss": 3.996183123541507, + "tokens_seen": 442040320 + }, + { + "epoch": 0.13, + "learning_rate": 0.000699827482447342, + "loss": 1.758, + "theoretical_loss": 3.9961184196729835, + "tokens_seen": 442105856 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006998114343029088, + "loss": 1.5094, + "theoretical_loss": 3.9960537280803408, + "tokens_seen": 442171392 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006997953861584754, + "loss": 1.5631, + "theoretical_loss": 3.995989048759432, + "tokens_seen": 442236928 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006997793380140422, + "loss": 1.6191, + "theoretical_loss": 3.99592438170611, + "tokens_seen": 442302464 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.9943990111351013, + "objective/train/avg_token_score": 0.005600954405963421, + "objective/train/docs_used": 166951, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5607874393463135, + "objective/train/original_loss": 1.569145917892456, + "objective/train/theoretical_loss": 3.9958597269162315, + "objective/train/tokens_used": 462828000, + "objective/train/unlikelihood_loss": 0.015102826058864594, + "theoretical_loss": 3.9958597269162315, + "tokens_seen": 442368000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006997632898696088, + "loss": 1.6249, + "theoretical_loss": 3.9958597269162315, + "tokens_seen": 442368000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006997472417251755, + "loss": 1.4905, + "theoretical_loss": 3.9957950843856542, + "tokens_seen": 442433536 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006997311935807422, + "loss": 1.5966, + "theoretical_loss": 3.9957304541102396, + "tokens_seen": 442499072 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006997151454363089, + "loss": 1.5776, + "theoretical_loss": 3.995665836085848, + "tokens_seen": 442564608 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006996990972918756, + "loss": 1.4728, + "theoretical_loss": 3.9956012303083446, + "tokens_seen": 442630144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006996830491474423, + "loss": 1.6246, + "theoretical_loss": 3.9955366367735947, + "tokens_seen": 442695680 + }, + { + "epoch": 0.13, + "learning_rate": 0.000699667001003009, + "loss": 1.505, + "theoretical_loss": 3.995472055477467, + "tokens_seen": 442761216 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006996509528585757, + "loss": 1.6969, + "theoretical_loss": 3.9954074864158313, + "tokens_seen": 442826752 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006996349047141424, + "loss": 1.5316, + "theoretical_loss": 3.9953429295845604, + "tokens_seen": 442892288 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006996188565697091, + "loss": 1.5399, + "theoretical_loss": 3.9952783849795277, + "tokens_seen": 442957824 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006996028084252758, + "loss": 1.4212, + "theoretical_loss": 3.99521385259661, + "tokens_seen": 443023360 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006995867602808425, + "loss": 1.7438, + "theoretical_loss": 3.9951493324316845, + "tokens_seen": 443088896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006995707121364092, + "loss": 1.4583, + "theoretical_loss": 3.9950848244806316, + "tokens_seen": 443154432 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006995546639919759, + "loss": 1.6082, + "theoretical_loss": 3.9950203287393347, + "tokens_seen": 443219968 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006995386158475426, + "loss": 1.4986, + "theoretical_loss": 3.994955845203676, + "tokens_seen": 443285504 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006995225677031093, + "loss": 1.6655, + "theoretical_loss": 3.994891373869543, + "tokens_seen": 443351040 + }, + { + "epoch": 0.13, + "learning_rate": 0.000699506519558676, + "loss": 1.5632, + "theoretical_loss": 3.9948269147328235, + "tokens_seen": 443416576 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006994904714142427, + "loss": 1.6116, + "theoretical_loss": 3.994762467789407, + "tokens_seen": 443482112 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006994744232698095, + "loss": 1.6825, + "theoretical_loss": 3.9946980330351862, + "tokens_seen": 443547648 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006994583751253761, + "loss": 1.5867, + "theoretical_loss": 3.994633610466056, + "tokens_seen": 443613184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006994423269809429, + "loss": 1.5611, + "theoretical_loss": 3.994569200077911, + "tokens_seen": 443678720 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006994262788365095, + "loss": 1.588, + "theoretical_loss": 3.9945048018666496, + "tokens_seen": 443744256 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006994102306920763, + "loss": 1.6109, + "theoretical_loss": 3.9944404158281723, + "tokens_seen": 443809792 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006993941825476429, + "loss": 1.542, + "theoretical_loss": 3.994376041958381, + "tokens_seen": 443875328 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006993781344032097, + "loss": 1.5807, + "theoretical_loss": 3.9943116802531797, + "tokens_seen": 443940864 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006993620862587763, + "loss": 1.533, + "theoretical_loss": 3.9942473307084745, + "tokens_seen": 444006400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006993460381143431, + "loss": 1.5705, + "theoretical_loss": 3.9941829933201736, + "tokens_seen": 444071936 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006993299899699097, + "loss": 1.6521, + "theoretical_loss": 3.9941186680841865, + "tokens_seen": 444137472 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006993139418254765, + "loss": 1.6693, + "theoretical_loss": 3.9940543549964254, + "tokens_seen": 444203008 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006992978936810431, + "loss": 1.468, + "theoretical_loss": 3.993990054052804, + "tokens_seen": 444268544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006992818455366099, + "loss": 1.4905, + "theoretical_loss": 3.993925765249239, + "tokens_seen": 444334080 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006992657973921765, + "loss": 1.5942, + "theoretical_loss": 3.993861488581647, + "tokens_seen": 444399616 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006992497492477433, + "loss": 1.7, + "theoretical_loss": 3.9937972240459487, + "tokens_seen": 444465152 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006992337011033099, + "loss": 1.572, + "theoretical_loss": 3.9937329716380665, + "tokens_seen": 444530688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006992176529588767, + "loss": 1.6234, + "theoretical_loss": 3.9936687313539228, + "tokens_seen": 444596224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006992016048144433, + "loss": 1.5463, + "theoretical_loss": 3.9936045031894443, + "tokens_seen": 444661760 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006991855566700101, + "loss": 1.5695, + "theoretical_loss": 3.9935402871405588, + "tokens_seen": 444727296 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006991695085255767, + "loss": 1.5578, + "theoretical_loss": 3.9934760832031957, + "tokens_seen": 444792832 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006991534603811435, + "loss": 1.6628, + "theoretical_loss": 3.9934118913732863, + "tokens_seen": 444858368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006991374122367101, + "loss": 1.6478, + "theoretical_loss": 3.9933477116467646, + "tokens_seen": 444923904 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006991213640922769, + "loss": 1.706, + "theoretical_loss": 3.993283544019567, + "tokens_seen": 444989440 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006991053159478435, + "loss": 1.5998, + "theoretical_loss": 3.9932193884876304, + "tokens_seen": 445054976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006990892678034103, + "loss": 1.5765, + "theoretical_loss": 3.9931552450468937, + "tokens_seen": 445120512 + }, + { + "epoch": 0.13, + "learning_rate": 0.000699073219658977, + "loss": 1.5676, + "theoretical_loss": 3.9930911136933, + "tokens_seen": 445186048 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006990571715145437, + "loss": 1.5496, + "theoretical_loss": 3.9930269944227907, + "tokens_seen": 445251584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006990411233701104, + "loss": 1.6302, + "theoretical_loss": 3.9929628872313128, + "tokens_seen": 445317120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006990250752256771, + "loss": 1.5926, + "theoretical_loss": 3.9928987921148127, + "tokens_seen": 445382656 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006990090270812438, + "loss": 1.5524, + "theoretical_loss": 3.992834709069241, + "tokens_seen": 445448192 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006989929789368105, + "loss": 1.5682, + "theoretical_loss": 3.9927706380905477, + "tokens_seen": 445513728 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006989769307923772, + "loss": 1.5766, + "theoretical_loss": 3.992706579174686, + "tokens_seen": 445579264 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.9929441809654236, + "objective/train/avg_token_score": 0.007055808790028095, + "objective/train/docs_used": 168097, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.7544177770614624, + "objective/train/original_loss": 1.769158124923706, + "objective/train/theoretical_loss": 3.9926425323176122, + "objective/train/tokens_used": 466104800, + "objective/train/unlikelihood_loss": 0.013834881596267223, + "theoretical_loss": 3.9926425323176122, + "tokens_seen": 445644800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006989608826479439, + "loss": 1.6732, + "theoretical_loss": 3.9926425323176122, + "tokens_seen": 445644800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006989448345035106, + "loss": 1.5355, + "theoretical_loss": 3.992578497515283, + "tokens_seen": 445710336 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006989287863590773, + "loss": 1.6352, + "theoretical_loss": 3.9925144747636576, + "tokens_seen": 445775872 + }, + { + "epoch": 0.14, + "learning_rate": 0.000698912738214644, + "loss": 1.5205, + "theoretical_loss": 3.992450464058696, + "tokens_seen": 445841408 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006988966900702107, + "loss": 1.4918, + "theoretical_loss": 3.9923864653963625, + "tokens_seen": 445906944 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006988806419257774, + "loss": 1.5638, + "theoretical_loss": 3.9923224787726213, + "tokens_seen": 445972480 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006988645937813441, + "loss": 1.5448, + "theoretical_loss": 3.9922585041834395, + "tokens_seen": 446038016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006988485456369108, + "loss": 1.5388, + "theoretical_loss": 3.9921945416247855, + "tokens_seen": 446103552 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006988324974924775, + "loss": 1.5284, + "theoretical_loss": 3.9921305910926312, + "tokens_seen": 446169088 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006988164493480442, + "loss": 1.5312, + "theoretical_loss": 3.992066652582948, + "tokens_seen": 446234624 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006988004012036109, + "loss": 1.596, + "theoretical_loss": 3.9920027260917115, + "tokens_seen": 446300160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006987843530591776, + "loss": 1.6521, + "theoretical_loss": 3.9919388116148977, + "tokens_seen": 446365696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006987683049147443, + "loss": 1.5437, + "theoretical_loss": 3.991874909148485, + "tokens_seen": 446431232 + }, + { + "epoch": 0.14, + "learning_rate": 0.000698752256770311, + "loss": 1.5601, + "theoretical_loss": 3.9918110186884546, + "tokens_seen": 446496768 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006987362086258777, + "loss": 1.6209, + "theoretical_loss": 3.991747140230788, + "tokens_seen": 446562304 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006987201604814444, + "loss": 1.7009, + "theoretical_loss": 3.9916832737714705, + "tokens_seen": 446627840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006987041123370111, + "loss": 1.6408, + "theoretical_loss": 3.9916194193064873, + "tokens_seen": 446693376 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006986880641925778, + "loss": 1.6097, + "theoretical_loss": 3.9915555768318276, + "tokens_seen": 446758912 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006986720160481445, + "loss": 1.5557, + "theoretical_loss": 3.9914917463434807, + "tokens_seen": 446824448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006986559679037112, + "loss": 1.5998, + "theoretical_loss": 3.9914279278374387, + "tokens_seen": 446889984 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006986399197592779, + "loss": 1.5903, + "theoretical_loss": 3.9913641213096964, + "tokens_seen": 446955520 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006986238716148446, + "loss": 1.5117, + "theoretical_loss": 3.9913003267562486, + "tokens_seen": 447021056 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006986078234704113, + "loss": 1.5275, + "theoretical_loss": 3.9912365441730944, + "tokens_seen": 447086592 + }, + { + "epoch": 0.14, + "learning_rate": 0.000698591775325978, + "loss": 1.6995, + "theoretical_loss": 3.9911727735562326, + "tokens_seen": 447152128 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006985757271815447, + "loss": 1.4103, + "theoretical_loss": 3.9911090149016646, + "tokens_seen": 447217664 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006985596790371114, + "loss": 1.6562, + "theoretical_loss": 3.9910452682053954, + "tokens_seen": 447283200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006985436308926781, + "loss": 1.5274, + "theoretical_loss": 3.9909815334634287, + "tokens_seen": 447348736 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006985275827482448, + "loss": 1.5018, + "theoretical_loss": 3.990917810671773, + "tokens_seen": 447414272 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006985115346038115, + "loss": 1.5431, + "theoretical_loss": 3.9908540998264375, + "tokens_seen": 447479808 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006984954864593781, + "loss": 1.6901, + "theoretical_loss": 3.9907904009234336, + "tokens_seen": 447545344 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006984794383149449, + "loss": 1.5602, + "theoretical_loss": 3.9907267139587743, + "tokens_seen": 447610880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006984633901705115, + "loss": 1.464, + "theoretical_loss": 3.9906630389284747, + "tokens_seen": 447676416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006984473420260783, + "loss": 1.5141, + "theoretical_loss": 3.990599375828552, + "tokens_seen": 447741952 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006984312938816449, + "loss": 1.5394, + "theoretical_loss": 3.9905357246550253, + "tokens_seen": 447807488 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006984152457372117, + "loss": 1.495, + "theoretical_loss": 3.9904720854039137, + "tokens_seen": 447873024 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006983991975927783, + "loss": 1.6412, + "theoretical_loss": 3.9904084580712427, + "tokens_seen": 447938560 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006983831494483451, + "loss": 1.5246, + "theoretical_loss": 3.9903448426530352, + "tokens_seen": 448004096 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006983671013039117, + "loss": 1.6873, + "theoretical_loss": 3.990281239145318, + "tokens_seen": 448069632 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006983510531594784, + "loss": 1.5705, + "theoretical_loss": 3.99021764754412, + "tokens_seen": 448135168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006983350050150451, + "loss": 1.5915, + "theoretical_loss": 3.990154067845472, + "tokens_seen": 448200704 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006983189568706118, + "loss": 1.6008, + "theoretical_loss": 3.990090500045404, + "tokens_seen": 448266240 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006983029087261785, + "loss": 1.6154, + "theoretical_loss": 3.990026944139953, + "tokens_seen": 448331776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006982868605817452, + "loss": 1.4744, + "theoretical_loss": 3.989963400125153, + "tokens_seen": 448397312 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006982708124373119, + "loss": 1.5349, + "theoretical_loss": 3.9898998679970434, + "tokens_seen": 448462848 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006982547642928786, + "loss": 1.5412, + "theoretical_loss": 3.9898363477516634, + "tokens_seen": 448528384 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006982387161484453, + "loss": 1.6222, + "theoretical_loss": 3.9897728393850547, + "tokens_seen": 448593920 + }, + { + "epoch": 0.14, + "learning_rate": 0.000698222668004012, + "loss": 1.4951, + "theoretical_loss": 3.9897093428932613, + "tokens_seen": 448659456 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006982066198595787, + "loss": 1.4914, + "theoretical_loss": 3.989645858272328, + "tokens_seen": 448724992 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006981905717151454, + "loss": 1.5286, + "theoretical_loss": 3.989582385518304, + "tokens_seen": 448790528 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006981745235707121, + "loss": 1.5551, + "theoretical_loss": 3.989518924627237, + "tokens_seen": 448856064 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.9919764995574951, + "objective/train/avg_token_score": 0.008023509755730629, + "objective/train/docs_used": 169309, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6579811573028564, + "objective/train/original_loss": 1.671844720840454, + "objective/train/theoretical_loss": 3.9894554755951788, + "objective/train/tokens_used": 469381600, + "objective/train/unlikelihood_loss": 0.01615428552031517, + "theoretical_loss": 3.9894554755951788, + "tokens_seen": 448921600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006981584754262788, + "loss": 1.5954, + "theoretical_loss": 3.9894554755951788, + "tokens_seen": 448921600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006981424272818456, + "loss": 1.5791, + "theoretical_loss": 3.989392038418182, + "tokens_seen": 448987136 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006981263791374122, + "loss": 1.4398, + "theoretical_loss": 3.9893286130923027, + "tokens_seen": 449052672 + }, + { + "epoch": 0.14, + "learning_rate": 0.000698110330992979, + "loss": 1.6573, + "theoretical_loss": 3.989265199613597, + "tokens_seen": 449118208 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006980942828485456, + "loss": 1.5298, + "theoretical_loss": 3.989201797978124, + "tokens_seen": 449183744 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006980782347041124, + "loss": 1.4953, + "theoretical_loss": 3.989138408181944, + "tokens_seen": 449249280 + }, + { + "epoch": 0.14, + "learning_rate": 0.000698062186559679, + "loss": 1.5741, + "theoretical_loss": 3.9890750302211204, + "tokens_seen": 449314816 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006980461384152458, + "loss": 1.571, + "theoretical_loss": 3.9890116640917173, + "tokens_seen": 449380352 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006980300902708124, + "loss": 1.6509, + "theoretical_loss": 3.9889483097898006, + "tokens_seen": 449445888 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006980140421263792, + "loss": 1.651, + "theoretical_loss": 3.9888849673114386, + "tokens_seen": 449511424 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006979979939819458, + "loss": 1.6213, + "theoretical_loss": 3.988821636652702, + "tokens_seen": 449576960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006979819458375126, + "loss": 1.5008, + "theoretical_loss": 3.988758317809662, + "tokens_seen": 449642496 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006979658976930792, + "loss": 1.5662, + "theoretical_loss": 3.988695010778393, + "tokens_seen": 449708032 + }, + { + "epoch": 0.14, + "learning_rate": 0.000697949849548646, + "loss": 1.6628, + "theoretical_loss": 3.9886317155549706, + "tokens_seen": 449773568 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006979338014042126, + "loss": 1.5326, + "theoretical_loss": 3.988568432135472, + "tokens_seen": 449839104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006979177532597794, + "loss": 1.5789, + "theoretical_loss": 3.988505160515978, + "tokens_seen": 449904640 + }, + { + "epoch": 0.14, + "learning_rate": 0.000697901705115346, + "loss": 1.6338, + "theoretical_loss": 3.9884419006925684, + "tokens_seen": 449970176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006978856569709128, + "loss": 1.5375, + "theoretical_loss": 3.9883786526613267, + "tokens_seen": 450035712 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006978696088264795, + "loss": 1.5532, + "theoretical_loss": 3.988315416418339, + "tokens_seen": 450101248 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006978535606820462, + "loss": 1.5755, + "theoretical_loss": 3.988252191959692, + "tokens_seen": 450166784 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006978375125376129, + "loss": 1.6749, + "theoretical_loss": 3.9881889792814733, + "tokens_seen": 450232320 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006978214643931796, + "loss": 1.6374, + "theoretical_loss": 3.9881257783797746, + "tokens_seen": 450297856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006978054162487463, + "loss": 1.5656, + "theoretical_loss": 3.9880625892506885, + "tokens_seen": 450363392 + }, + { + "epoch": 0.14, + "learning_rate": 0.000697789368104313, + "loss": 1.598, + "theoretical_loss": 3.9879994118903093, + "tokens_seen": 450428928 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006977733199598797, + "loss": 1.6214, + "theoretical_loss": 3.9879362462947334, + "tokens_seen": 450494464 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006977572718154464, + "loss": 1.6437, + "theoretical_loss": 3.9878730924600583, + "tokens_seen": 450560000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006977412236710131, + "loss": 1.6119, + "theoretical_loss": 3.9878099503823847, + "tokens_seen": 450625536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006977251755265798, + "loss": 1.5809, + "theoretical_loss": 3.9877468200578146, + "tokens_seen": 450691072 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006977091273821465, + "loss": 1.6003, + "theoretical_loss": 3.9876837014824513, + "tokens_seen": 450756608 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006976930792377132, + "loss": 1.5961, + "theoretical_loss": 3.9876205946524, + "tokens_seen": 450822144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006976770310932799, + "loss": 1.5193, + "theoretical_loss": 3.9875574995637697, + "tokens_seen": 450887680 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006976609829488466, + "loss": 1.5805, + "theoretical_loss": 3.987494416212668, + "tokens_seen": 450953216 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006976449348044133, + "loss": 1.5811, + "theoretical_loss": 3.987431344595207, + "tokens_seen": 451018752 + }, + { + "epoch": 0.14, + "learning_rate": 0.00069762888665998, + "loss": 1.4353, + "theoretical_loss": 3.9873682847074994, + "tokens_seen": 451084288 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006976128385155467, + "loss": 1.5658, + "theoretical_loss": 3.9873052365456605, + "tokens_seen": 451149824 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006975967903711134, + "loss": 1.4333, + "theoretical_loss": 3.9872422001058063, + "tokens_seen": 451215360 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006975807422266801, + "loss": 1.5367, + "theoretical_loss": 3.987179175384056, + "tokens_seen": 451280896 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006975646940822468, + "loss": 1.5305, + "theoretical_loss": 3.9871161623765294, + "tokens_seen": 451346432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006975486459378135, + "loss": 1.6007, + "theoretical_loss": 3.9870531610793494, + "tokens_seen": 451411968 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006975325977933802, + "loss": 1.5883, + "theoretical_loss": 3.9869901714886398, + "tokens_seen": 451477504 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006975165496489469, + "loss": 1.512, + "theoretical_loss": 3.9869271936005264, + "tokens_seen": 451543040 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006975005015045136, + "loss": 1.6011, + "theoretical_loss": 3.9868642274111377, + "tokens_seen": 451608576 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006974844533600803, + "loss": 1.5661, + "theoretical_loss": 3.986801272916603, + "tokens_seen": 451674112 + }, + { + "epoch": 0.14, + "learning_rate": 0.000697468405215647, + "loss": 1.5693, + "theoretical_loss": 3.9867383301130532, + "tokens_seen": 451739648 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006974523570712137, + "loss": 1.5633, + "theoretical_loss": 3.9866753989966224, + "tokens_seen": 451805184 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006974363089267804, + "loss": 1.7019, + "theoretical_loss": 3.986612479563445, + "tokens_seen": 451870720 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006974202607823471, + "loss": 1.6403, + "theoretical_loss": 3.9865495718096584, + "tokens_seen": 451936256 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006974042126379138, + "loss": 1.5549, + "theoretical_loss": 3.9864866757314017, + "tokens_seen": 452001792 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006973881644934805, + "loss": 1.5957, + "theoretical_loss": 3.9864237913248157, + "tokens_seen": 452067328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006973721163490472, + "loss": 1.5987, + "theoretical_loss": 3.9863609185860422, + "tokens_seen": 452132864 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.9939202666282654, + "objective/train/avg_token_score": 0.006079712882637978, + "objective/train/docs_used": 170598, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6768851280212402, + "objective/train/original_loss": 1.6874475479125977, + "objective/train/theoretical_loss": 3.9862980575112257, + "objective/train/tokens_used": 472658400, + "objective/train/unlikelihood_loss": 0.014337508007884026, + "theoretical_loss": 3.9862980575112257, + "tokens_seen": 452198400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006973560682046139, + "loss": 1.6214, + "theoretical_loss": 3.9862980575112257, + "tokens_seen": 452198400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006973400200601806, + "loss": 1.635, + "theoretical_loss": 3.986235208096513, + "tokens_seen": 452263936 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006973239719157473, + "loss": 1.6341, + "theoretical_loss": 3.9861723703380516, + "tokens_seen": 452329472 + }, + { + "epoch": 0.14, + "learning_rate": 0.000697307923771314, + "loss": 1.641, + "theoretical_loss": 3.986109544231992, + "tokens_seen": 452395008 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006972918756268807, + "loss": 1.6116, + "theoretical_loss": 3.9860467297744844, + "tokens_seen": 452460544 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006972758274824474, + "loss": 1.5413, + "theoretical_loss": 3.985983926961684, + "tokens_seen": 452526080 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006972597793380141, + "loss": 1.5903, + "theoretical_loss": 3.985921135789745, + "tokens_seen": 452591616 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006972437311935808, + "loss": 1.4969, + "theoretical_loss": 3.9858583562548247, + "tokens_seen": 452657152 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006972276830491475, + "loss": 1.6674, + "theoretical_loss": 3.9857955883530827, + "tokens_seen": 452722688 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006972116349047142, + "loss": 1.6343, + "theoretical_loss": 3.985732832080679, + "tokens_seen": 452788224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006971955867602809, + "loss": 1.5675, + "theoretical_loss": 3.985670087433777, + "tokens_seen": 452853760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006971795386158476, + "loss": 1.6213, + "theoretical_loss": 3.9856073544085406, + "tokens_seen": 452919296 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006971634904714143, + "loss": 1.7037, + "theoretical_loss": 3.9855446330011364, + "tokens_seen": 452984832 + }, + { + "epoch": 0.14, + "learning_rate": 0.000697147442326981, + "loss": 1.6228, + "theoretical_loss": 3.9854819232077316, + "tokens_seen": 453050368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006971313941825477, + "loss": 1.5823, + "theoretical_loss": 3.985419225024498, + "tokens_seen": 453115904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006971153460381144, + "loss": 1.6443, + "theoretical_loss": 3.985356538447605, + "tokens_seen": 453181440 + }, + { + "epoch": 0.14, + "learning_rate": 0.000697099297893681, + "loss": 1.7069, + "theoretical_loss": 3.9852938634732276, + "tokens_seen": 453246976 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006970832497492478, + "loss": 1.606, + "theoretical_loss": 3.985231200097541, + "tokens_seen": 453312512 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006970672016048144, + "loss": 1.5551, + "theoretical_loss": 3.9851685483167225, + "tokens_seen": 453378048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006970511534603812, + "loss": 1.5509, + "theoretical_loss": 3.9851059081269504, + "tokens_seen": 453443584 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006970351053159478, + "loss": 1.521, + "theoretical_loss": 3.9850432795244055, + "tokens_seen": 453509120 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006970190571715146, + "loss": 1.4895, + "theoretical_loss": 3.9849806625052713, + "tokens_seen": 453574656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006970030090270812, + "loss": 1.4981, + "theoretical_loss": 3.984918057065731, + "tokens_seen": 453640192 + }, + { + "epoch": 0.14, + "learning_rate": 0.000696986960882648, + "loss": 1.5394, + "theoretical_loss": 3.984855463201972, + "tokens_seen": 453705728 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006969709127382146, + "loss": 1.5005, + "theoretical_loss": 3.984792880910182, + "tokens_seen": 453771264 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006969548645937813, + "loss": 1.611, + "theoretical_loss": 3.98473031018655, + "tokens_seen": 453836800 + }, + { + "epoch": 0.14, + "learning_rate": 0.000696938816449348, + "loss": 1.5847, + "theoretical_loss": 3.984667751027268, + "tokens_seen": 453902336 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006969227683049147, + "loss": 1.5294, + "theoretical_loss": 3.98460520342853, + "tokens_seen": 453967872 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006969067201604814, + "loss": 1.5997, + "theoretical_loss": 3.984542667386531, + "tokens_seen": 454033408 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006968906720160481, + "loss": 1.5463, + "theoretical_loss": 3.9844801428974685, + "tokens_seen": 454098944 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006968746238716148, + "loss": 1.5804, + "theoretical_loss": 3.98441762995754, + "tokens_seen": 454164480 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006968585757271815, + "loss": 1.5601, + "theoretical_loss": 3.984355128562947, + "tokens_seen": 454230016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006968425275827482, + "loss": 1.6713, + "theoretical_loss": 3.984292638709892, + "tokens_seen": 454295552 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006968264794383149, + "loss": 1.5095, + "theoretical_loss": 3.9842301603945787, + "tokens_seen": 454361088 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006968104312938816, + "loss": 1.5661, + "theoretical_loss": 3.9841676936132133, + "tokens_seen": 454426624 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006967943831494483, + "loss": 1.5321, + "theoretical_loss": 3.984105238362005, + "tokens_seen": 454492160 + }, + { + "epoch": 0.14, + "learning_rate": 0.000696778335005015, + "loss": 1.5332, + "theoretical_loss": 3.984042794637161, + "tokens_seen": 454557696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006967622868605817, + "loss": 1.7125, + "theoretical_loss": 3.9839803624348944, + "tokens_seen": 454623232 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006967462387161485, + "loss": 1.5196, + "theoretical_loss": 3.983917941751418, + "tokens_seen": 454688768 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006967301905717151, + "loss": 1.5921, + "theoretical_loss": 3.983855532582947, + "tokens_seen": 454754304 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006967141424272819, + "loss": 1.5913, + "theoretical_loss": 3.983793134925697, + "tokens_seen": 454819840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006966980942828485, + "loss": 1.4966, + "theoretical_loss": 3.9837307487758884, + "tokens_seen": 454885376 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006966820461384153, + "loss": 1.578, + "theoretical_loss": 3.9836683741297403, + "tokens_seen": 454950912 + }, + { + "epoch": 0.14, + "learning_rate": 0.000696665997993982, + "loss": 1.5298, + "theoretical_loss": 3.9836060109834746, + "tokens_seen": 455016448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006966499498495487, + "loss": 1.5291, + "theoretical_loss": 3.983543659333316, + "tokens_seen": 455081984 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006966339017051154, + "loss": 1.6042, + "theoretical_loss": 3.9834813191754908, + "tokens_seen": 455147520 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006966178535606821, + "loss": 1.6405, + "theoretical_loss": 3.983418990506225, + "tokens_seen": 455213056 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006966018054162488, + "loss": 1.5567, + "theoretical_loss": 3.9833566733217483, + "tokens_seen": 455278592 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006965857572718155, + "loss": 1.6198, + "theoretical_loss": 3.983294367618292, + "tokens_seen": 455344128 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006965697091273822, + "loss": 1.6012, + "theoretical_loss": 3.983232073392089, + "tokens_seen": 455409664 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.9899298548698425, + "objective/train/avg_token_score": 0.010070164687931538, + "objective/train/docs_used": 171735, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6228110790252686, + "objective/train/original_loss": 1.6402380466461182, + "objective/train/theoretical_loss": 3.983169790639374, + "objective/train/tokens_used": 475935200, + "objective/train/unlikelihood_loss": 0.023413343355059624, + "theoretical_loss": 3.983169790639374, + "tokens_seen": 455475200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006965536609829489, + "loss": 1.6705, + "theoretical_loss": 3.983169790639374, + "tokens_seen": 455475200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006965376128385156, + "loss": 1.5454, + "theoretical_loss": 3.983107519356383, + "tokens_seen": 455540736 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006965215646940823, + "loss": 1.6535, + "theoretical_loss": 3.9830452595393546, + "tokens_seen": 455606272 + }, + { + "epoch": 0.14, + "learning_rate": 0.000696505516549649, + "loss": 1.5838, + "theoretical_loss": 3.9829830111845284, + "tokens_seen": 455671808 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006964894684052157, + "loss": 1.6016, + "theoretical_loss": 3.9829207742881456, + "tokens_seen": 455737344 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006964734202607824, + "loss": 1.558, + "theoretical_loss": 3.982858548846451, + "tokens_seen": 455802880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006964573721163491, + "loss": 1.6081, + "theoretical_loss": 3.982796334855689, + "tokens_seen": 455868416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006964413239719158, + "loss": 1.5507, + "theoretical_loss": 3.9827341323121064, + "tokens_seen": 455933952 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006964252758274825, + "loss": 1.6177, + "theoretical_loss": 3.9826719412119527, + "tokens_seen": 455999488 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006964092276830492, + "loss": 1.5165, + "theoretical_loss": 3.982609761551478, + "tokens_seen": 456065024 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006963931795386159, + "loss": 1.5628, + "theoretical_loss": 3.982547593326934, + "tokens_seen": 456130560 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006963771313941826, + "loss": 1.5654, + "theoretical_loss": 3.982485436534576, + "tokens_seen": 456196096 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006963610832497493, + "loss": 1.5428, + "theoretical_loss": 3.98242329117066, + "tokens_seen": 456261632 + }, + { + "epoch": 0.14, + "learning_rate": 0.000696345035105316, + "loss": 1.6234, + "theoretical_loss": 3.982361157231442, + "tokens_seen": 456327168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006963289869608827, + "loss": 1.5428, + "theoretical_loss": 3.982299034713183, + "tokens_seen": 456392704 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006963129388164494, + "loss": 1.5492, + "theoretical_loss": 3.9822369236121427, + "tokens_seen": 456458240 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006962968906720161, + "loss": 1.4657, + "theoretical_loss": 3.982174823924586, + "tokens_seen": 456523776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006962808425275828, + "loss": 1.6264, + "theoretical_loss": 3.9821127356467754, + "tokens_seen": 456589312 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006962647943831495, + "loss": 1.558, + "theoretical_loss": 3.9820506587749787, + "tokens_seen": 456654848 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006962487462387162, + "loss": 1.4469, + "theoretical_loss": 3.9819885933054637, + "tokens_seen": 456720384 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006962326980942829, + "loss": 1.5244, + "theoretical_loss": 3.9819265392345, + "tokens_seen": 456785920 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006962166499498496, + "loss": 1.5091, + "theoretical_loss": 3.9818644965583596, + "tokens_seen": 456851456 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006962006018054163, + "loss": 1.5049, + "theoretical_loss": 3.981802465273316, + "tokens_seen": 456916992 + }, + { + "epoch": 0.14, + "learning_rate": 0.000696184553660983, + "loss": 1.5397, + "theoretical_loss": 3.9817404453756446, + "tokens_seen": 456982528 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006961685055165497, + "loss": 1.6257, + "theoretical_loss": 3.981678436861622, + "tokens_seen": 457048064 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006961524573721164, + "loss": 1.5258, + "theoretical_loss": 3.981616439727527, + "tokens_seen": 457113600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006961364092276831, + "loss": 1.6251, + "theoretical_loss": 3.9815544539696397, + "tokens_seen": 457179136 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006961203610832498, + "loss": 1.6545, + "theoretical_loss": 3.981492479584243, + "tokens_seen": 457244672 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006961043129388165, + "loss": 1.5547, + "theoretical_loss": 3.9814305165676203, + "tokens_seen": 457310208 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006960882647943832, + "loss": 1.5503, + "theoretical_loss": 3.9813685649160577, + "tokens_seen": 457375744 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006960722166499499, + "loss": 1.4031, + "theoretical_loss": 3.981306624625842, + "tokens_seen": 457441280 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006960561685055166, + "loss": 1.6198, + "theoretical_loss": 3.9812446956932632, + "tokens_seen": 457506816 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006960401203610833, + "loss": 1.6835, + "theoretical_loss": 3.9811827781146123, + "tokens_seen": 457572352 + }, + { + "epoch": 0.14, + "learning_rate": 0.00069602407221665, + "loss": 1.5598, + "theoretical_loss": 3.981120871886181, + "tokens_seen": 457637888 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006960080240722167, + "loss": 1.5906, + "theoretical_loss": 3.9810589770042637, + "tokens_seen": 457703424 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006959919759277834, + "loss": 1.5664, + "theoretical_loss": 3.9809970934651577, + "tokens_seen": 457768960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006959759277833501, + "loss": 1.628, + "theoretical_loss": 3.9809352212651596, + "tokens_seen": 457834496 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006959598796389168, + "loss": 1.628, + "theoretical_loss": 3.9808733604005706, + "tokens_seen": 457900032 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006959438314944835, + "loss": 1.5665, + "theoretical_loss": 3.9808115108676905, + "tokens_seen": 457965568 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006959277833500502, + "loss": 1.6133, + "theoretical_loss": 3.9807496726628235, + "tokens_seen": 458031104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006959117352056169, + "loss": 1.4959, + "theoretical_loss": 3.9806878457822736, + "tokens_seen": 458096640 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006958956870611836, + "loss": 1.5899, + "theoretical_loss": 3.9806260302223477, + "tokens_seen": 458162176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006958796389167503, + "loss": 1.6867, + "theoretical_loss": 3.9805642259793546, + "tokens_seen": 458227712 + }, + { + "epoch": 0.14, + "learning_rate": 0.000695863590772317, + "loss": 1.7176, + "theoretical_loss": 3.9805024330496037, + "tokens_seen": 458293248 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006958475426278837, + "loss": 1.5939, + "theoretical_loss": 3.980440651429407, + "tokens_seen": 458358784 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006958314944834504, + "loss": 1.6307, + "theoretical_loss": 3.9803788811150778, + "tokens_seen": 458424320 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006958154463390171, + "loss": 1.5605, + "theoretical_loss": 3.980317122102931, + "tokens_seen": 458489856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006957993981945838, + "loss": 1.5551, + "theoretical_loss": 3.9802553743892846, + "tokens_seen": 458555392 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006957833500501505, + "loss": 1.538, + "theoretical_loss": 3.9801936379704568, + "tokens_seen": 458620928 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006957673019057172, + "loss": 1.6308, + "theoretical_loss": 3.9801319128427677, + "tokens_seen": 458686464 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.9943303465843201, + "objective/train/avg_token_score": 0.005669646896421909, + "objective/train/docs_used": 173009, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.539778470993042, + "objective/train/original_loss": 1.5484857559204102, + "objective/train/theoretical_loss": 3.9800701990025393, + "objective/train/tokens_used": 479212000, + "objective/train/unlikelihood_loss": 0.01388686802238226, + "theoretical_loss": 3.9800701990025393, + "tokens_seen": 458752000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006957512537612839, + "loss": 1.5179, + "theoretical_loss": 3.9800701990025393, + "tokens_seen": 458752000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006957352056168506, + "loss": 1.6306, + "theoretical_loss": 3.9800084964460964, + "tokens_seen": 458817536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006957191574724173, + "loss": 1.4791, + "theoretical_loss": 3.9799468051697633, + "tokens_seen": 458883072 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006957031093279839, + "loss": 1.5044, + "theoretical_loss": 3.9798851251698686, + "tokens_seen": 458948608 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006956870611835507, + "loss": 1.4599, + "theoretical_loss": 3.97982345644274, + "tokens_seen": 459014144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006956710130391173, + "loss": 1.604, + "theoretical_loss": 3.9797617989847094, + "tokens_seen": 459079680 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006956549648946841, + "loss": 1.6291, + "theoretical_loss": 3.979700152792108, + "tokens_seen": 459145216 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006956389167502507, + "loss": 1.5773, + "theoretical_loss": 3.9796385178612717, + "tokens_seen": 459210752 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006956228686058175, + "loss": 1.5376, + "theoretical_loss": 3.9795768941885346, + "tokens_seen": 459276288 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006956068204613841, + "loss": 1.5907, + "theoretical_loss": 3.9795152817702357, + "tokens_seen": 459341824 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006955907723169509, + "loss": 1.5541, + "theoretical_loss": 3.979453680602713, + "tokens_seen": 459407360 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006955747241725175, + "loss": 1.6426, + "theoretical_loss": 3.9793920906823086, + "tokens_seen": 459472896 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006955586760280842, + "loss": 1.4791, + "theoretical_loss": 3.979330512005365, + "tokens_seen": 459538432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006955426278836509, + "loss": 1.5719, + "theoretical_loss": 3.9792689445682257, + "tokens_seen": 459603968 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006955265797392176, + "loss": 1.5189, + "theoretical_loss": 3.9792073883672385, + "tokens_seen": 459669504 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006955105315947843, + "loss": 1.5804, + "theoretical_loss": 3.9791458433987503, + "tokens_seen": 459735040 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006954944834503511, + "loss": 1.6127, + "theoretical_loss": 3.97908430965911, + "tokens_seen": 459800576 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006954784353059177, + "loss": 1.6203, + "theoretical_loss": 3.979022787144671, + "tokens_seen": 459866112 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006954623871614845, + "loss": 1.6039, + "theoretical_loss": 3.9789612758517836, + "tokens_seen": 459931648 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006954463390170511, + "loss": 1.6234, + "theoretical_loss": 3.9788997757768043, + "tokens_seen": 459997184 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006954302908726179, + "loss": 1.6329, + "theoretical_loss": 3.9788382869160897, + "tokens_seen": 460062720 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006954142427281846, + "loss": 1.5921, + "theoretical_loss": 3.9787768092659963, + "tokens_seen": 460128256 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006953981945837513, + "loss": 1.5831, + "theoretical_loss": 3.978715342822885, + "tokens_seen": 460193792 + }, + { + "epoch": 0.14, + "learning_rate": 0.000695382146439318, + "loss": 1.6411, + "theoretical_loss": 3.9786538875831168, + "tokens_seen": 460259328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006953660982948847, + "loss": 1.6713, + "theoretical_loss": 3.9785924435430555, + "tokens_seen": 460324864 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006953500501504514, + "loss": 1.5334, + "theoretical_loss": 3.978531010699066, + "tokens_seen": 460390400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006953340020060181, + "loss": 1.5576, + "theoretical_loss": 3.9784695890475144, + "tokens_seen": 460455936 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006953179538615848, + "loss": 1.6051, + "theoretical_loss": 3.9784081785847687, + "tokens_seen": 460521472 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006953019057171515, + "loss": 1.5351, + "theoretical_loss": 3.9783467793071994, + "tokens_seen": 460587008 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006952858575727182, + "loss": 1.5092, + "theoretical_loss": 3.978285391211178, + "tokens_seen": 460652544 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006952698094282849, + "loss": 1.5514, + "theoretical_loss": 3.9782240142930787, + "tokens_seen": 460718080 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006952537612838516, + "loss": 1.5779, + "theoretical_loss": 3.978162648549275, + "tokens_seen": 460783616 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006952377131394183, + "loss": 1.5386, + "theoretical_loss": 3.9781012939761453, + "tokens_seen": 460849152 + }, + { + "epoch": 0.14, + "learning_rate": 0.000695221664994985, + "loss": 1.5397, + "theoretical_loss": 3.978039950570067, + "tokens_seen": 460914688 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006952056168505517, + "loss": 1.6428, + "theoretical_loss": 3.9779786183274197, + "tokens_seen": 460980224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006951895687061184, + "loss": 1.5855, + "theoretical_loss": 3.977917297244587, + "tokens_seen": 461045760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006951735205616851, + "loss": 1.6574, + "theoretical_loss": 3.9778559873179504, + "tokens_seen": 461111296 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006951574724172518, + "loss": 1.7137, + "theoretical_loss": 3.977794688543897, + "tokens_seen": 461176832 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006951414242728185, + "loss": 1.5054, + "theoretical_loss": 3.9777334009188117, + "tokens_seen": 461242368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006951253761283852, + "loss": 1.5556, + "theoretical_loss": 3.9776721244390845, + "tokens_seen": 461307904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006951093279839519, + "loss": 1.6072, + "theoretical_loss": 3.9776108591011052, + "tokens_seen": 461373440 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006950932798395186, + "loss": 1.5093, + "theoretical_loss": 3.9775496049012657, + "tokens_seen": 461438976 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006950772316950853, + "loss": 1.5019, + "theoretical_loss": 3.9774883618359596, + "tokens_seen": 461504512 + }, + { + "epoch": 0.14, + "learning_rate": 0.000695061183550652, + "loss": 1.5345, + "theoretical_loss": 3.9774271299015824, + "tokens_seen": 461570048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006950451354062187, + "loss": 1.4286, + "theoretical_loss": 3.977365909094531, + "tokens_seen": 461635584 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006950290872617854, + "loss": 1.7426, + "theoretical_loss": 3.977304699411204, + "tokens_seen": 461701120 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006950130391173521, + "loss": 1.6809, + "theoretical_loss": 3.977243500848001, + "tokens_seen": 461766656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006949969909729188, + "loss": 1.6088, + "theoretical_loss": 3.977182313401325, + "tokens_seen": 461832192 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006949809428284855, + "loss": 1.6804, + "theoretical_loss": 3.977121137067579, + "tokens_seen": 461897728 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006949648946840522, + "loss": 1.5342, + "theoretical_loss": 3.9770599718431687, + "tokens_seen": 461963264 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.992968738079071, + "objective/train/avg_token_score": 0.007031233049929142, + "objective/train/docs_used": 174193, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.7175836563110352, + "objective/train/original_loss": 1.7296380996704102, + "objective/train/theoretical_loss": 3.9769988177245015, + "objective/train/tokens_used": 482488800, + "objective/train/unlikelihood_loss": 0.015945736318826675, + "theoretical_loss": 3.9769988177245015, + "tokens_seen": 462028800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006949488465396189, + "loss": 1.6805, + "theoretical_loss": 3.9769988177245015, + "tokens_seen": 462028800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006949327983951856, + "loss": 1.6616, + "theoretical_loss": 3.976937674707985, + "tokens_seen": 462094336 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006949167502507523, + "loss": 1.7042, + "theoretical_loss": 3.9768765427900306, + "tokens_seen": 462159872 + }, + { + "epoch": 0.14, + "learning_rate": 0.000694900702106319, + "loss": 1.4418, + "theoretical_loss": 3.97681542196705, + "tokens_seen": 462225408 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006948846539618857, + "loss": 1.5401, + "theoretical_loss": 3.976754312235456, + "tokens_seen": 462290944 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006948686058174524, + "loss": 1.5625, + "theoretical_loss": 3.9766932135916653, + "tokens_seen": 462356480 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006948525576730191, + "loss": 1.6609, + "theoretical_loss": 3.9766321260320945, + "tokens_seen": 462422016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006948365095285858, + "loss": 1.567, + "theoretical_loss": 3.976571049553162, + "tokens_seen": 462487552 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006948204613841525, + "loss": 1.5816, + "theoretical_loss": 3.9765099841512885, + "tokens_seen": 462553088 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006948044132397192, + "loss": 1.5758, + "theoretical_loss": 3.976448929822895, + "tokens_seen": 462618624 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006947883650952859, + "loss": 1.4573, + "theoretical_loss": 3.9763878865644067, + "tokens_seen": 462684160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006947723169508526, + "loss": 1.5482, + "theoretical_loss": 3.9763268543722483, + "tokens_seen": 462749696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006947562688064193, + "loss": 1.5314, + "theoretical_loss": 3.976265833242846, + "tokens_seen": 462815232 + }, + { + "epoch": 0.14, + "learning_rate": 0.000694740220661986, + "loss": 1.6012, + "theoretical_loss": 3.9762048231726297, + "tokens_seen": 462880768 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006947241725175527, + "loss": 1.71, + "theoretical_loss": 3.9761438241580294, + "tokens_seen": 462946304 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006947081243731194, + "loss": 1.5856, + "theoretical_loss": 3.976082836195477, + "tokens_seen": 463011840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006946920762286861, + "loss": 1.5671, + "theoretical_loss": 3.976021859281406, + "tokens_seen": 463077376 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006946760280842528, + "loss": 1.5678, + "theoretical_loss": 3.975960893412251, + "tokens_seen": 463142912 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006946599799398195, + "loss": 1.489, + "theoretical_loss": 3.9758999385844502, + "tokens_seen": 463208448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006946439317953862, + "loss": 1.4827, + "theoretical_loss": 3.975838994794441, + "tokens_seen": 463273984 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006946278836509529, + "loss": 1.5419, + "theoretical_loss": 3.975778062038665, + "tokens_seen": 463339520 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006946118355065196, + "loss": 1.6756, + "theoretical_loss": 3.975717140313564, + "tokens_seen": 463405056 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006945957873620863, + "loss": 1.6559, + "theoretical_loss": 3.9756562296155797, + "tokens_seen": 463470592 + }, + { + "epoch": 0.14, + "learning_rate": 0.000694579739217653, + "loss": 1.4332, + "theoretical_loss": 3.975595329941159, + "tokens_seen": 463536128 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006945636910732197, + "loss": 1.5515, + "theoretical_loss": 3.975534441286748, + "tokens_seen": 463601664 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006945476429287864, + "loss": 1.5143, + "theoretical_loss": 3.975473563648796, + "tokens_seen": 463667200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006945315947843531, + "loss": 1.6881, + "theoretical_loss": 3.975412697023753, + "tokens_seen": 463732736 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006945155466399198, + "loss": 1.554, + "theoretical_loss": 3.9753518414080693, + "tokens_seen": 463798272 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006944994984954865, + "loss": 1.532, + "theoretical_loss": 3.9752909967982, + "tokens_seen": 463863808 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006944834503510532, + "loss": 1.6627, + "theoretical_loss": 3.9752301631906, + "tokens_seen": 463929344 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006944674022066199, + "loss": 1.5631, + "theoretical_loss": 3.9751693405817248, + "tokens_seen": 463994880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006944513540621866, + "loss": 1.7128, + "theoretical_loss": 3.9751085289680343, + "tokens_seen": 464060416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006944353059177533, + "loss": 1.7823, + "theoretical_loss": 3.975047728345987, + "tokens_seen": 464125952 + }, + { + "epoch": 0.14, + "learning_rate": 0.00069441925777332, + "loss": 1.6777, + "theoretical_loss": 3.9749869387120462, + "tokens_seen": 464191488 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006944032096288867, + "loss": 1.4855, + "theoretical_loss": 3.974926160062674, + "tokens_seen": 464257024 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006943871614844534, + "loss": 1.5624, + "theoretical_loss": 3.9748653923943356, + "tokens_seen": 464322560 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006943711133400201, + "loss": 1.7012, + "theoretical_loss": 3.9748046357034976, + "tokens_seen": 464388096 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006943550651955868, + "loss": 1.5901, + "theoretical_loss": 3.9747438899866276, + "tokens_seen": 464453632 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006943390170511535, + "loss": 1.5329, + "theoretical_loss": 3.9746831552401964, + "tokens_seen": 464519168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006943229689067202, + "loss": 1.5378, + "theoretical_loss": 3.9746224314606753, + "tokens_seen": 464584704 + }, + { + "epoch": 0.14, + "learning_rate": 0.000694306920762287, + "loss": 1.5739, + "theoretical_loss": 3.9745617186445372, + "tokens_seen": 464650240 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006942908726178536, + "loss": 1.6888, + "theoretical_loss": 3.974501016788256, + "tokens_seen": 464715776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006942748244734202, + "loss": 1.6939, + "theoretical_loss": 3.9744403258883096, + "tokens_seen": 464781312 + }, + { + "epoch": 0.14, + "learning_rate": 0.000694258776328987, + "loss": 1.4947, + "theoretical_loss": 3.974379645941174, + "tokens_seen": 464846848 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006942427281845536, + "loss": 1.5452, + "theoretical_loss": 3.974318976943331, + "tokens_seen": 464912384 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006942266800401204, + "loss": 1.6336, + "theoretical_loss": 3.974258318891261, + "tokens_seen": 464977920 + }, + { + "epoch": 0.14, + "learning_rate": 0.000694210631895687, + "loss": 1.5032, + "theoretical_loss": 3.974197671781446, + "tokens_seen": 465043456 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006941945837512538, + "loss": 1.5009, + "theoretical_loss": 3.9741370356103713, + "tokens_seen": 465108992 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006941785356068204, + "loss": 1.5687, + "theoretical_loss": 3.9740764103745225, + "tokens_seen": 465174528 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006941624874623872, + "loss": 1.5982, + "theoretical_loss": 3.974015796070388, + "tokens_seen": 465240064 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.9939460158348083, + "objective/train/avg_token_score": 0.00605401024222374, + "objective/train/docs_used": 175471, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.346516489982605, + "objective/train/original_loss": 1.3547816276550293, + "objective/train/theoretical_loss": 3.973955192694457, + "objective/train/tokens_used": 485765600, + "objective/train/unlikelihood_loss": 0.016714992001652718, + "theoretical_loss": 3.973955192694457, + "tokens_seen": 465305600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006941464393179538, + "loss": 1.4568, + "theoretical_loss": 3.973955192694457, + "tokens_seen": 465305600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006941303911735206, + "loss": 1.508, + "theoretical_loss": 3.97389460024322, + "tokens_seen": 465371136 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006941143430290872, + "loss": 1.549, + "theoretical_loss": 3.9738340187131698, + "tokens_seen": 465436672 + }, + { + "epoch": 0.14, + "learning_rate": 0.000694098294884654, + "loss": 1.6099, + "theoretical_loss": 3.973773448100801, + "tokens_seen": 465502208 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006940822467402206, + "loss": 1.5326, + "theoretical_loss": 3.973712888402609, + "tokens_seen": 465567744 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006940661985957874, + "loss": 1.5842, + "theoretical_loss": 3.9736523396150916, + "tokens_seen": 465633280 + }, + { + "epoch": 0.14, + "learning_rate": 0.000694050150451354, + "loss": 1.4776, + "theoretical_loss": 3.973591801734748, + "tokens_seen": 465698816 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006940341023069208, + "loss": 1.6061, + "theoretical_loss": 3.9735312747580775, + "tokens_seen": 465764352 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006940180541624875, + "loss": 1.6576, + "theoretical_loss": 3.973470758681584, + "tokens_seen": 465829888 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006940020060180542, + "loss": 1.5818, + "theoretical_loss": 3.9734102535017715, + "tokens_seen": 465895424 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006939859578736209, + "loss": 1.5694, + "theoretical_loss": 3.9733497592151443, + "tokens_seen": 465960960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006939699097291876, + "loss": 1.4543, + "theoretical_loss": 3.9732892758182103, + "tokens_seen": 466026496 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006939538615847543, + "loss": 1.5885, + "theoretical_loss": 3.973228803307478, + "tokens_seen": 466092032 + }, + { + "epoch": 0.14, + "learning_rate": 0.000693937813440321, + "loss": 1.4159, + "theoretical_loss": 3.9731683416794583, + "tokens_seen": 466157568 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006939217652958877, + "loss": 1.5649, + "theoretical_loss": 3.9731078909306623, + "tokens_seen": 466223104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006939057171514544, + "loss": 1.7228, + "theoretical_loss": 3.9730474510576044, + "tokens_seen": 466288640 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006938896690070211, + "loss": 1.6165, + "theoretical_loss": 3.9729870220567993, + "tokens_seen": 466354176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006938736208625878, + "loss": 1.6423, + "theoretical_loss": 3.9729266039247637, + "tokens_seen": 466419712 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006938575727181545, + "loss": 1.5594, + "theoretical_loss": 3.9728661966580163, + "tokens_seen": 466485248 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006938415245737212, + "loss": 1.5178, + "theoretical_loss": 3.9728058002530773, + "tokens_seen": 466550784 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006938254764292879, + "loss": 1.6196, + "theoretical_loss": 3.972745414706468, + "tokens_seen": 466616320 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006938094282848546, + "loss": 1.4884, + "theoretical_loss": 3.972685040014711, + "tokens_seen": 466681856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006937933801404213, + "loss": 1.446, + "theoretical_loss": 3.9726246761743322, + "tokens_seen": 466747392 + }, + { + "epoch": 0.14, + "learning_rate": 0.000693777331995988, + "loss": 1.5277, + "theoretical_loss": 3.9725643231818575, + "tokens_seen": 466812928 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006937612838515547, + "loss": 1.597, + "theoretical_loss": 3.9725039810338147, + "tokens_seen": 466878464 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006937452357071214, + "loss": 1.6185, + "theoretical_loss": 3.972443649726734, + "tokens_seen": 466944000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006937291875626881, + "loss": 1.5424, + "theoretical_loss": 3.9723833292571458, + "tokens_seen": 467009536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006937131394182548, + "loss": 1.4781, + "theoretical_loss": 3.9723230196215837, + "tokens_seen": 467075072 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006936970912738215, + "loss": 1.6678, + "theoretical_loss": 3.9722627208165817, + "tokens_seen": 467140608 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006936810431293882, + "loss": 1.5599, + "theoretical_loss": 3.9722024328386754, + "tokens_seen": 467206144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006936649949849549, + "loss": 1.52, + "theoretical_loss": 3.9721421556844034, + "tokens_seen": 467271680 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006936489468405216, + "loss": 1.588, + "theoretical_loss": 3.972081889350304, + "tokens_seen": 467337216 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006936328986960883, + "loss": 1.579, + "theoretical_loss": 3.9720216338329184, + "tokens_seen": 467402752 + }, + { + "epoch": 0.14, + "learning_rate": 0.000693616850551655, + "loss": 1.6067, + "theoretical_loss": 3.971961389128789, + "tokens_seen": 467468288 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006936008024072217, + "loss": 1.5613, + "theoretical_loss": 3.97190115523446, + "tokens_seen": 467533824 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006935847542627884, + "loss": 1.5053, + "theoretical_loss": 3.9718409321464754, + "tokens_seen": 467599360 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006935687061183551, + "loss": 1.6359, + "theoretical_loss": 3.9717807198613837, + "tokens_seen": 467664896 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006935526579739218, + "loss": 1.6735, + "theoretical_loss": 3.971720518375734, + "tokens_seen": 467730432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006935366098294885, + "loss": 1.5989, + "theoretical_loss": 3.971660327686075, + "tokens_seen": 467795968 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006935205616850552, + "loss": 1.4861, + "theoretical_loss": 3.97160014778896, + "tokens_seen": 467861504 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006935045135406219, + "loss": 1.6551, + "theoretical_loss": 3.9715399786809424, + "tokens_seen": 467927040 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006934884653961886, + "loss": 1.5261, + "theoretical_loss": 3.971479820358576, + "tokens_seen": 467992576 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006934724172517553, + "loss": 1.4847, + "theoretical_loss": 3.9714196728184197, + "tokens_seen": 468058112 + }, + { + "epoch": 0.14, + "learning_rate": 0.000693456369107322, + "loss": 1.4971, + "theoretical_loss": 3.971359536057029, + "tokens_seen": 468123648 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006934403209628887, + "loss": 1.5557, + "theoretical_loss": 3.9712994100709653, + "tokens_seen": 468189184 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006934242728184554, + "loss": 1.5615, + "theoretical_loss": 3.97123929485679, + "tokens_seen": 468254720 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006934082246740221, + "loss": 1.5552, + "theoretical_loss": 3.971179190411066, + "tokens_seen": 468320256 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006933921765295888, + "loss": 1.5454, + "theoretical_loss": 3.9711190967303565, + "tokens_seen": 468385792 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006933761283851555, + "loss": 1.5564, + "theoretical_loss": 3.971059013811229, + "tokens_seen": 468451328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006933600802407222, + "loss": 1.4867, + "theoretical_loss": 3.9709989416502514, + "tokens_seen": 468516864 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.9959079623222351, + "objective/train/avg_token_score": 0.00409207958728075, + "objective/train/docs_used": 176758, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.7020609378814697, + "objective/train/original_loss": 1.709524154663086, + "objective/train/theoretical_loss": 3.9709388802439918, + "objective/train/tokens_used": 489042400, + "objective/train/unlikelihood_loss": 0.00859786942601204, + "theoretical_loss": 3.9709388802439918, + "tokens_seen": 468582400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006933440320962889, + "loss": 1.6549, + "theoretical_loss": 3.9709388802439918, + "tokens_seen": 468582400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006933279839518556, + "loss": 1.5782, + "theoretical_loss": 3.9708788295890223, + "tokens_seen": 468647936 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006933119358074223, + "loss": 1.6419, + "theoretical_loss": 3.9708187896819136, + "tokens_seen": 468713472 + }, + { + "epoch": 0.14, + "learning_rate": 0.000693295887662989, + "loss": 1.6213, + "theoretical_loss": 3.9707587605192414, + "tokens_seen": 468779008 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006932798395185557, + "loss": 1.5485, + "theoretical_loss": 3.9706987420975803, + "tokens_seen": 468844544 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006932637913741224, + "loss": 1.5724, + "theoretical_loss": 3.970638734413508, + "tokens_seen": 468910080 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006932477432296891, + "loss": 1.5883, + "theoretical_loss": 3.970578737463603, + "tokens_seen": 468975616 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006932316950852558, + "loss": 1.618, + "theoretical_loss": 3.9705187512444446, + "tokens_seen": 469041152 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006932156469408225, + "loss": 1.5975, + "theoretical_loss": 3.9704587757526157, + "tokens_seen": 469106688 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006931995987963892, + "loss": 1.6684, + "theoretical_loss": 3.9703988109846997, + "tokens_seen": 469172224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006931835506519559, + "loss": 1.5144, + "theoretical_loss": 3.970338856937281, + "tokens_seen": 469237760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006931675025075226, + "loss": 1.5771, + "theoretical_loss": 3.970278913606947, + "tokens_seen": 469303296 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006931514543630893, + "loss": 1.6002, + "theoretical_loss": 3.9702189809902846, + "tokens_seen": 469368832 + }, + { + "epoch": 0.14, + "learning_rate": 0.000693135406218656, + "loss": 1.5927, + "theoretical_loss": 3.9701590590838842, + "tokens_seen": 469434368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006931193580742227, + "loss": 1.5113, + "theoretical_loss": 3.970099147884336, + "tokens_seen": 469499904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006931033099297894, + "loss": 1.5417, + "theoretical_loss": 3.9700392473882347, + "tokens_seen": 469565440 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006930872617853562, + "loss": 1.6198, + "theoretical_loss": 3.969979357592173, + "tokens_seen": 469630976 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006930712136409228, + "loss": 1.5996, + "theoretical_loss": 3.969919478492747, + "tokens_seen": 469696512 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006930551654964896, + "loss": 1.5757, + "theoretical_loss": 3.9698596100865546, + "tokens_seen": 469762048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006930391173520562, + "loss": 1.6262, + "theoretical_loss": 3.9697997523701947, + "tokens_seen": 469827584 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006930230692076229, + "loss": 1.5628, + "theoretical_loss": 3.969739905340268, + "tokens_seen": 469893120 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006930070210631896, + "loss": 1.6482, + "theoretical_loss": 3.9696800689933758, + "tokens_seen": 469958656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006929909729187563, + "loss": 1.6296, + "theoretical_loss": 3.9696202433261227, + "tokens_seen": 470024192 + }, + { + "epoch": 0.14, + "learning_rate": 0.000692974924774323, + "loss": 1.5884, + "theoretical_loss": 3.9695604283351136, + "tokens_seen": 470089728 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006929588766298897, + "loss": 1.5089, + "theoretical_loss": 3.9695006240169546, + "tokens_seen": 470155264 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006929428284854564, + "loss": 1.5596, + "theoretical_loss": 3.9694408303682556, + "tokens_seen": 470220800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006929267803410231, + "loss": 1.5181, + "theoretical_loss": 3.9693810473856246, + "tokens_seen": 470286336 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006929107321965898, + "loss": 1.5264, + "theoretical_loss": 3.969321275065674, + "tokens_seen": 470351872 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006928946840521565, + "loss": 1.6228, + "theoretical_loss": 3.969261513405017, + "tokens_seen": 470417408 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006928786359077231, + "loss": 1.6622, + "theoretical_loss": 3.9692017624002682, + "tokens_seen": 470482944 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006928625877632899, + "loss": 1.5664, + "theoretical_loss": 3.9691420220480422, + "tokens_seen": 470548480 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006928465396188565, + "loss": 1.4697, + "theoretical_loss": 3.9690822923449582, + "tokens_seen": 470614016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006928304914744233, + "loss": 1.6653, + "theoretical_loss": 3.969022573287635, + "tokens_seen": 470679552 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006928144433299899, + "loss": 1.5218, + "theoretical_loss": 3.968962864872693, + "tokens_seen": 470745088 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006927983951855567, + "loss": 1.5918, + "theoretical_loss": 3.9689031670967543, + "tokens_seen": 470810624 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006927823470411233, + "loss": 1.5839, + "theoretical_loss": 3.9688434799564427, + "tokens_seen": 470876160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006927662988966901, + "loss": 1.5531, + "theoretical_loss": 3.9687838034483844, + "tokens_seen": 470941696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006927502507522567, + "loss": 1.5178, + "theoretical_loss": 3.9687241375692053, + "tokens_seen": 471007232 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006927342026078235, + "loss": 1.5798, + "theoretical_loss": 3.9686644823155337, + "tokens_seen": 471072768 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006927181544633901, + "loss": 1.6689, + "theoretical_loss": 3.9686048376840004, + "tokens_seen": 471138304 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006927021063189569, + "loss": 1.6645, + "theoretical_loss": 3.968545203671236, + "tokens_seen": 471203840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006926860581745235, + "loss": 1.5789, + "theoretical_loss": 3.9684855802738745, + "tokens_seen": 471269376 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006926700100300903, + "loss": 1.6152, + "theoretical_loss": 3.9684259674885496, + "tokens_seen": 471334912 + }, + { + "epoch": 0.14, + "learning_rate": 0.000692653961885657, + "loss": 1.4754, + "theoretical_loss": 3.9683663653118977, + "tokens_seen": 471400448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006926379137412237, + "loss": 1.547, + "theoretical_loss": 3.968306773740556, + "tokens_seen": 471465984 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006926218655967904, + "loss": 1.6915, + "theoretical_loss": 3.9682471927711642, + "tokens_seen": 471531520 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006926058174523571, + "loss": 1.5257, + "theoretical_loss": 3.9681876224003627, + "tokens_seen": 471597056 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006925897693079238, + "loss": 1.678, + "theoretical_loss": 3.9681280626247943, + "tokens_seen": 471662592 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006925737211634905, + "loss": 1.6034, + "theoretical_loss": 3.9680685134411022, + "tokens_seen": 471728128 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006925576730190572, + "loss": 1.5635, + "theoretical_loss": 3.9680089748459313, + "tokens_seen": 471793664 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.9907188415527344, + "objective/train/avg_token_score": 0.00928118359297514, + "objective/train/docs_used": 177948, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6490741968154907, + "objective/train/original_loss": 1.6633743047714233, + "objective/train/theoretical_loss": 3.967949446835929, + "objective/train/tokens_used": 492319200, + "objective/train/unlikelihood_loss": 0.020627666264772415, + "theoretical_loss": 3.967949446835929, + "tokens_seen": 471859200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006925416248746239, + "loss": 1.6064, + "theoretical_loss": 3.967949446835929, + "tokens_seen": 471859200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006925255767301906, + "loss": 1.4287, + "theoretical_loss": 3.967889929407743, + "tokens_seen": 471924736 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006925095285857573, + "loss": 1.5608, + "theoretical_loss": 3.9678304225580248, + "tokens_seen": 471990272 + }, + { + "epoch": 0.14, + "learning_rate": 0.000692493480441324, + "loss": 1.4541, + "theoretical_loss": 3.967770926283424, + "tokens_seen": 472055808 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006924774322968907, + "loss": 1.5154, + "theoretical_loss": 3.9677114405805938, + "tokens_seen": 472121344 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006924613841524574, + "loss": 1.5038, + "theoretical_loss": 3.967651965446189, + "tokens_seen": 472186880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006924453360080241, + "loss": 1.6218, + "theoretical_loss": 3.9675925008768655, + "tokens_seen": 472252416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006924292878635908, + "loss": 1.5803, + "theoretical_loss": 3.9675330468692804, + "tokens_seen": 472317952 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006924132397191575, + "loss": 1.518, + "theoretical_loss": 3.9674736034200935, + "tokens_seen": 472383488 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006923971915747242, + "loss": 1.6272, + "theoretical_loss": 3.967414170525965, + "tokens_seen": 472449024 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006923811434302909, + "loss": 1.5876, + "theoretical_loss": 3.9673547481835567, + "tokens_seen": 472514560 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006923650952858576, + "loss": 1.5906, + "theoretical_loss": 3.9672953363895322, + "tokens_seen": 472580096 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006923490471414243, + "loss": 1.554, + "theoretical_loss": 3.9672359351405566, + "tokens_seen": 472645632 + }, + { + "epoch": 0.14, + "learning_rate": 0.000692332998996991, + "loss": 1.6537, + "theoretical_loss": 3.967176544433296, + "tokens_seen": 472711168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006923169508525577, + "loss": 1.6023, + "theoretical_loss": 3.96711716426442, + "tokens_seen": 472776704 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006923009027081244, + "loss": 1.4456, + "theoretical_loss": 3.9670577946305965, + "tokens_seen": 472842240 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006922848545636911, + "loss": 1.7044, + "theoretical_loss": 3.9669984355284975, + "tokens_seen": 472907776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006922688064192578, + "loss": 1.5653, + "theoretical_loss": 3.9669390869547954, + "tokens_seen": 472973312 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006922527582748245, + "loss": 1.5504, + "theoretical_loss": 3.966879748906165, + "tokens_seen": 473038848 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006922367101303912, + "loss": 1.4717, + "theoretical_loss": 3.966820421379281, + "tokens_seen": 473104384 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006922206619859579, + "loss": 1.6541, + "theoretical_loss": 3.966761104370821, + "tokens_seen": 473169920 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006922046138415246, + "loss": 1.5185, + "theoretical_loss": 3.9667017978774632, + "tokens_seen": 473235456 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006921885656970913, + "loss": 1.637, + "theoretical_loss": 3.966642501895889, + "tokens_seen": 473300992 + }, + { + "epoch": 0.14, + "learning_rate": 0.000692172517552658, + "loss": 1.7024, + "theoretical_loss": 3.9665832164227792, + "tokens_seen": 473366528 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006921564694082247, + "loss": 1.5498, + "theoretical_loss": 3.966523941454817, + "tokens_seen": 473432064 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006921404212637914, + "loss": 1.5923, + "theoretical_loss": 3.9664646769886875, + "tokens_seen": 473497600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006921243731193581, + "loss": 1.6074, + "theoretical_loss": 3.966405423021077, + "tokens_seen": 473563136 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006921083249749248, + "loss": 1.4872, + "theoretical_loss": 3.9663461795486725, + "tokens_seen": 473628672 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006920922768304915, + "loss": 1.566, + "theoretical_loss": 3.9662869465681636, + "tokens_seen": 473694208 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006920762286860582, + "loss": 1.5768, + "theoretical_loss": 3.9662277240762416, + "tokens_seen": 473759744 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006920601805416249, + "loss": 1.6214, + "theoretical_loss": 3.9661685120695975, + "tokens_seen": 473825280 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006920441323971916, + "loss": 1.5132, + "theoretical_loss": 3.9661093105449265, + "tokens_seen": 473890816 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006920280842527583, + "loss": 1.595, + "theoretical_loss": 3.966050119498923, + "tokens_seen": 473956352 + }, + { + "epoch": 0.14, + "learning_rate": 0.000692012036108325, + "loss": 1.4884, + "theoretical_loss": 3.9659909389282832, + "tokens_seen": 474021888 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006919959879638917, + "loss": 1.6068, + "theoretical_loss": 3.9659317688297064, + "tokens_seen": 474087424 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006919799398194584, + "loss": 1.573, + "theoretical_loss": 3.965872609199892, + "tokens_seen": 474152960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006919638916750251, + "loss": 1.488, + "theoretical_loss": 3.965813460035541, + "tokens_seen": 474218496 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006919478435305918, + "loss": 1.6159, + "theoretical_loss": 3.965754321333356, + "tokens_seen": 474284032 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006919317953861585, + "loss": 1.6518, + "theoretical_loss": 3.965695193090042, + "tokens_seen": 474349568 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006919157472417252, + "loss": 1.5834, + "theoretical_loss": 3.9656360753023034, + "tokens_seen": 474415104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006918996990972919, + "loss": 1.6099, + "theoretical_loss": 3.9655769679668484, + "tokens_seen": 474480640 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006918836509528587, + "loss": 1.668, + "theoretical_loss": 3.965517871080386, + "tokens_seen": 474546176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006918676028084253, + "loss": 1.557, + "theoretical_loss": 3.965458784639625, + "tokens_seen": 474611712 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006918515546639921, + "loss": 1.7312, + "theoretical_loss": 3.965399708641278, + "tokens_seen": 474677248 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006918355065195587, + "loss": 1.617, + "theoretical_loss": 3.9653406430820586, + "tokens_seen": 474742784 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006918194583751255, + "loss": 1.5417, + "theoretical_loss": 3.9652815879586805, + "tokens_seen": 474808320 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006918034102306921, + "loss": 1.4764, + "theoretical_loss": 3.96522254326786, + "tokens_seen": 474873856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006917873620862589, + "loss": 1.6551, + "theoretical_loss": 3.9651635090063153, + "tokens_seen": 474939392 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006917713139418255, + "loss": 1.5588, + "theoretical_loss": 3.965104485170765, + "tokens_seen": 475004928 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006917552657973923, + "loss": 1.6172, + "theoretical_loss": 3.9650454717579295, + "tokens_seen": 475070464 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.9953663945198059, + "objective/train/avg_token_score": 0.004633630160242319, + "objective/train/docs_used": 179087, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4645859003067017, + "objective/train/original_loss": 1.4715452194213867, + "objective/train/theoretical_loss": 3.9649864687645318, + "objective/train/tokens_used": 495596000, + "objective/train/unlikelihood_loss": 0.013906018808484077, + "theoretical_loss": 3.9649864687645318, + "tokens_seen": 475136000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006917392176529589, + "loss": 1.4705, + "theoretical_loss": 3.9649864687645318, + "tokens_seen": 475136000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006917231695085257, + "loss": 1.5334, + "theoretical_loss": 3.9649274761872944, + "tokens_seen": 475201536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006917071213640923, + "loss": 1.5513, + "theoretical_loss": 3.9648684940229426, + "tokens_seen": 475267072 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006916910732196591, + "loss": 1.5602, + "theoretical_loss": 3.9648095222682036, + "tokens_seen": 475332608 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006916750250752257, + "loss": 1.5707, + "theoretical_loss": 3.9647505609198044, + "tokens_seen": 475398144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006916589769307925, + "loss": 1.7272, + "theoretical_loss": 3.9646916099744756, + "tokens_seen": 475463680 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006916429287863591, + "loss": 1.5988, + "theoretical_loss": 3.9646326694289473, + "tokens_seen": 475529216 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006916268806419258, + "loss": 1.6453, + "theoretical_loss": 3.9645737392799516, + "tokens_seen": 475594752 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006916108324974925, + "loss": 1.5335, + "theoretical_loss": 3.964514819524224, + "tokens_seen": 475660288 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006915947843530592, + "loss": 1.5316, + "theoretical_loss": 3.964455910158498, + "tokens_seen": 475725824 + }, + { + "epoch": 0.14, + "learning_rate": 0.000691578736208626, + "loss": 1.4879, + "theoretical_loss": 3.9643970111795115, + "tokens_seen": 475791360 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006915626880641926, + "loss": 1.629, + "theoretical_loss": 3.964338122584003, + "tokens_seen": 475856896 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006915466399197593, + "loss": 1.4873, + "theoretical_loss": 3.964279244368712, + "tokens_seen": 475922432 + }, + { + "epoch": 0.14, + "learning_rate": 0.000691530591775326, + "loss": 1.5014, + "theoretical_loss": 3.9642203765303794, + "tokens_seen": 475987968 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006915145436308928, + "loss": 1.4133, + "theoretical_loss": 3.9641615190657484, + "tokens_seen": 476053504 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006914984954864594, + "loss": 1.5964, + "theoretical_loss": 3.9641026719715633, + "tokens_seen": 476119040 + }, + { + "epoch": 0.14, + "learning_rate": 0.000691482447342026, + "loss": 1.5548, + "theoretical_loss": 3.9640438352445697, + "tokens_seen": 476184576 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006914663991975928, + "loss": 1.6144, + "theoretical_loss": 3.9639850088815143, + "tokens_seen": 476250112 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006914503510531594, + "loss": 1.5175, + "theoretical_loss": 3.9639261928791463, + "tokens_seen": 476315648 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006914343029087262, + "loss": 1.5072, + "theoretical_loss": 3.963867387234216, + "tokens_seen": 476381184 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006914182547642928, + "loss": 1.4922, + "theoretical_loss": 3.963808591943474, + "tokens_seen": 476446720 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006914022066198596, + "loss": 1.5179, + "theoretical_loss": 3.9637498070036745, + "tokens_seen": 476512256 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006913861584754262, + "loss": 1.5846, + "theoretical_loss": 3.963691032411571, + "tokens_seen": 476577792 + }, + { + "epoch": 0.14, + "learning_rate": 0.000691370110330993, + "loss": 1.7107, + "theoretical_loss": 3.9636322681639196, + "tokens_seen": 476643328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006913540621865596, + "loss": 1.4636, + "theoretical_loss": 3.9635735142574786, + "tokens_seen": 476708864 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006913380140421264, + "loss": 1.6509, + "theoretical_loss": 3.963514770689006, + "tokens_seen": 476774400 + }, + { + "epoch": 0.14, + "learning_rate": 0.000691321965897693, + "loss": 1.4943, + "theoretical_loss": 3.9634560374552623, + "tokens_seen": 476839936 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006913059177532598, + "loss": 1.5774, + "theoretical_loss": 3.96339731455301, + "tokens_seen": 476905472 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006912898696088265, + "loss": 1.4224, + "theoretical_loss": 3.963338601979011, + "tokens_seen": 476971008 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006912738214643932, + "loss": 1.5374, + "theoretical_loss": 3.9632798997300314, + "tokens_seen": 477036544 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006912577733199599, + "loss": 1.6298, + "theoretical_loss": 3.963221207802836, + "tokens_seen": 477102080 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006912417251755266, + "loss": 1.4068, + "theoretical_loss": 3.9631625261941945, + "tokens_seen": 477167616 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006912256770310933, + "loss": 1.5208, + "theoretical_loss": 3.963103854900874, + "tokens_seen": 477233152 + }, + { + "epoch": 0.14, + "learning_rate": 0.00069120962888666, + "loss": 1.5425, + "theoretical_loss": 3.963045193919646, + "tokens_seen": 477298688 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006911935807422267, + "loss": 1.4762, + "theoretical_loss": 3.9629865432472817, + "tokens_seen": 477364224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006911775325977934, + "loss": 1.58, + "theoretical_loss": 3.9629279028805553, + "tokens_seen": 477429760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006911614844533601, + "loss": 1.6613, + "theoretical_loss": 3.962869272816242, + "tokens_seen": 477495296 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006911454363089268, + "loss": 1.6078, + "theoretical_loss": 3.9628106530511173, + "tokens_seen": 477560832 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006911293881644935, + "loss": 1.6185, + "theoretical_loss": 3.9627520435819594, + "tokens_seen": 477626368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006911133400200602, + "loss": 1.5867, + "theoretical_loss": 3.9626934444055477, + "tokens_seen": 477691904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006910972918756269, + "loss": 1.5443, + "theoretical_loss": 3.962634855518663, + "tokens_seen": 477757440 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006910812437311936, + "loss": 1.4834, + "theoretical_loss": 3.962576276918087, + "tokens_seen": 477822976 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006910651955867603, + "loss": 1.5891, + "theoretical_loss": 3.962517708600603, + "tokens_seen": 477888512 + }, + { + "epoch": 0.14, + "learning_rate": 0.000691049147442327, + "loss": 1.616, + "theoretical_loss": 3.9624591505629976, + "tokens_seen": 477954048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006910330992978937, + "loss": 1.6543, + "theoretical_loss": 3.962400602802056, + "tokens_seen": 478019584 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006910170511534604, + "loss": 1.5914, + "theoretical_loss": 3.962342065314566, + "tokens_seen": 478085120 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006910010030090271, + "loss": 1.5359, + "theoretical_loss": 3.962283538097318, + "tokens_seen": 478150656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006909849548645938, + "loss": 1.5715, + "theoretical_loss": 3.962225021147102, + "tokens_seen": 478216192 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006909689067201605, + "loss": 1.4663, + "theoretical_loss": 3.9621665144607103, + "tokens_seen": 478281728 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006909528585757272, + "loss": 1.5979, + "theoretical_loss": 3.9621080180349373, + "tokens_seen": 478347264 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.9937477111816406, + "objective/train/avg_token_score": 0.006252339109778404, + "objective/train/docs_used": 180329, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6108191013336182, + "objective/train/original_loss": 1.6204922199249268, + "objective/train/theoretical_loss": 3.9620495318665774, + "objective/train/tokens_used": 498872800, + "objective/train/unlikelihood_loss": 0.01693345420062542, + "theoretical_loss": 3.9620495318665774, + "tokens_seen": 478412800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006909368104312939, + "loss": 1.6072, + "theoretical_loss": 3.9620495318665774, + "tokens_seen": 478412800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006909207622868606, + "loss": 1.6783, + "theoretical_loss": 3.9619910559524274, + "tokens_seen": 478478336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006909047141424273, + "loss": 1.4989, + "theoretical_loss": 3.9619325902892855, + "tokens_seen": 478543872 + }, + { + "epoch": 0.15, + "learning_rate": 0.000690888665997994, + "loss": 1.557, + "theoretical_loss": 3.961874134873951, + "tokens_seen": 478609408 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006908726178535607, + "loss": 1.5662, + "theoretical_loss": 3.9618156897032257, + "tokens_seen": 478674944 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006908565697091274, + "loss": 1.6258, + "theoretical_loss": 3.961757254773911, + "tokens_seen": 478740480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006908405215646941, + "loss": 1.5375, + "theoretical_loss": 3.9616988300828098, + "tokens_seen": 478806016 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006908244734202608, + "loss": 1.5821, + "theoretical_loss": 3.9616404156267286, + "tokens_seen": 478871552 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006908084252758275, + "loss": 1.4755, + "theoretical_loss": 3.961582011402475, + "tokens_seen": 478937088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006907923771313942, + "loss": 1.4828, + "theoretical_loss": 3.9615236174068547, + "tokens_seen": 479002624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006907763289869609, + "loss": 1.6074, + "theoretical_loss": 3.961465233636679, + "tokens_seen": 479068160 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006907602808425276, + "loss": 1.5951, + "theoretical_loss": 3.9614068600887578, + "tokens_seen": 479133696 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006907442326980943, + "loss": 1.5479, + "theoretical_loss": 3.9613484967599044, + "tokens_seen": 479199232 + }, + { + "epoch": 0.15, + "learning_rate": 0.000690728184553661, + "loss": 1.4103, + "theoretical_loss": 3.961290143646932, + "tokens_seen": 479264768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006907121364092277, + "loss": 1.6422, + "theoretical_loss": 3.961231800746656, + "tokens_seen": 479330304 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006906960882647944, + "loss": 1.511, + "theoretical_loss": 3.961173468055893, + "tokens_seen": 479395840 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006906800401203612, + "loss": 1.4334, + "theoretical_loss": 3.961115145571461, + "tokens_seen": 479461376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006906639919759278, + "loss": 1.575, + "theoretical_loss": 3.96105683329018, + "tokens_seen": 479526912 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006906479438314946, + "loss": 1.5522, + "theoretical_loss": 3.9609985312088707, + "tokens_seen": 479592448 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006906318956870612, + "loss": 1.6016, + "theoretical_loss": 3.960940239324355, + "tokens_seen": 479657984 + }, + { + "epoch": 0.15, + "learning_rate": 0.000690615847542628, + "loss": 1.5887, + "theoretical_loss": 3.960881957633457, + "tokens_seen": 479723520 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006905997993981946, + "loss": 1.6311, + "theoretical_loss": 3.9608236861330024, + "tokens_seen": 479789056 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006905837512537614, + "loss": 1.5543, + "theoretical_loss": 3.960765424819817, + "tokens_seen": 479854592 + }, + { + "epoch": 0.15, + "learning_rate": 0.000690567703109328, + "loss": 1.5011, + "theoretical_loss": 3.9607071736907296, + "tokens_seen": 479920128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006905516549648948, + "loss": 1.7091, + "theoretical_loss": 3.9606489327425694, + "tokens_seen": 479985664 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006905356068204614, + "loss": 1.6179, + "theoretical_loss": 3.960590701972167, + "tokens_seen": 480051200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006905195586760282, + "loss": 1.5018, + "theoretical_loss": 3.960532481376355, + "tokens_seen": 480116736 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006905035105315948, + "loss": 1.5345, + "theoretical_loss": 3.960474270951967, + "tokens_seen": 480182272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006904874623871616, + "loss": 1.4907, + "theoretical_loss": 3.9604160706958385, + "tokens_seen": 480247808 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006904714142427282, + "loss": 1.5777, + "theoretical_loss": 3.960357880604806, + "tokens_seen": 480313344 + }, + { + "epoch": 0.15, + "learning_rate": 0.000690455366098295, + "loss": 1.5295, + "theoretical_loss": 3.9602997006757072, + "tokens_seen": 480378880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006904393179538616, + "loss": 1.6044, + "theoretical_loss": 3.960241530905382, + "tokens_seen": 480444416 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006904232698094284, + "loss": 1.5563, + "theoretical_loss": 3.96018337129067, + "tokens_seen": 480509952 + }, + { + "epoch": 0.15, + "learning_rate": 0.000690407221664995, + "loss": 1.6532, + "theoretical_loss": 3.960125221828415, + "tokens_seen": 480575488 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006903911735205618, + "loss": 1.6263, + "theoretical_loss": 3.9600670825154594, + "tokens_seen": 480641024 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006903751253761284, + "loss": 1.6367, + "theoretical_loss": 3.960008953348649, + "tokens_seen": 480706560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006903590772316952, + "loss": 1.5658, + "theoretical_loss": 3.9599508343248297, + "tokens_seen": 480772096 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006903430290872618, + "loss": 1.6665, + "theoretical_loss": 3.9598927254408505, + "tokens_seen": 480837632 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006903269809428286, + "loss": 1.6123, + "theoretical_loss": 3.9598346266935596, + "tokens_seen": 480903168 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006903109327983952, + "loss": 1.5379, + "theoretical_loss": 3.959776538079808, + "tokens_seen": 480968704 + }, + { + "epoch": 0.15, + "learning_rate": 0.000690294884653962, + "loss": 1.5436, + "theoretical_loss": 3.9597184595964476, + "tokens_seen": 481034240 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006902788365095286, + "loss": 1.462, + "theoretical_loss": 3.959660391240332, + "tokens_seen": 481099776 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006902627883650954, + "loss": 1.528, + "theoretical_loss": 3.9596023330083163, + "tokens_seen": 481165312 + }, + { + "epoch": 0.15, + "learning_rate": 0.000690246740220662, + "loss": 1.5334, + "theoretical_loss": 3.9595442848972566, + "tokens_seen": 481230848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006902306920762287, + "loss": 1.6213, + "theoretical_loss": 3.959486246904011, + "tokens_seen": 481296384 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006902146439317954, + "loss": 1.5523, + "theoretical_loss": 3.959428219025438, + "tokens_seen": 481361920 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006901985957873621, + "loss": 1.585, + "theoretical_loss": 3.9593702012583987, + "tokens_seen": 481427456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006901825476429288, + "loss": 1.561, + "theoretical_loss": 3.9593121935997546, + "tokens_seen": 481492992 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006901664994984955, + "loss": 1.5237, + "theoretical_loss": 3.95925419604637, + "tokens_seen": 481558528 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006901504513540622, + "loss": 1.5167, + "theoretical_loss": 3.959196208595108, + "tokens_seen": 481624064 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.9928545355796814, + "objective/train/avg_token_score": 0.0071455007418990135, + "objective/train/docs_used": 181505, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5485954284667969, + "objective/train/original_loss": 1.5614221096038818, + "objective/train/theoretical_loss": 3.959138231242836, + "objective/train/tokens_used": 502149600, + "objective/train/unlikelihood_loss": 0.01597467251121998, + "theoretical_loss": 3.959138231242836, + "tokens_seen": 481689600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006901344032096289, + "loss": 1.5117, + "theoretical_loss": 3.959138231242836, + "tokens_seen": 481689600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006901183550651957, + "loss": 1.6526, + "theoretical_loss": 3.959080263986421, + "tokens_seen": 481755136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006901023069207623, + "loss": 1.5248, + "theoretical_loss": 3.9590223068227317, + "tokens_seen": 481820672 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006900862587763289, + "loss": 1.5774, + "theoretical_loss": 3.9589643597486393, + "tokens_seen": 481886208 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006900702106318957, + "loss": 1.6392, + "theoretical_loss": 3.9589064227610145, + "tokens_seen": 481951744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006900541624874623, + "loss": 1.5916, + "theoretical_loss": 3.9588484958567314, + "tokens_seen": 482017280 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006900381143430291, + "loss": 1.6459, + "theoretical_loss": 3.958790579032663, + "tokens_seen": 482082816 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006900220661985957, + "loss": 1.6008, + "theoretical_loss": 3.958732672285687, + "tokens_seen": 482148352 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006900060180541625, + "loss": 1.6606, + "theoretical_loss": 3.95867477561268, + "tokens_seen": 482213888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006899899699097291, + "loss": 1.5632, + "theoretical_loss": 3.95861688901052, + "tokens_seen": 482279424 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006899739217652959, + "loss": 1.4731, + "theoretical_loss": 3.958559012476088, + "tokens_seen": 482344960 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006899578736208625, + "loss": 1.5147, + "theoretical_loss": 3.958501146006265, + "tokens_seen": 482410496 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006899418254764293, + "loss": 1.5794, + "theoretical_loss": 3.958443289597934, + "tokens_seen": 482476032 + }, + { + "epoch": 0.15, + "learning_rate": 0.000689925777331996, + "loss": 1.585, + "theoretical_loss": 3.958385443247979, + "tokens_seen": 482541568 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006899097291875627, + "loss": 1.5407, + "theoretical_loss": 3.958327606953286, + "tokens_seen": 482607104 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006898936810431294, + "loss": 1.5859, + "theoretical_loss": 3.958269780710742, + "tokens_seen": 482672640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006898776328986961, + "loss": 1.5478, + "theoretical_loss": 3.9582119645172344, + "tokens_seen": 482738176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006898615847542628, + "loss": 1.4642, + "theoretical_loss": 3.9581541583696547, + "tokens_seen": 482803712 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006898455366098295, + "loss": 1.5489, + "theoretical_loss": 3.958096362264893, + "tokens_seen": 482869248 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006898294884653962, + "loss": 1.6063, + "theoretical_loss": 3.958038576199842, + "tokens_seen": 482934784 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006898134403209629, + "loss": 1.6268, + "theoretical_loss": 3.957980800171396, + "tokens_seen": 483000320 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006897973921765296, + "loss": 1.5814, + "theoretical_loss": 3.9579230341764493, + "tokens_seen": 483065856 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006897813440320963, + "loss": 1.5926, + "theoretical_loss": 3.9578652782118997, + "tokens_seen": 483131392 + }, + { + "epoch": 0.15, + "learning_rate": 0.000689765295887663, + "loss": 1.5685, + "theoretical_loss": 3.957807532274645, + "tokens_seen": 483196928 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006897492477432297, + "loss": 1.3337, + "theoretical_loss": 3.957749796361585, + "tokens_seen": 483262464 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006897331995987964, + "loss": 1.5377, + "theoretical_loss": 3.9576920704696192, + "tokens_seen": 483328000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006897171514543631, + "loss": 1.5213, + "theoretical_loss": 3.957634354595651, + "tokens_seen": 483393536 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006897011033099298, + "loss": 1.5646, + "theoretical_loss": 3.957576648736584, + "tokens_seen": 483459072 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006896850551654965, + "loss": 1.4929, + "theoretical_loss": 3.9575189528893233, + "tokens_seen": 483524608 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006896690070210632, + "loss": 1.5127, + "theoretical_loss": 3.957461267050774, + "tokens_seen": 483590144 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006896529588766299, + "loss": 1.569, + "theoretical_loss": 3.957403591217845, + "tokens_seen": 483655680 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006896369107321966, + "loss": 1.6013, + "theoretical_loss": 3.9573459253874455, + "tokens_seen": 483721216 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006896208625877633, + "loss": 1.5942, + "theoretical_loss": 3.9572882695564853, + "tokens_seen": 483786752 + }, + { + "epoch": 0.15, + "learning_rate": 0.00068960481444333, + "loss": 1.5142, + "theoretical_loss": 3.9572306237218764, + "tokens_seen": 483852288 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006895887662988967, + "loss": 1.4666, + "theoretical_loss": 3.957172987880532, + "tokens_seen": 483917824 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006895727181544634, + "loss": 1.5482, + "theoretical_loss": 3.9571153620293673, + "tokens_seen": 483983360 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006895566700100301, + "loss": 1.581, + "theoretical_loss": 3.9570577461652974, + "tokens_seen": 484048896 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006895406218655968, + "loss": 1.6535, + "theoretical_loss": 3.95700014028524, + "tokens_seen": 484114432 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006895245737211635, + "loss": 1.4853, + "theoretical_loss": 3.956942544386114, + "tokens_seen": 484179968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006895085255767302, + "loss": 1.5546, + "theoretical_loss": 3.9568849584648396, + "tokens_seen": 484245504 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006894924774322969, + "loss": 1.5012, + "theoretical_loss": 3.9568273825183375, + "tokens_seen": 484311040 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006894764292878637, + "loss": 1.5838, + "theoretical_loss": 3.9567698165435305, + "tokens_seen": 484376576 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006894603811434303, + "loss": 1.6887, + "theoretical_loss": 3.9567122605373437, + "tokens_seen": 484442112 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006894443329989971, + "loss": 1.5027, + "theoretical_loss": 3.956654714496702, + "tokens_seen": 484507648 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006894282848545637, + "loss": 1.5791, + "theoretical_loss": 3.9565971784185328, + "tokens_seen": 484573184 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006894122367101305, + "loss": 1.5869, + "theoretical_loss": 3.9565396522997633, + "tokens_seen": 484638720 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006893961885656971, + "loss": 1.5522, + "theoretical_loss": 3.9564821361373244, + "tokens_seen": 484704256 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006893801404212639, + "loss": 1.5813, + "theoretical_loss": 3.956424629928146, + "tokens_seen": 484769792 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006893640922768305, + "loss": 1.55, + "theoretical_loss": 3.9563671336691613, + "tokens_seen": 484835328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006893480441323973, + "loss": 1.6478, + "theoretical_loss": 3.956309647357303, + "tokens_seen": 484900864 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.9908996224403381, + "objective/train/avg_token_score": 0.009100399911403656, + "objective/train/docs_used": 182723, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.3807555437088013, + "objective/train/original_loss": 1.3952088356018066, + "objective/train/theoretical_loss": 3.956252170989507, + "objective/train/tokens_used": 505426400, + "objective/train/unlikelihood_loss": 0.025053469464182854, + "theoretical_loss": 3.956252170989507, + "tokens_seen": 484966400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006893319959879639, + "loss": 1.4822, + "theoretical_loss": 3.956252170989507, + "tokens_seen": 484966400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006893159478435307, + "loss": 1.5558, + "theoretical_loss": 3.9561947045627095, + "tokens_seen": 485031936 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006892998996990973, + "loss": 1.6272, + "theoretical_loss": 3.9561372480738486, + "tokens_seen": 485097472 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006892838515546641, + "loss": 1.554, + "theoretical_loss": 3.956079801519863, + "tokens_seen": 485163008 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006892678034102307, + "loss": 1.5616, + "theoretical_loss": 3.9560223648976924, + "tokens_seen": 485228544 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006892517552657975, + "loss": 1.6291, + "theoretical_loss": 3.9559649382042803, + "tokens_seen": 485294080 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006892357071213641, + "loss": 1.5457, + "theoretical_loss": 3.955907521436569, + "tokens_seen": 485359616 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006892196589769309, + "loss": 1.6502, + "theoretical_loss": 3.9558501145915033, + "tokens_seen": 485425152 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006892036108324975, + "loss": 1.5563, + "theoretical_loss": 3.9557927176660286, + "tokens_seen": 485490688 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006891875626880643, + "loss": 1.5929, + "theoretical_loss": 3.9557353306570926, + "tokens_seen": 485556224 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006891715145436309, + "loss": 1.5318, + "theoretical_loss": 3.9556779535616444, + "tokens_seen": 485621760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006891554663991977, + "loss": 1.5957, + "theoretical_loss": 3.955620586376633, + "tokens_seen": 485687296 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006891394182547643, + "loss": 1.5194, + "theoretical_loss": 3.9555632290990097, + "tokens_seen": 485752832 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006891233701103311, + "loss": 1.6338, + "theoretical_loss": 3.9555058817257285, + "tokens_seen": 485818368 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006891073219658977, + "loss": 1.5568, + "theoretical_loss": 3.9554485442537417, + "tokens_seen": 485883904 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006890912738214645, + "loss": 1.4295, + "theoretical_loss": 3.955391216680006, + "tokens_seen": 485949440 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006890752256770311, + "loss": 1.6004, + "theoretical_loss": 3.9553338990014772, + "tokens_seen": 486014976 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006890591775325979, + "loss": 1.5645, + "theoretical_loss": 3.9552765912151138, + "tokens_seen": 486080512 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006890431293881645, + "loss": 1.5962, + "theoretical_loss": 3.9552192933178745, + "tokens_seen": 486146048 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006890270812437313, + "loss": 1.5538, + "theoretical_loss": 3.9551620053067214, + "tokens_seen": 486211584 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006890110330992979, + "loss": 1.4446, + "theoretical_loss": 3.9551047271786155, + "tokens_seen": 486277120 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006889949849548647, + "loss": 1.7009, + "theoretical_loss": 3.9550474589305202, + "tokens_seen": 486342656 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006889789368104313, + "loss": 1.5374, + "theoretical_loss": 3.954990200559401, + "tokens_seen": 486408192 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006889628886659981, + "loss": 1.5383, + "theoretical_loss": 3.9549329520622236, + "tokens_seen": 486473728 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006889468405215647, + "loss": 1.4603, + "theoretical_loss": 3.954875713435955, + "tokens_seen": 486539264 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006889307923771315, + "loss": 1.5304, + "theoretical_loss": 3.9548184846775647, + "tokens_seen": 486604800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006889147442326981, + "loss": 1.5607, + "theoretical_loss": 3.9547612657840228, + "tokens_seen": 486670336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006888986960882649, + "loss": 1.5535, + "theoretical_loss": 3.9547040567523, + "tokens_seen": 486735872 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006888826479438315, + "loss": 1.6071, + "theoretical_loss": 3.9546468575793696, + "tokens_seen": 486801408 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006888665997993983, + "loss": 1.5729, + "theoretical_loss": 3.9545896682622064, + "tokens_seen": 486866944 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688850551654965, + "loss": 1.5108, + "theoretical_loss": 3.954532488797785, + "tokens_seen": 486932480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006888345035105317, + "loss": 1.5062, + "theoretical_loss": 3.954475319183082, + "tokens_seen": 486998016 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006888184553660983, + "loss": 1.5423, + "theoretical_loss": 3.9544181594150762, + "tokens_seen": 487063552 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688802407221665, + "loss": 1.5793, + "theoretical_loss": 3.9543610094907464, + "tokens_seen": 487129088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006887863590772317, + "loss": 1.4452, + "theoretical_loss": 3.9543038694070747, + "tokens_seen": 487194624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006887703109327984, + "loss": 1.5008, + "theoretical_loss": 3.9542467391610416, + "tokens_seen": 487260160 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006887542627883652, + "loss": 1.5913, + "theoretical_loss": 3.9541896187496315, + "tokens_seen": 487325696 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006887382146439318, + "loss": 1.4978, + "theoretical_loss": 3.9541325081698293, + "tokens_seen": 487391232 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006887221664994986, + "loss": 1.5564, + "theoretical_loss": 3.954075407418621, + "tokens_seen": 487456768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006887061183550652, + "loss": 1.4444, + "theoretical_loss": 3.954018316492994, + "tokens_seen": 487522304 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006886900702106318, + "loss": 1.4523, + "theoretical_loss": 3.9539612353899365, + "tokens_seen": 487587840 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006886740220661986, + "loss": 1.5001, + "theoretical_loss": 3.9539041641064396, + "tokens_seen": 487653376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006886579739217652, + "loss": 1.4753, + "theoretical_loss": 3.953847102639494, + "tokens_seen": 487718912 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688641925777332, + "loss": 1.5952, + "theoretical_loss": 3.9537900509860933, + "tokens_seen": 487784448 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006886258776328986, + "loss": 1.6444, + "theoretical_loss": 3.953733009143231, + "tokens_seen": 487849984 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006886098294884654, + "loss": 1.4986, + "theoretical_loss": 3.9536759771079018, + "tokens_seen": 487915520 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688593781344032, + "loss": 1.5359, + "theoretical_loss": 3.9536189548771037, + "tokens_seen": 487981056 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006885777331995988, + "loss": 1.5546, + "theoretical_loss": 3.953561942447834, + "tokens_seen": 488046592 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006885616850551655, + "loss": 1.5107, + "theoretical_loss": 3.953504939817093, + "tokens_seen": 488112128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006885456369107322, + "loss": 1.5266, + "theoretical_loss": 3.953447946981881, + "tokens_seen": 488177664 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.9882850646972656, + "objective/train/avg_token_score": 0.011714895255863667, + "objective/train/docs_used": 183865, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6134263277053833, + "objective/train/original_loss": 1.6356902122497559, + "objective/train/theoretical_loss": 3.9533909639391993, + "objective/train/tokens_used": 508703200, + "objective/train/unlikelihood_loss": 0.025736918672919273, + "theoretical_loss": 3.9533909639391993, + "tokens_seen": 488243200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006885295887662989, + "loss": 1.6269, + "theoretical_loss": 3.9533909639391993, + "tokens_seen": 488243200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006885135406218656, + "loss": 1.5057, + "theoretical_loss": 3.953333990686051, + "tokens_seen": 488308736 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006884974924774323, + "loss": 1.5679, + "theoretical_loss": 3.9532770272194426, + "tokens_seen": 488374272 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688481444332999, + "loss": 1.5142, + "theoretical_loss": 3.9532200735363787, + "tokens_seen": 488439808 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006884653961885657, + "loss": 1.6225, + "theoretical_loss": 3.953163129633867, + "tokens_seen": 488505344 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006884493480441324, + "loss": 1.3966, + "theoretical_loss": 3.9531061955089157, + "tokens_seen": 488570880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006884332998996991, + "loss": 1.5123, + "theoretical_loss": 3.9530492711585357, + "tokens_seen": 488636416 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006884172517552658, + "loss": 1.5517, + "theoretical_loss": 3.9529923565797374, + "tokens_seen": 488701952 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006884012036108325, + "loss": 1.4818, + "theoretical_loss": 3.952935451769533, + "tokens_seen": 488767488 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006883851554663992, + "loss": 1.5181, + "theoretical_loss": 3.9528785567249374, + "tokens_seen": 488833024 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006883691073219659, + "loss": 1.5809, + "theoretical_loss": 3.9528216714429654, + "tokens_seen": 488898560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006883530591775326, + "loss": 1.5641, + "theoretical_loss": 3.9527647959206336, + "tokens_seen": 488964096 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006883370110330993, + "loss": 1.566, + "theoretical_loss": 3.9527079301549595, + "tokens_seen": 489029632 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688320962888666, + "loss": 1.5711, + "theoretical_loss": 3.9526510741429624, + "tokens_seen": 489095168 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006883049147442327, + "loss": 1.5573, + "theoretical_loss": 3.9525942278816633, + "tokens_seen": 489160704 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006882888665997994, + "loss": 1.5914, + "theoretical_loss": 3.952537391368083, + "tokens_seen": 489226240 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006882728184553662, + "loss": 1.4212, + "theoretical_loss": 3.9524805645992447, + "tokens_seen": 489291776 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006882567703109328, + "loss": 1.5583, + "theoretical_loss": 3.9524237475721726, + "tokens_seen": 489357312 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006882407221664996, + "loss": 1.5953, + "theoretical_loss": 3.952366940283894, + "tokens_seen": 489422848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006882246740220662, + "loss": 1.548, + "theoretical_loss": 3.952310142731434, + "tokens_seen": 489488384 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688208625877633, + "loss": 1.5811, + "theoretical_loss": 3.9522533549118215, + "tokens_seen": 489553920 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006881925777331996, + "loss": 1.5023, + "theoretical_loss": 3.952196576822086, + "tokens_seen": 489619456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006881765295887664, + "loss": 1.6776, + "theoretical_loss": 3.9521398084592585, + "tokens_seen": 489684992 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688160481444333, + "loss": 1.6322, + "theoretical_loss": 3.952083049820372, + "tokens_seen": 489750528 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006881444332998998, + "loss": 1.4911, + "theoretical_loss": 3.9520263009024585, + "tokens_seen": 489816064 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006881283851554664, + "loss": 1.5957, + "theoretical_loss": 3.951969561702554, + "tokens_seen": 489881600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006881123370110332, + "loss": 1.6297, + "theoretical_loss": 3.9519128322176935, + "tokens_seen": 489947136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006880962888665998, + "loss": 1.6737, + "theoretical_loss": 3.951856112444915, + "tokens_seen": 490012672 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006880802407221666, + "loss": 1.5588, + "theoretical_loss": 3.951799402381258, + "tokens_seen": 490078208 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006880641925777332, + "loss": 1.4781, + "theoretical_loss": 3.951742702023761, + "tokens_seen": 490143744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006880481444333, + "loss": 1.4785, + "theoretical_loss": 3.951686011369466, + "tokens_seen": 490209280 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006880320962888666, + "loss": 1.6037, + "theoretical_loss": 3.9516293304154155, + "tokens_seen": 490274816 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006880160481444334, + "loss": 1.5447, + "theoretical_loss": 3.9515726591586544, + "tokens_seen": 490340352 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688, + "loss": 1.6025, + "theoretical_loss": 3.9515159975962257, + "tokens_seen": 490405888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006879839518555668, + "loss": 1.4327, + "theoretical_loss": 3.951459345725178, + "tokens_seen": 490471424 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006879679037111334, + "loss": 1.5089, + "theoretical_loss": 3.9514027035425583, + "tokens_seen": 490536960 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006879518555667002, + "loss": 1.5929, + "theoretical_loss": 3.951346071045415, + "tokens_seen": 490602496 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006879358074222668, + "loss": 1.5612, + "theoretical_loss": 3.951289448230799, + "tokens_seen": 490668032 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006879197592778336, + "loss": 1.5201, + "theoretical_loss": 3.9512328350957624, + "tokens_seen": 490733568 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006879037111334002, + "loss": 1.5783, + "theoretical_loss": 3.9511762316373575, + "tokens_seen": 490799104 + }, + { + "epoch": 0.15, + "learning_rate": 0.000687887662988967, + "loss": 1.6334, + "theoretical_loss": 3.951119637852639, + "tokens_seen": 490864640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006878716148445336, + "loss": 1.5048, + "theoretical_loss": 3.951063053738662, + "tokens_seen": 490930176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006878555667001004, + "loss": 1.5661, + "theoretical_loss": 3.951006479292484, + "tokens_seen": 490995712 + }, + { + "epoch": 0.15, + "learning_rate": 0.000687839518555667, + "loss": 1.5929, + "theoretical_loss": 3.9509499145111624, + "tokens_seen": 491061248 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006878234704112338, + "loss": 1.5639, + "theoretical_loss": 3.950893359391756, + "tokens_seen": 491126784 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006878074222668004, + "loss": 1.4534, + "theoretical_loss": 3.9508368139313275, + "tokens_seen": 491192320 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006877913741223672, + "loss": 1.5595, + "theoretical_loss": 3.950780278126937, + "tokens_seen": 491257856 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006877753259779338, + "loss": 1.5883, + "theoretical_loss": 3.9507237519756484, + "tokens_seen": 491323392 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006877592778335006, + "loss": 1.5606, + "theoretical_loss": 3.9506672354745263, + "tokens_seen": 491388928 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006877432296890672, + "loss": 1.5476, + "theoretical_loss": 3.950610728620636, + "tokens_seen": 491454464 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.9913667440414429, + "objective/train/avg_token_score": 0.008633281104266644, + "objective/train/docs_used": 185162, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5210930109024048, + "objective/train/original_loss": 1.5357173681259155, + "objective/train/theoretical_loss": 3.950554231411046, + "objective/train/tokens_used": 511980000, + "objective/train/unlikelihood_loss": 0.021009406074881554, + "theoretical_loss": 3.950554231411046, + "tokens_seen": 491520000 + }, + { + "epoch": 0.15, + "learning_rate": 0.000687727181544634, + "loss": 1.5612, + "theoretical_loss": 3.950554231411046, + "tokens_seen": 491520000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006877111334002006, + "loss": 1.5042, + "theoretical_loss": 3.9504977438428233, + "tokens_seen": 491585536 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006876950852557674, + "loss": 1.4886, + "theoretical_loss": 3.950441265913038, + "tokens_seen": 491651072 + }, + { + "epoch": 0.15, + "learning_rate": 0.000687679037111334, + "loss": 1.4465, + "theoretical_loss": 3.950384797618761, + "tokens_seen": 491716608 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006876629889669008, + "loss": 1.6149, + "theoretical_loss": 3.950328338957064, + "tokens_seen": 491782144 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006876469408224674, + "loss": 1.5202, + "theoretical_loss": 3.950271889925022, + "tokens_seen": 491847680 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006876308926780342, + "loss": 1.6271, + "theoretical_loss": 3.9502154505197087, + "tokens_seen": 491913216 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006876148445336008, + "loss": 1.6449, + "theoretical_loss": 3.9501590207382, + "tokens_seen": 491978752 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006875987963891676, + "loss": 1.5816, + "theoretical_loss": 3.9501026005775732, + "tokens_seen": 492044288 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006875827482447342, + "loss": 1.5579, + "theoretical_loss": 3.9500461900349078, + "tokens_seen": 492109824 + }, + { + "epoch": 0.15, + "learning_rate": 0.000687566700100301, + "loss": 1.5711, + "theoretical_loss": 3.949989789107283, + "tokens_seen": 492175360 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006875506519558676, + "loss": 1.5289, + "theoretical_loss": 3.9499333977917805, + "tokens_seen": 492240896 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006875346038114344, + "loss": 1.6457, + "theoretical_loss": 3.949877016085482, + "tokens_seen": 492306432 + }, + { + "epoch": 0.15, + "learning_rate": 0.000687518555667001, + "loss": 1.5597, + "theoretical_loss": 3.949820643985472, + "tokens_seen": 492371968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006875025075225678, + "loss": 1.6282, + "theoretical_loss": 3.9497642814888345, + "tokens_seen": 492437504 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006874864593781344, + "loss": 1.5307, + "theoretical_loss": 3.949707928592657, + "tokens_seen": 492503040 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006874704112337012, + "loss": 1.6246, + "theoretical_loss": 3.949651585294026, + "tokens_seen": 492568576 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006874543630892678, + "loss": 1.6106, + "theoretical_loss": 3.9495952515900306, + "tokens_seen": 492634112 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006874383149448346, + "loss": 1.6034, + "theoretical_loss": 3.9495389274777613, + "tokens_seen": 492699648 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006874222668004012, + "loss": 1.4389, + "theoretical_loss": 3.9494826129543084, + "tokens_seen": 492765184 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006874062186559679, + "loss": 1.6155, + "theoretical_loss": 3.9494263080167658, + "tokens_seen": 492830720 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006873901705115347, + "loss": 1.6199, + "theoretical_loss": 3.949370012662227, + "tokens_seen": 492896256 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006873741223671013, + "loss": 1.5476, + "theoretical_loss": 3.9493137268877865, + "tokens_seen": 492961792 + }, + { + "epoch": 0.15, + "learning_rate": 0.000687358074222668, + "loss": 1.5734, + "theoretical_loss": 3.9492574506905416, + "tokens_seen": 493027328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006873420260782347, + "loss": 1.4993, + "theoretical_loss": 3.949201184067589, + "tokens_seen": 493092864 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006873259779338015, + "loss": 1.6043, + "theoretical_loss": 3.949144927016029, + "tokens_seen": 493158400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006873099297893681, + "loss": 1.584, + "theoretical_loss": 3.9490886795329603, + "tokens_seen": 493223936 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006872938816449349, + "loss": 1.5386, + "theoretical_loss": 3.9490324416154854, + "tokens_seen": 493289472 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006872778335005015, + "loss": 1.5981, + "theoretical_loss": 3.9489762132607065, + "tokens_seen": 493355008 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006872617853560681, + "loss": 1.5813, + "theoretical_loss": 3.948919994465728, + "tokens_seen": 493420544 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006872457372116349, + "loss": 1.7252, + "theoretical_loss": 3.9488637852276547, + "tokens_seen": 493486080 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006872296890672015, + "loss": 1.6184, + "theoretical_loss": 3.9488075855435936, + "tokens_seen": 493551616 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006872136409227683, + "loss": 1.7098, + "theoretical_loss": 3.9487513954106523, + "tokens_seen": 493617152 + }, + { + "epoch": 0.15, + "learning_rate": 0.000687197592778335, + "loss": 1.5674, + "theoretical_loss": 3.94869521482594, + "tokens_seen": 493682688 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006871815446339017, + "loss": 1.4995, + "theoretical_loss": 3.9486390437865664, + "tokens_seen": 493748224 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006871654964894684, + "loss": 1.5463, + "theoretical_loss": 3.948582882289644, + "tokens_seen": 493813760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006871494483450351, + "loss": 1.558, + "theoretical_loss": 3.9485267303322846, + "tokens_seen": 493879296 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006871334002006018, + "loss": 1.4686, + "theoretical_loss": 3.9484705879116033, + "tokens_seen": 493944832 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006871173520561685, + "loss": 1.4498, + "theoretical_loss": 3.9484144550247144, + "tokens_seen": 494010368 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006871013039117352, + "loss": 1.6635, + "theoretical_loss": 3.948358331668735, + "tokens_seen": 494075904 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006870852557673019, + "loss": 1.5325, + "theoretical_loss": 3.9483022178407827, + "tokens_seen": 494141440 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006870692076228687, + "loss": 1.6892, + "theoretical_loss": 3.9482461135379774, + "tokens_seen": 494206976 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006870531594784353, + "loss": 1.5811, + "theoretical_loss": 3.9481900187574386, + "tokens_seen": 494272512 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006870371113340021, + "loss": 1.5704, + "theoretical_loss": 3.948133933496288, + "tokens_seen": 494338048 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006870210631895687, + "loss": 1.5292, + "theoretical_loss": 3.948077857751649, + "tokens_seen": 494403584 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006870050150451355, + "loss": 1.6231, + "theoretical_loss": 3.948021791520645, + "tokens_seen": 494469120 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006869889669007021, + "loss": 1.6211, + "theoretical_loss": 3.9479657348004014, + "tokens_seen": 494534656 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006869729187562689, + "loss": 1.6319, + "theoretical_loss": 3.9479096875880453, + "tokens_seen": 494600192 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006869568706118355, + "loss": 1.7346, + "theoretical_loss": 3.947853649880704, + "tokens_seen": 494665728 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006869408224674023, + "loss": 1.5692, + "theoretical_loss": 3.947797621675507, + "tokens_seen": 494731264 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.9944104552268982, + "objective/train/avg_token_score": 0.005589539185166359, + "objective/train/docs_used": 186433, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6102079153060913, + "objective/train/original_loss": 1.6202467679977417, + "objective/train/theoretical_loss": 3.947741602969585, + "objective/train/tokens_used": 515256800, + "objective/train/unlikelihood_loss": 0.011375971138477325, + "theoretical_loss": 3.947741602969585, + "tokens_seen": 494796800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006869247743229689, + "loss": 1.5114, + "theoretical_loss": 3.947741602969585, + "tokens_seen": 494796800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006869087261785357, + "loss": 1.4568, + "theoretical_loss": 3.9476855937600686, + "tokens_seen": 494862336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006868926780341023, + "loss": 1.4422, + "theoretical_loss": 3.9476295940440913, + "tokens_seen": 494927872 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006868766298896691, + "loss": 1.5957, + "theoretical_loss": 3.9475736038187867, + "tokens_seen": 494993408 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006868605817452357, + "loss": 1.5116, + "theoretical_loss": 3.947517623081291, + "tokens_seen": 495058944 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006868445336008025, + "loss": 1.5633, + "theoretical_loss": 3.9474616518287395, + "tokens_seen": 495124480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006868284854563691, + "loss": 1.5785, + "theoretical_loss": 3.9474056900582717, + "tokens_seen": 495190016 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006868124373119359, + "loss": 1.6838, + "theoretical_loss": 3.9473497377670244, + "tokens_seen": 495255552 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006867963891675025, + "loss": 1.7043, + "theoretical_loss": 3.94729379495214, + "tokens_seen": 495321088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006867803410230693, + "loss": 1.5211, + "theoretical_loss": 3.947237861610759, + "tokens_seen": 495386624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006867642928786359, + "loss": 1.568, + "theoretical_loss": 3.9471819377400243, + "tokens_seen": 495452160 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006867482447342027, + "loss": 1.4779, + "theoretical_loss": 3.94712602333708, + "tokens_seen": 495517696 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006867321965897693, + "loss": 1.4567, + "theoretical_loss": 3.9470701183990715, + "tokens_seen": 495583232 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006867161484453361, + "loss": 1.6471, + "theoretical_loss": 3.947014222923145, + "tokens_seen": 495648768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006867001003009027, + "loss": 1.5339, + "theoretical_loss": 3.946958336906449, + "tokens_seen": 495714304 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006866840521564695, + "loss": 1.4494, + "theoretical_loss": 3.9469024603461307, + "tokens_seen": 495779840 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006866680040120361, + "loss": 1.6979, + "theoretical_loss": 3.946846593239342, + "tokens_seen": 495845376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006866519558676029, + "loss": 1.6298, + "theoretical_loss": 3.946790735583234, + "tokens_seen": 495910912 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006866359077231695, + "loss": 1.5067, + "theoretical_loss": 3.9467348873749586, + "tokens_seen": 495976448 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006866198595787363, + "loss": 1.6159, + "theoretical_loss": 3.9466790486116707, + "tokens_seen": 496041984 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006866038114343029, + "loss": 1.5913, + "theoretical_loss": 3.946623219290525, + "tokens_seen": 496107520 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006865877632898697, + "loss": 1.5442, + "theoretical_loss": 3.9465673994086776, + "tokens_seen": 496173056 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006865717151454363, + "loss": 1.5989, + "theoretical_loss": 3.9465115889632862, + "tokens_seen": 496238592 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006865556670010031, + "loss": 1.6031, + "theoretical_loss": 3.9464557879515105, + "tokens_seen": 496304128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006865396188565697, + "loss": 1.488, + "theoretical_loss": 3.946399996370509, + "tokens_seen": 496369664 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006865235707121365, + "loss": 1.5689, + "theoretical_loss": 3.9463442142174445, + "tokens_seen": 496435200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006865075225677031, + "loss": 1.6294, + "theoretical_loss": 3.9462884414894788, + "tokens_seen": 496500736 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006864914744232699, + "loss": 1.5142, + "theoretical_loss": 3.946232678183775, + "tokens_seen": 496566272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006864754262788365, + "loss": 1.5159, + "theoretical_loss": 3.9461769242974998, + "tokens_seen": 496631808 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006864593781344033, + "loss": 1.6194, + "theoretical_loss": 3.946121179827818, + "tokens_seen": 496697344 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006864433299899699, + "loss": 1.6175, + "theoretical_loss": 3.9460654447718975, + "tokens_seen": 496762880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006864272818455367, + "loss": 1.5936, + "theoretical_loss": 3.9460097191269066, + "tokens_seen": 496828416 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006864112337011033, + "loss": 1.5744, + "theoretical_loss": 3.9459540028900157, + "tokens_seen": 496893952 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006863951855566701, + "loss": 1.5616, + "theoretical_loss": 3.945898296058396, + "tokens_seen": 496959488 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006863791374122367, + "loss": 1.5956, + "theoretical_loss": 3.9458425986292194, + "tokens_seen": 497025024 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006863630892678035, + "loss": 1.5513, + "theoretical_loss": 3.9457869105996597, + "tokens_seen": 497090560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006863470411233701, + "loss": 1.5165, + "theoretical_loss": 3.9457312319668914, + "tokens_seen": 497156096 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006863309929789369, + "loss": 1.622, + "theoretical_loss": 3.9456755627280904, + "tokens_seen": 497221632 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006863149448345035, + "loss": 1.4375, + "theoretical_loss": 3.945619902880434, + "tokens_seen": 497287168 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006862988966900703, + "loss": 1.65, + "theoretical_loss": 3.945564252421102, + "tokens_seen": 497352704 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006862828485456369, + "loss": 1.6672, + "theoretical_loss": 3.9455086113472717, + "tokens_seen": 497418240 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006862668004012037, + "loss": 1.5358, + "theoretical_loss": 3.9454529796561255, + "tokens_seen": 497483776 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006862507522567703, + "loss": 1.5884, + "theoretical_loss": 3.9453973573448455, + "tokens_seen": 497549312 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006862347041123371, + "loss": 1.5324, + "theoretical_loss": 3.945341744410615, + "tokens_seen": 497614848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006862186559679037, + "loss": 1.5488, + "theoretical_loss": 3.9452861408506172, + "tokens_seen": 497680384 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006862026078234705, + "loss": 1.5801, + "theoretical_loss": 3.945230546662039, + "tokens_seen": 497745920 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006861865596790371, + "loss": 1.5242, + "theoretical_loss": 3.945174961842068, + "tokens_seen": 497811456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006861705115346039, + "loss": 1.5945, + "theoretical_loss": 3.9451193863878906, + "tokens_seen": 497876992 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006861544633901705, + "loss": 1.5487, + "theoretical_loss": 3.945063820296698, + "tokens_seen": 497942528 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006861384152457373, + "loss": 1.604, + "theoretical_loss": 3.9450082635656796, + "tokens_seen": 498008064 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.9921091198921204, + "objective/train/avg_token_score": 0.007890935987234116, + "objective/train/docs_used": 187487, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6623060703277588, + "objective/train/original_loss": 1.674206018447876, + "objective/train/theoretical_loss": 3.9449527161920273, + "objective/train/tokens_used": 518533600, + "objective/train/unlikelihood_loss": 0.020677069202065468, + "theoretical_loss": 3.9449527161920273, + "tokens_seen": 498073600 + }, + { + "epoch": 0.15, + "learning_rate": 0.000686122367101304, + "loss": 1.6232, + "theoretical_loss": 3.9449527161920273, + "tokens_seen": 498073600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006861063189568707, + "loss": 1.5237, + "theoretical_loss": 3.944897178172935, + "tokens_seen": 498139136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006860902708124373, + "loss": 1.6532, + "theoretical_loss": 3.9448416495055962, + "tokens_seen": 498204672 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006860742226680041, + "loss": 1.584, + "theoretical_loss": 3.9447861301872065, + "tokens_seen": 498270208 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006860581745235707, + "loss": 1.4375, + "theoretical_loss": 3.944730620214962, + "tokens_seen": 498335744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006860421263791375, + "loss": 1.4799, + "theoretical_loss": 3.944675119586061, + "tokens_seen": 498401280 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006860260782347042, + "loss": 1.5722, + "theoretical_loss": 3.944619628297704, + "tokens_seen": 498466816 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006860100300902708, + "loss": 1.5564, + "theoretical_loss": 3.944564146347089, + "tokens_seen": 498532352 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006859939819458376, + "loss": 1.4437, + "theoretical_loss": 3.9445086737314186, + "tokens_seen": 498597888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006859779338014042, + "loss": 1.5943, + "theoretical_loss": 3.9444532104478958, + "tokens_seen": 498663424 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685961885656971, + "loss": 1.5927, + "theoretical_loss": 3.9443977564937236, + "tokens_seen": 498728960 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006859458375125376, + "loss": 1.5887, + "theoretical_loss": 3.9443423118661074, + "tokens_seen": 498794496 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006859297893681044, + "loss": 1.5475, + "theoretical_loss": 3.944286876562254, + "tokens_seen": 498860032 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685913741223671, + "loss": 1.4675, + "theoretical_loss": 3.9442314505793705, + "tokens_seen": 498925568 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006858976930792378, + "loss": 1.6537, + "theoretical_loss": 3.944176033914666, + "tokens_seen": 498991104 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006858816449348044, + "loss": 1.6003, + "theoretical_loss": 3.9441206265653497, + "tokens_seen": 499056640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006858655967903712, + "loss": 1.5288, + "theoretical_loss": 3.9440652285286335, + "tokens_seen": 499122176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006858495486459378, + "loss": 1.5852, + "theoretical_loss": 3.9440098398017294, + "tokens_seen": 499187712 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006858335005015046, + "loss": 1.5437, + "theoretical_loss": 3.9439544603818506, + "tokens_seen": 499253248 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006858174523570712, + "loss": 1.6421, + "theoretical_loss": 3.9438990902662128, + "tokens_seen": 499318784 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685801404212638, + "loss": 1.5686, + "theoretical_loss": 3.9438437294520305, + "tokens_seen": 499384320 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006857853560682046, + "loss": 1.6916, + "theoretical_loss": 3.943788377936522, + "tokens_seen": 499449856 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006857693079237714, + "loss": 1.5687, + "theoretical_loss": 3.943733035716905, + "tokens_seen": 499515392 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685753259779338, + "loss": 1.518, + "theoretical_loss": 3.9436777027903993, + "tokens_seen": 499580928 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006857372116349048, + "loss": 1.6936, + "theoretical_loss": 3.9436223791542258, + "tokens_seen": 499646464 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006857211634904714, + "loss": 1.4983, + "theoretical_loss": 3.943567064805606, + "tokens_seen": 499712000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006857051153460382, + "loss": 1.4589, + "theoretical_loss": 3.943511759741763, + "tokens_seen": 499777536 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006856890672016048, + "loss": 1.4722, + "theoretical_loss": 3.9434564639599214, + "tokens_seen": 499843072 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006856730190571716, + "loss": 1.5716, + "theoretical_loss": 3.9434011774573063, + "tokens_seen": 499908608 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006856569709127382, + "loss": 1.5179, + "theoretical_loss": 3.9433459002311446, + "tokens_seen": 499974144 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685640922768305, + "loss": 1.5156, + "theoretical_loss": 3.9432906322786643, + "tokens_seen": 500039680 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006856248746238716, + "loss": 1.5588, + "theoretical_loss": 3.943235373597094, + "tokens_seen": 500105216 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006856088264794384, + "loss": 1.6363, + "theoretical_loss": 3.943180124183665, + "tokens_seen": 500170752 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685592778335005, + "loss": 1.5679, + "theoretical_loss": 3.9431248840356075, + "tokens_seen": 500236288 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006855767301905718, + "loss": 1.597, + "theoretical_loss": 3.943069653150155, + "tokens_seen": 500301824 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006855606820461384, + "loss": 1.5658, + "theoretical_loss": 3.9430144315245412, + "tokens_seen": 500367360 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006855446339017052, + "loss": 1.5813, + "theoretical_loss": 3.942959219156, + "tokens_seen": 500432896 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006855285857572718, + "loss": 1.6832, + "theoretical_loss": 3.942904016041769, + "tokens_seen": 500498432 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006855125376128386, + "loss": 1.6019, + "theoretical_loss": 3.9428488221790854, + "tokens_seen": 500563968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006854964894684052, + "loss": 1.6417, + "theoretical_loss": 3.9427936375651877, + "tokens_seen": 500629504 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685480441323972, + "loss": 1.5524, + "theoretical_loss": 3.9427384621973154, + "tokens_seen": 500695040 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006854643931795386, + "loss": 1.5136, + "theoretical_loss": 3.9426832960727087, + "tokens_seen": 500760576 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006854483450351054, + "loss": 1.4588, + "theoretical_loss": 3.9426281391886113, + "tokens_seen": 500826112 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685432296890672, + "loss": 1.5508, + "theoretical_loss": 3.9425729915422654, + "tokens_seen": 500891648 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006854162487462388, + "loss": 1.5099, + "theoretical_loss": 3.942517853130916, + "tokens_seen": 500957184 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006854002006018054, + "loss": 1.5182, + "theoretical_loss": 3.9424627239518086, + "tokens_seen": 501022720 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006853841524573722, + "loss": 1.6525, + "theoretical_loss": 3.9424076040021903, + "tokens_seen": 501088256 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006853681043129388, + "loss": 1.5532, + "theoretical_loss": 3.9423524932793086, + "tokens_seen": 501153792 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006853520561685056, + "loss": 1.5913, + "theoretical_loss": 3.9422973917804134, + "tokens_seen": 501219328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006853360080240722, + "loss": 1.5524, + "theoretical_loss": 3.9422422995027544, + "tokens_seen": 501284864 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.9944915175437927, + "objective/train/avg_token_score": 0.005508484318852425, + "objective/train/docs_used": 188727, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4089889526367188, + "objective/train/original_loss": 1.4180376529693604, + "objective/train/theoretical_loss": 3.9421872164435836, + "objective/train/tokens_used": 521810400, + "objective/train/unlikelihood_loss": 0.013741646893322468, + "theoretical_loss": 3.9421872164435836, + "tokens_seen": 501350400 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685319959879639, + "loss": 1.5426, + "theoretical_loss": 3.9421872164435836, + "tokens_seen": 501350400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006853039117352056, + "loss": 1.4888, + "theoretical_loss": 3.942132142600154, + "tokens_seen": 501415936 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006852878635907724, + "loss": 1.5358, + "theoretical_loss": 3.942077077969719, + "tokens_seen": 501481472 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685271815446339, + "loss": 1.653, + "theoretical_loss": 3.9420220225495335, + "tokens_seen": 501547008 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006852557673019058, + "loss": 1.4765, + "theoretical_loss": 3.9419669763368548, + "tokens_seen": 501612544 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006852397191574724, + "loss": 1.5674, + "theoretical_loss": 3.9419119393289397, + "tokens_seen": 501678080 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006852236710130392, + "loss": 1.482, + "theoretical_loss": 3.941856911523047, + "tokens_seen": 501743616 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006852076228686058, + "loss": 1.508, + "theoretical_loss": 3.9418018929164362, + "tokens_seen": 501809152 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006851915747241726, + "loss": 1.573, + "theoretical_loss": 3.9417468835063687, + "tokens_seen": 501874688 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006851755265797392, + "loss": 1.5934, + "theoretical_loss": 3.9416918832901064, + "tokens_seen": 501940224 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685159478435306, + "loss": 1.5648, + "theoretical_loss": 3.9416368922649134, + "tokens_seen": 502005760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006851434302908726, + "loss": 1.5155, + "theoretical_loss": 3.9415819104280527, + "tokens_seen": 502071296 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006851273821464394, + "loss": 1.5688, + "theoretical_loss": 3.941526937776791, + "tokens_seen": 502136832 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685111334002006, + "loss": 1.4242, + "theoretical_loss": 3.941471974308395, + "tokens_seen": 502202368 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006850952858575728, + "loss": 1.6615, + "theoretical_loss": 3.9414170200201326, + "tokens_seen": 502267904 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006850792377131394, + "loss": 1.4877, + "theoretical_loss": 3.9413620749092733, + "tokens_seen": 502333440 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006850631895687062, + "loss": 1.4476, + "theoretical_loss": 3.9413071389730874, + "tokens_seen": 502398976 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006850471414242728, + "loss": 1.4549, + "theoretical_loss": 3.941252212208846, + "tokens_seen": 502464512 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006850310932798396, + "loss": 1.564, + "theoretical_loss": 3.941197294613822, + "tokens_seen": 502530048 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006850150451354062, + "loss": 1.5794, + "theoretical_loss": 3.9411423861852892, + "tokens_seen": 502595584 + }, + { + "epoch": 0.15, + "learning_rate": 0.000684998996990973, + "loss": 1.5685, + "theoretical_loss": 3.941087486920523, + "tokens_seen": 502661120 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006849829488465396, + "loss": 1.5833, + "theoretical_loss": 3.9410325968167994, + "tokens_seen": 502726656 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006849669007021064, + "loss": 1.5003, + "theoretical_loss": 3.9409777158713952, + "tokens_seen": 502792192 + }, + { + "epoch": 0.15, + "learning_rate": 0.000684950852557673, + "loss": 1.5791, + "theoretical_loss": 3.9409228440815895, + "tokens_seen": 502857728 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006849348044132398, + "loss": 1.6204, + "theoretical_loss": 3.9408679814446623, + "tokens_seen": 502923264 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006849187562688064, + "loss": 1.5394, + "theoretical_loss": 3.9408131279578935, + "tokens_seen": 502988800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006849027081243732, + "loss": 1.4883, + "theoretical_loss": 3.9407582836185657, + "tokens_seen": 503054336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006848866599799398, + "loss": 1.6258, + "theoretical_loss": 3.940703448423962, + "tokens_seen": 503119872 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006848706118355066, + "loss": 1.6503, + "theoretical_loss": 3.9406486223713664, + "tokens_seen": 503185408 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006848545636910732, + "loss": 1.6039, + "theoretical_loss": 3.940593805458065, + "tokens_seen": 503250944 + }, + { + "epoch": 0.15, + "learning_rate": 0.00068483851554664, + "loss": 1.5306, + "theoretical_loss": 3.940538997681344, + "tokens_seen": 503316480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006848224674022066, + "loss": 1.5564, + "theoretical_loss": 3.9404841990384916, + "tokens_seen": 503382016 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006848064192577734, + "loss": 1.5636, + "theoretical_loss": 3.940429409526796, + "tokens_seen": 503447552 + }, + { + "epoch": 0.15, + "learning_rate": 0.00068479037111334, + "loss": 1.5932, + "theoretical_loss": 3.940374629143548, + "tokens_seen": 503513088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006847743229689068, + "loss": 1.6124, + "theoretical_loss": 3.940319857886039, + "tokens_seen": 503578624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006847582748244734, + "loss": 1.4446, + "theoretical_loss": 3.9402650957515606, + "tokens_seen": 503644160 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006847422266800402, + "loss": 1.5724, + "theoretical_loss": 3.940210342737407, + "tokens_seen": 503709696 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006847261785356068, + "loss": 1.5277, + "theoretical_loss": 3.940155598840873, + "tokens_seen": 503775232 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006847101303911736, + "loss": 1.479, + "theoretical_loss": 3.9401008640592545, + "tokens_seen": 503840768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006846940822467402, + "loss": 1.557, + "theoretical_loss": 3.940046138389848, + "tokens_seen": 503906304 + }, + { + "epoch": 0.15, + "learning_rate": 0.000684678034102307, + "loss": 1.6554, + "theoretical_loss": 3.9399914218299523, + "tokens_seen": 503971840 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006846619859578737, + "loss": 1.7118, + "theoretical_loss": 3.9399367143768664, + "tokens_seen": 504037376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006846459378134404, + "loss": 1.5631, + "theoretical_loss": 3.939882016027891, + "tokens_seen": 504102912 + }, + { + "epoch": 0.15, + "learning_rate": 0.000684629889669007, + "loss": 1.5704, + "theoretical_loss": 3.939827326780328, + "tokens_seen": 504168448 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006846138415245737, + "loss": 1.6194, + "theoretical_loss": 3.9397726466314795, + "tokens_seen": 504233984 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006845977933801405, + "loss": 1.4953, + "theoretical_loss": 3.93971797557865, + "tokens_seen": 504299520 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006845817452357071, + "loss": 1.6176, + "theoretical_loss": 3.939663313619145, + "tokens_seen": 504365056 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006845656970912739, + "loss": 1.5137, + "theoretical_loss": 3.9396086607502694, + "tokens_seen": 504430592 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006845496489468405, + "loss": 1.5404, + "theoretical_loss": 3.939554016969332, + "tokens_seen": 504496128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006845336008024073, + "loss": 1.5469, + "theoretical_loss": 3.939499382273641, + "tokens_seen": 504561664 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.9919562339782715, + "objective/train/avg_token_score": 0.008043761365115643, + "objective/train/docs_used": 189911, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.429068684577942, + "objective/train/original_loss": 1.4424084424972534, + "objective/train/theoretical_loss": 3.939444756660505, + "objective/train/tokens_used": 525087200, + "objective/train/unlikelihood_loss": 0.019743306562304497, + "theoretical_loss": 3.939444756660505, + "tokens_seen": 504627200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006845175526579739, + "loss": 1.5333, + "theoretical_loss": 3.939444756660505, + "tokens_seen": 504627200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006845015045135407, + "loss": 1.5278, + "theoretical_loss": 3.9393901401272364, + "tokens_seen": 504692736 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006844854563691073, + "loss": 1.5457, + "theoretical_loss": 3.939335532671146, + "tokens_seen": 504758272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006844694082246741, + "loss": 1.5334, + "theoretical_loss": 3.939280934289548, + "tokens_seen": 504823808 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006844533600802407, + "loss": 1.5511, + "theoretical_loss": 3.9392263449797564, + "tokens_seen": 504889344 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006844373119358075, + "loss": 1.594, + "theoretical_loss": 3.939171764739086, + "tokens_seen": 504954880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006844212637913741, + "loss": 1.5337, + "theoretical_loss": 3.9391171935648535, + "tokens_seen": 505020416 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006844052156469409, + "loss": 1.617, + "theoretical_loss": 3.9390626314543766, + "tokens_seen": 505085952 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006843891675025075, + "loss": 1.7216, + "theoretical_loss": 3.939008078404975, + "tokens_seen": 505151488 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006843731193580743, + "loss": 1.5985, + "theoretical_loss": 3.938953534413968, + "tokens_seen": 505217024 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006843570712136409, + "loss": 1.4912, + "theoretical_loss": 3.9388989994786767, + "tokens_seen": 505282560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006843410230692077, + "loss": 1.5747, + "theoretical_loss": 3.938844473596423, + "tokens_seen": 505348096 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006843249749247743, + "loss": 1.5556, + "theoretical_loss": 3.9387899567645315, + "tokens_seen": 505413632 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006843089267803411, + "loss": 1.5394, + "theoretical_loss": 3.9387354489803252, + "tokens_seen": 505479168 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006842928786359077, + "loss": 1.5765, + "theoretical_loss": 3.938680950241131, + "tokens_seen": 505544704 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006842768304914745, + "loss": 1.5969, + "theoretical_loss": 3.938626460544275, + "tokens_seen": 505610240 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006842607823470411, + "loss": 1.6102, + "theoretical_loss": 3.9385719798870857, + "tokens_seen": 505675776 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006842447342026079, + "loss": 1.6559, + "theoretical_loss": 3.9385175082668926, + "tokens_seen": 505741312 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006842286860581745, + "loss": 1.564, + "theoretical_loss": 3.9384630456810243, + "tokens_seen": 505806848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006842126379137413, + "loss": 1.5728, + "theoretical_loss": 3.9384085921268133, + "tokens_seen": 505872384 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006841965897693079, + "loss": 1.5392, + "theoretical_loss": 3.9383541476015926, + "tokens_seen": 505937920 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006841805416248747, + "loss": 1.5764, + "theoretical_loss": 3.9382997121026944, + "tokens_seen": 506003456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006841644934804413, + "loss": 1.6386, + "theoretical_loss": 3.938245285627455, + "tokens_seen": 506068992 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006841484453360081, + "loss": 1.4917, + "theoretical_loss": 3.9381908681732085, + "tokens_seen": 506134528 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006841323971915747, + "loss": 1.6252, + "theoretical_loss": 3.9381364597372936, + "tokens_seen": 506200064 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006841163490471415, + "loss": 1.7044, + "theoretical_loss": 3.938082060317048, + "tokens_seen": 506265600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006841003009027081, + "loss": 1.6411, + "theoretical_loss": 3.9380276699098102, + "tokens_seen": 506331136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006840842527582749, + "loss": 1.4664, + "theoretical_loss": 3.9379732885129215, + "tokens_seen": 506396672 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006840682046138415, + "loss": 1.5316, + "theoretical_loss": 3.9379189161237234, + "tokens_seen": 506462208 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006840521564694083, + "loss": 1.659, + "theoretical_loss": 3.9378645527395575, + "tokens_seen": 506527744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006840361083249749, + "loss": 1.4817, + "theoretical_loss": 3.937810198357769, + "tokens_seen": 506593280 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006840200601805417, + "loss": 1.5467, + "theoretical_loss": 3.9377558529757026, + "tokens_seen": 506658816 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006840040120361083, + "loss": 1.6308, + "theoretical_loss": 3.937701516590703, + "tokens_seen": 506724352 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006839879638916751, + "loss": 1.5089, + "theoretical_loss": 3.9376471892001197, + "tokens_seen": 506789888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006839719157472417, + "loss": 1.6646, + "theoretical_loss": 3.9375928708012986, + "tokens_seen": 506855424 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006839558676028085, + "loss": 1.5922, + "theoretical_loss": 3.9375385613915905, + "tokens_seen": 506920960 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006839398194583751, + "loss": 1.5576, + "theoretical_loss": 3.937484260968346, + "tokens_seen": 506986496 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006839237713139419, + "loss": 1.6442, + "theoretical_loss": 3.9374299695289157, + "tokens_seen": 507052032 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006839077231695085, + "loss": 1.5733, + "theoretical_loss": 3.9373756870706536, + "tokens_seen": 507117568 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006838916750250753, + "loss": 1.6073, + "theoretical_loss": 3.937321413590913, + "tokens_seen": 507183104 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006838756268806419, + "loss": 1.7082, + "theoretical_loss": 3.9372671490870492, + "tokens_seen": 507248640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006838595787362087, + "loss": 1.4477, + "theoretical_loss": 3.937212893556418, + "tokens_seen": 507314176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006838435305917753, + "loss": 1.5368, + "theoretical_loss": 3.9371586469963775, + "tokens_seen": 507379712 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006838274824473421, + "loss": 1.581, + "theoretical_loss": 3.937104409404285, + "tokens_seen": 507445248 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006838114343029087, + "loss": 1.4914, + "theoretical_loss": 3.9370501807775007, + "tokens_seen": 507510784 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006837953861584755, + "loss": 1.5118, + "theoretical_loss": 3.936995961113385, + "tokens_seen": 507576320 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006837793380140421, + "loss": 1.549, + "theoretical_loss": 3.9369417504093, + "tokens_seen": 507641856 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006837632898696089, + "loss": 1.5614, + "theoretical_loss": 3.936887548662608, + "tokens_seen": 507707392 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006837472417251755, + "loss": 1.4779, + "theoretical_loss": 3.9368333558706734, + "tokens_seen": 507772928 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006837311935807423, + "loss": 1.6397, + "theoretical_loss": 3.936779172030861, + "tokens_seen": 507838464 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.9955716729164124, + "objective/train/avg_token_score": 0.004428323823958635, + "objective/train/docs_used": 191028, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4892574548721313, + "objective/train/original_loss": 1.496321678161621, + "objective/train/theoretical_loss": 3.936724997140538, + "objective/train/tokens_used": 528364000, + "objective/train/unlikelihood_loss": 0.009298749268054962, + "theoretical_loss": 3.936724997140538, + "tokens_seen": 507904000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006837151454363089, + "loss": 1.4797, + "theoretical_loss": 3.936724997140538, + "tokens_seen": 507904000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006836990972918757, + "loss": 1.5717, + "theoretical_loss": 3.9366708311970706, + "tokens_seen": 507969536 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006836830491474423, + "loss": 1.4437, + "theoretical_loss": 3.9366166741978272, + "tokens_seen": 508035072 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006836670010030091, + "loss": 1.5116, + "theoretical_loss": 3.936562526140178, + "tokens_seen": 508100608 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006836509528585757, + "loss": 1.4674, + "theoretical_loss": 3.936508387021494, + "tokens_seen": 508166144 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006836349047141425, + "loss": 1.5628, + "theoretical_loss": 3.9364542568391454, + "tokens_seen": 508231680 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006836188565697091, + "loss": 1.4964, + "theoretical_loss": 3.9364001355905067, + "tokens_seen": 508297216 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006836028084252759, + "loss": 1.5472, + "theoretical_loss": 3.936346023272952, + "tokens_seen": 508362752 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006835867602808425, + "loss": 1.5213, + "theoretical_loss": 3.936291919883855, + "tokens_seen": 508428288 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006835707121364093, + "loss": 1.5537, + "theoretical_loss": 3.9362378254205925, + "tokens_seen": 508493824 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006835546639919759, + "loss": 1.6761, + "theoretical_loss": 3.936183739880543, + "tokens_seen": 508559360 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006835386158475427, + "loss": 1.6426, + "theoretical_loss": 3.9361296632610827, + "tokens_seen": 508624896 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006835225677031093, + "loss": 1.5821, + "theoretical_loss": 3.9360755955595934, + "tokens_seen": 508690432 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006835065195586761, + "loss": 1.6955, + "theoretical_loss": 3.936021536773455, + "tokens_seen": 508755968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006834904714142427, + "loss": 1.5554, + "theoretical_loss": 3.9359674869000485, + "tokens_seen": 508821504 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006834744232698095, + "loss": 1.5594, + "theoretical_loss": 3.9359134459367575, + "tokens_seen": 508887040 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006834583751253763, + "loss": 1.5748, + "theoretical_loss": 3.9358594138809657, + "tokens_seen": 508952576 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006834423269809429, + "loss": 1.608, + "theoretical_loss": 3.9358053907300588, + "tokens_seen": 509018112 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006834262788365097, + "loss": 1.5509, + "theoretical_loss": 3.935751376481422, + "tokens_seen": 509083648 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006834102306920763, + "loss": 1.48, + "theoretical_loss": 3.9356973711324432, + "tokens_seen": 509149184 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006833941825476431, + "loss": 1.5793, + "theoretical_loss": 3.9356433746805113, + "tokens_seen": 509214720 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006833781344032097, + "loss": 1.6037, + "theoretical_loss": 3.9355893871230143, + "tokens_seen": 509280256 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006833620862587765, + "loss": 1.5692, + "theoretical_loss": 3.935535408457344, + "tokens_seen": 509345792 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006833460381143431, + "loss": 1.491, + "theoretical_loss": 3.9354814386808923, + "tokens_seen": 509411328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006833299899699097, + "loss": 1.5433, + "theoretical_loss": 3.935427477791051, + "tokens_seen": 509476864 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006833139418254765, + "loss": 1.5605, + "theoretical_loss": 3.9353735257852147, + "tokens_seen": 509542400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006832978936810431, + "loss": 1.507, + "theoretical_loss": 3.935319582660778, + "tokens_seen": 509607936 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006832818455366099, + "loss": 1.6062, + "theoretical_loss": 3.935265648415138, + "tokens_seen": 509673472 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006832657973921766, + "loss": 1.4144, + "theoretical_loss": 3.9352117230456902, + "tokens_seen": 509739008 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006832497492477433, + "loss": 1.587, + "theoretical_loss": 3.9351578065498343, + "tokens_seen": 509804544 + }, + { + "epoch": 0.15, + "learning_rate": 0.00068323370110331, + "loss": 1.6582, + "theoretical_loss": 3.935103898924969, + "tokens_seen": 509870080 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006832176529588766, + "loss": 1.6457, + "theoretical_loss": 3.935050000168495, + "tokens_seen": 509935616 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006832016048144434, + "loss": 1.4312, + "theoretical_loss": 3.9349961102778135, + "tokens_seen": 510001152 + }, + { + "epoch": 0.15, + "learning_rate": 0.00068318555667001, + "loss": 1.5957, + "theoretical_loss": 3.934942229250328, + "tokens_seen": 510066688 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006831695085255768, + "loss": 1.4977, + "theoretical_loss": 3.934888357083442, + "tokens_seen": 510132224 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006831534603811434, + "loss": 1.5042, + "theoretical_loss": 3.9348344937745594, + "tokens_seen": 510197760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006831374122367102, + "loss": 1.4611, + "theoretical_loss": 3.9347806393210876, + "tokens_seen": 510263296 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006831213640922768, + "loss": 1.6583, + "theoretical_loss": 3.934726793720433, + "tokens_seen": 510328832 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006831053159478436, + "loss": 1.4986, + "theoretical_loss": 3.9346729569700036, + "tokens_seen": 510394368 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006830892678034102, + "loss": 1.488, + "theoretical_loss": 3.934619129067209, + "tokens_seen": 510459904 + }, + { + "epoch": 0.15, + "learning_rate": 0.000683073219658977, + "loss": 1.5137, + "theoretical_loss": 3.934565310009459, + "tokens_seen": 510525440 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006830571715145436, + "loss": 1.6232, + "theoretical_loss": 3.9345114997941657, + "tokens_seen": 510590976 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006830411233701104, + "loss": 1.5604, + "theoretical_loss": 3.934457698418741, + "tokens_seen": 510656512 + }, + { + "epoch": 0.15, + "learning_rate": 0.000683025075225677, + "loss": 1.5533, + "theoretical_loss": 3.9344039058805986, + "tokens_seen": 510722048 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006830090270812438, + "loss": 1.5341, + "theoretical_loss": 3.934350122177154, + "tokens_seen": 510787584 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006829929789368104, + "loss": 1.448, + "theoretical_loss": 3.9342963473058212, + "tokens_seen": 510853120 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006829769307923772, + "loss": 1.5513, + "theoretical_loss": 3.9342425812640194, + "tokens_seen": 510918656 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006829608826479438, + "loss": 1.3911, + "theoretical_loss": 3.9341888240491647, + "tokens_seen": 510984192 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006829448345035106, + "loss": 1.6508, + "theoretical_loss": 3.9341350756586766, + "tokens_seen": 511049728 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006829287863590772, + "loss": 1.5567, + "theoretical_loss": 3.934081336089976, + "tokens_seen": 511115264 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.9954502582550049, + "objective/train/avg_token_score": 0.004549727775156498, + "objective/train/docs_used": 192198, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6080023050308228, + "objective/train/original_loss": 1.6155927181243896, + "objective/train/theoretical_loss": 3.934027605340483, + "objective/train/tokens_used": 531640800, + "objective/train/unlikelihood_loss": 0.009813708253204823, + "theoretical_loss": 3.934027605340483, + "tokens_seen": 511180800 + }, + { + "epoch": 0.15, + "learning_rate": 0.000682912738214644, + "loss": 1.641, + "theoretical_loss": 3.934027605340483, + "tokens_seen": 511180800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006828966900702106, + "loss": 1.4496, + "theoretical_loss": 3.93397388340762, + "tokens_seen": 511246336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006828806419257774, + "loss": 1.5632, + "theoretical_loss": 3.933920170288811, + "tokens_seen": 511311872 + }, + { + "epoch": 0.15, + "learning_rate": 0.000682864593781344, + "loss": 1.5085, + "theoretical_loss": 3.93386646598148, + "tokens_seen": 511377408 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006828485456369108, + "loss": 1.56, + "theoretical_loss": 3.9338127704830534, + "tokens_seen": 511442944 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006828324974924774, + "loss": 1.5507, + "theoretical_loss": 3.933759083790957, + "tokens_seen": 511508480 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006828164493480442, + "loss": 1.4919, + "theoretical_loss": 3.933705405902619, + "tokens_seen": 511574016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006828004012036108, + "loss": 1.5511, + "theoretical_loss": 3.9336517368154666, + "tokens_seen": 511639552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006827843530591776, + "loss": 1.6341, + "theoretical_loss": 3.9335980765269323, + "tokens_seen": 511705088 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006827683049147442, + "loss": 1.3591, + "theoretical_loss": 3.9335444250344445, + "tokens_seen": 511770624 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682752256770311, + "loss": 1.5131, + "theoretical_loss": 3.933490782335437, + "tokens_seen": 511836160 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006827362086258776, + "loss": 1.6209, + "theoretical_loss": 3.933437148427342, + "tokens_seen": 511901696 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006827201604814444, + "loss": 1.5443, + "theoretical_loss": 3.933383523307594, + "tokens_seen": 511967232 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682704112337011, + "loss": 1.6429, + "theoretical_loss": 3.9333299069736283, + "tokens_seen": 512032768 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006826880641925778, + "loss": 1.4156, + "theoretical_loss": 3.9332762994228814, + "tokens_seen": 512098304 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006826720160481444, + "loss": 1.5072, + "theoretical_loss": 3.93322270065279, + "tokens_seen": 512163840 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006826559679037112, + "loss": 1.4965, + "theoretical_loss": 3.933169110660793, + "tokens_seen": 512229376 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006826399197592778, + "loss": 1.541, + "theoretical_loss": 3.93311552944433, + "tokens_seen": 512294912 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006826238716148446, + "loss": 1.7229, + "theoretical_loss": 3.9330619570008416, + "tokens_seen": 512360448 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006826078234704112, + "loss": 1.467, + "theoretical_loss": 3.9330083933277695, + "tokens_seen": 512425984 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682591775325978, + "loss": 1.4996, + "theoretical_loss": 3.9329548384225568, + "tokens_seen": 512491520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006825757271815446, + "loss": 1.5309, + "theoretical_loss": 3.932901292282647, + "tokens_seen": 512557056 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006825596790371114, + "loss": 1.6469, + "theoretical_loss": 3.932847754905485, + "tokens_seen": 512622592 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682543630892678, + "loss": 1.6149, + "theoretical_loss": 3.9327942262885167, + "tokens_seen": 512688128 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006825275827482448, + "loss": 1.5995, + "theoretical_loss": 3.9327407064291897, + "tokens_seen": 512753664 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006825115346038114, + "loss": 1.4606, + "theoretical_loss": 3.9326871953249514, + "tokens_seen": 512819200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006824954864593782, + "loss": 1.4597, + "theoretical_loss": 3.9326336929732513, + "tokens_seen": 512884736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006824794383149448, + "loss": 1.5275, + "theoretical_loss": 3.9325801993715404, + "tokens_seen": 512950272 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006824633901705116, + "loss": 1.5365, + "theoretical_loss": 3.9325267145172687, + "tokens_seen": 513015808 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006824473420260782, + "loss": 1.622, + "theoretical_loss": 3.932473238407889, + "tokens_seen": 513081344 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682431293881645, + "loss": 1.6214, + "theoretical_loss": 3.932419771040856, + "tokens_seen": 513146880 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006824152457372116, + "loss": 1.4746, + "theoretical_loss": 3.932366312413623, + "tokens_seen": 513212416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006823991975927784, + "loss": 1.5725, + "theoretical_loss": 3.932312862523646, + "tokens_seen": 513277952 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682383149448345, + "loss": 1.5457, + "theoretical_loss": 3.9322594213683812, + "tokens_seen": 513343488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006823671013039118, + "loss": 1.449, + "theoretical_loss": 3.9322059889452867, + "tokens_seen": 513409024 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006823510531594784, + "loss": 1.5645, + "theoretical_loss": 3.9321525652518217, + "tokens_seen": 513474560 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006823350050150452, + "loss": 1.5723, + "theoretical_loss": 3.9320991502854454, + "tokens_seen": 513540096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006823189568706118, + "loss": 1.6288, + "theoretical_loss": 3.932045744043619, + "tokens_seen": 513605632 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006823029087261786, + "loss": 1.615, + "theoretical_loss": 3.9319923465238054, + "tokens_seen": 513671168 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006822868605817452, + "loss": 1.4536, + "theoretical_loss": 3.9319389577234665, + "tokens_seen": 513736704 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682270812437312, + "loss": 1.5986, + "theoretical_loss": 3.931885577640066, + "tokens_seen": 513802240 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006822547642928787, + "loss": 1.5179, + "theoretical_loss": 3.9318322062710704, + "tokens_seen": 513867776 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006822387161484454, + "loss": 1.4239, + "theoretical_loss": 3.931778843613945, + "tokens_seen": 513933312 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006822226680040121, + "loss": 1.5447, + "theoretical_loss": 3.9317254896661575, + "tokens_seen": 513998848 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006822066198595788, + "loss": 1.6182, + "theoretical_loss": 3.9316721444251765, + "tokens_seen": 514064384 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006821905717151455, + "loss": 1.5281, + "theoretical_loss": 3.9316188078884706, + "tokens_seen": 514129920 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006821745235707122, + "loss": 1.4729, + "theoretical_loss": 3.931565480053511, + "tokens_seen": 514195456 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682158475426279, + "loss": 1.5531, + "theoretical_loss": 3.9315121609177694, + "tokens_seen": 514260992 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006821424272818456, + "loss": 1.5333, + "theoretical_loss": 3.931458850478718, + "tokens_seen": 514326528 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006821263791374124, + "loss": 1.5704, + "theoretical_loss": 3.9314055487338297, + "tokens_seen": 514392064 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.9945394396781921, + "objective/train/avg_token_score": 0.0054605877958238125, + "objective/train/docs_used": 193343, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.473525047302246, + "objective/train/original_loss": 1.4812278747558594, + "objective/train/theoretical_loss": 3.93135225568058, + "objective/train/tokens_used": 534917600, + "objective/train/unlikelihood_loss": 0.017018338665366173, + "theoretical_loss": 3.93135225568058, + "tokens_seen": 514457600 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682110330992979, + "loss": 1.5684, + "theoretical_loss": 3.93135225568058, + "tokens_seen": 514457600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006820942828485458, + "loss": 1.5723, + "theoretical_loss": 3.9312989713164455, + "tokens_seen": 514523136 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006820782347041124, + "loss": 1.5398, + "theoretical_loss": 3.9312456956389017, + "tokens_seen": 514588672 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006820621865596792, + "loss": 1.6456, + "theoretical_loss": 3.931192428645427, + "tokens_seen": 514654208 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006820461384152458, + "loss": 1.4868, + "theoretical_loss": 3.9311391703334997, + "tokens_seen": 514719744 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006820300902708126, + "loss": 1.4854, + "theoretical_loss": 3.9310859207006006, + "tokens_seen": 514785280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006820140421263792, + "loss": 1.5952, + "theoretical_loss": 3.9310326797442103, + "tokens_seen": 514850816 + }, + { + "epoch": 0.16, + "learning_rate": 0.000681997993981946, + "loss": 1.5436, + "theoretical_loss": 3.930979447461811, + "tokens_seen": 514916352 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006819819458375126, + "loss": 1.5679, + "theoretical_loss": 3.9309262238508857, + "tokens_seen": 514981888 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006819658976930794, + "loss": 1.5657, + "theoretical_loss": 3.930873008908919, + "tokens_seen": 515047424 + }, + { + "epoch": 0.16, + "learning_rate": 0.000681949849548646, + "loss": 1.6424, + "theoretical_loss": 3.9308198026333954, + "tokens_seen": 515112960 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006819338014042126, + "loss": 1.5899, + "theoretical_loss": 3.9307666050218018, + "tokens_seen": 515178496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006819177532597794, + "loss": 1.5039, + "theoretical_loss": 3.930713416071625, + "tokens_seen": 515244032 + }, + { + "epoch": 0.16, + "learning_rate": 0.000681901705115346, + "loss": 1.5544, + "theoretical_loss": 3.930660235780354, + "tokens_seen": 515309568 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006818856569709128, + "loss": 1.6557, + "theoretical_loss": 3.9306070641454776, + "tokens_seen": 515375104 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006818696088264795, + "loss": 1.5329, + "theoretical_loss": 3.9305539011644863, + "tokens_seen": 515440640 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006818535606820462, + "loss": 1.5007, + "theoretical_loss": 3.9305007468348725, + "tokens_seen": 515506176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006818375125376129, + "loss": 1.6316, + "theoretical_loss": 3.9304476011541274, + "tokens_seen": 515571712 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006818214643931796, + "loss": 1.5393, + "theoretical_loss": 3.930394464119746, + "tokens_seen": 515637248 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006818054162487463, + "loss": 1.6247, + "theoretical_loss": 3.9303413357292216, + "tokens_seen": 515702784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006817893681043129, + "loss": 1.5349, + "theoretical_loss": 3.930288215980051, + "tokens_seen": 515768320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006817733199598797, + "loss": 1.5937, + "theoretical_loss": 3.9302351048697304, + "tokens_seen": 515833856 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006817572718154463, + "loss": 1.5521, + "theoretical_loss": 3.9301820023957577, + "tokens_seen": 515899392 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006817412236710131, + "loss": 1.5669, + "theoretical_loss": 3.930128908555632, + "tokens_seen": 515964928 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006817251755265797, + "loss": 1.6468, + "theoretical_loss": 3.9300758233468525, + "tokens_seen": 516030464 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006817091273821465, + "loss": 1.5385, + "theoretical_loss": 3.9300227467669204, + "tokens_seen": 516096000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006816930792377131, + "loss": 1.5736, + "theoretical_loss": 3.929969678813338, + "tokens_seen": 516161536 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006816770310932799, + "loss": 1.5211, + "theoretical_loss": 3.929916619483608, + "tokens_seen": 516227072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006816609829488465, + "loss": 1.4716, + "theoretical_loss": 3.9298635687752337, + "tokens_seen": 516292608 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006816449348044133, + "loss": 1.4646, + "theoretical_loss": 3.9298105266857215, + "tokens_seen": 516358144 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006816288866599799, + "loss": 1.541, + "theoretical_loss": 3.929757493212577, + "tokens_seen": 516423680 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006816128385155467, + "loss": 1.5545, + "theoretical_loss": 3.929704468353307, + "tokens_seen": 516489216 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006815967903711133, + "loss": 1.5313, + "theoretical_loss": 3.9296514521054196, + "tokens_seen": 516554752 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006815807422266801, + "loss": 1.5382, + "theoretical_loss": 3.929598444466425, + "tokens_seen": 516620288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006815646940822467, + "loss": 1.5828, + "theoretical_loss": 3.9295454454338326, + "tokens_seen": 516685824 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006815486459378135, + "loss": 1.5034, + "theoretical_loss": 3.9294924550051533, + "tokens_seen": 516751360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006815325977933801, + "loss": 1.4982, + "theoretical_loss": 3.9294394731779008, + "tokens_seen": 516816896 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006815165496489469, + "loss": 1.5805, + "theoretical_loss": 3.929386499949587, + "tokens_seen": 516882432 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006815005015045135, + "loss": 1.6534, + "theoretical_loss": 3.929333535317727, + "tokens_seen": 516947968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006814844533600803, + "loss": 1.4194, + "theoretical_loss": 3.929280579279837, + "tokens_seen": 517013504 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006814684052156469, + "loss": 1.418, + "theoretical_loss": 3.929227631833432, + "tokens_seen": 517079040 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006814523570712137, + "loss": 1.5326, + "theoretical_loss": 3.92917469297603, + "tokens_seen": 517144576 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006814363089267803, + "loss": 1.5733, + "theoretical_loss": 3.9291217627051496, + "tokens_seen": 517210112 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006814202607823471, + "loss": 1.4784, + "theoretical_loss": 3.9290688410183106, + "tokens_seen": 517275648 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006814042126379137, + "loss": 1.5844, + "theoretical_loss": 3.9290159279130332, + "tokens_seen": 517341184 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006813881644934805, + "loss": 1.5083, + "theoretical_loss": 3.928963023386839, + "tokens_seen": 517406720 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006813721163490471, + "loss": 1.5264, + "theoretical_loss": 3.9289101274372515, + "tokens_seen": 517472256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006813560682046139, + "loss": 1.4043, + "theoretical_loss": 3.9288572400617934, + "tokens_seen": 517537792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006813400200601805, + "loss": 1.504, + "theoretical_loss": 3.9288043612579893, + "tokens_seen": 517603328 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006813239719157473, + "loss": 1.5656, + "theoretical_loss": 3.928751491023366, + "tokens_seen": 517668864 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.9904393553733826, + "objective/train/avg_token_score": 0.009560601785779, + "objective/train/docs_used": 194581, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5729306936264038, + "objective/train/original_loss": 1.588305950164795, + "objective/train/theoretical_loss": 3.9286986293554493, + "objective/train/tokens_used": 538194400, + "objective/train/unlikelihood_loss": 0.02152993157505989, + "theoretical_loss": 3.9286986293554493, + "tokens_seen": 517734400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006813079237713139, + "loss": 1.5307, + "theoretical_loss": 3.9286986293554493, + "tokens_seen": 517734400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006812918756268807, + "loss": 1.5488, + "theoretical_loss": 3.9286457762517673, + "tokens_seen": 517799936 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006812758274824473, + "loss": 1.5465, + "theoretical_loss": 3.928592931709849, + "tokens_seen": 517865472 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006812597793380141, + "loss": 1.5909, + "theoretical_loss": 3.9285400957272234, + "tokens_seen": 517931008 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006812437311935807, + "loss": 1.4905, + "theoretical_loss": 3.9284872683014225, + "tokens_seen": 517996544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006812276830491475, + "loss": 1.4871, + "theoretical_loss": 3.9284344494299774, + "tokens_seen": 518062080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006812116349047141, + "loss": 1.5918, + "theoretical_loss": 3.9283816391104214, + "tokens_seen": 518127616 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006811955867602809, + "loss": 1.515, + "theoretical_loss": 3.9283288373402883, + "tokens_seen": 518193152 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006811795386158475, + "loss": 1.5547, + "theoretical_loss": 3.9282760441171134, + "tokens_seen": 518258688 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006811634904714143, + "loss": 1.4682, + "theoretical_loss": 3.9282232594384316, + "tokens_seen": 518324224 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006811474423269809, + "loss": 1.4201, + "theoretical_loss": 3.9281704833017814, + "tokens_seen": 518389760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006811313941825477, + "loss": 1.5204, + "theoretical_loss": 3.9281177157047003, + "tokens_seen": 518455296 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006811153460381143, + "loss": 1.4876, + "theoretical_loss": 3.928064956644726, + "tokens_seen": 518520832 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006810992978936811, + "loss": 1.5194, + "theoretical_loss": 3.9280122061194005, + "tokens_seen": 518586368 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006810832497492477, + "loss": 1.517, + "theoretical_loss": 3.927959464126264, + "tokens_seen": 518651904 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006810672016048145, + "loss": 1.5391, + "theoretical_loss": 3.9279067306628583, + "tokens_seen": 518717440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006810511534603812, + "loss": 1.4874, + "theoretical_loss": 3.9278540057267266, + "tokens_seen": 518782976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006810351053159479, + "loss": 1.6543, + "theoretical_loss": 3.927801289315414, + "tokens_seen": 518848512 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006810190571715146, + "loss": 1.4846, + "theoretical_loss": 3.9277485814264645, + "tokens_seen": 518914048 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006810030090270813, + "loss": 1.4744, + "theoretical_loss": 3.9276958820574253, + "tokens_seen": 518979584 + }, + { + "epoch": 0.16, + "learning_rate": 0.000680986960882648, + "loss": 1.5525, + "theoretical_loss": 3.9276431912058425, + "tokens_seen": 519045120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006809709127382147, + "loss": 1.5954, + "theoretical_loss": 3.9275905088692653, + "tokens_seen": 519110656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006809548645937814, + "loss": 1.4705, + "theoretical_loss": 3.927537835045242, + "tokens_seen": 519176192 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006809388164493481, + "loss": 1.5038, + "theoretical_loss": 3.927485169731323, + "tokens_seen": 519241728 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006809227683049148, + "loss": 1.5521, + "theoretical_loss": 3.9274325129250602, + "tokens_seen": 519307264 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006809067201604815, + "loss": 1.4617, + "theoretical_loss": 3.9273798646240055, + "tokens_seen": 519372800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006808906720160482, + "loss": 1.5391, + "theoretical_loss": 3.9273272248257123, + "tokens_seen": 519438336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006808746238716149, + "loss": 1.4935, + "theoretical_loss": 3.9272745935277347, + "tokens_seen": 519503872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006808585757271816, + "loss": 1.4353, + "theoretical_loss": 3.927221970727628, + "tokens_seen": 519569408 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006808425275827483, + "loss": 1.462, + "theoretical_loss": 3.927169356422948, + "tokens_seen": 519634944 + }, + { + "epoch": 0.16, + "learning_rate": 0.000680826479438315, + "loss": 1.4786, + "theoretical_loss": 3.9271167506112534, + "tokens_seen": 519700480 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006808104312938817, + "loss": 1.4594, + "theoretical_loss": 3.927064153290101, + "tokens_seen": 519766016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006807943831494484, + "loss": 1.5321, + "theoretical_loss": 3.9270115644570516, + "tokens_seen": 519831552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006807783350050151, + "loss": 1.5678, + "theoretical_loss": 3.9269589841096644, + "tokens_seen": 519897088 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006807622868605819, + "loss": 1.6111, + "theoretical_loss": 3.9269064122455006, + "tokens_seen": 519962624 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006807462387161485, + "loss": 1.4385, + "theoretical_loss": 3.926853848862124, + "tokens_seen": 520028160 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006807301905717153, + "loss": 1.4544, + "theoretical_loss": 3.9268012939570967, + "tokens_seen": 520093696 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006807141424272819, + "loss": 1.4588, + "theoretical_loss": 3.9267487475279834, + "tokens_seen": 520159232 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006806980942828487, + "loss": 1.6088, + "theoretical_loss": 3.9266962095723494, + "tokens_seen": 520224768 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006806820461384153, + "loss": 1.5216, + "theoretical_loss": 3.9266436800877615, + "tokens_seen": 520290304 + }, + { + "epoch": 0.16, + "learning_rate": 0.000680665997993982, + "loss": 1.5342, + "theoretical_loss": 3.926591159071787, + "tokens_seen": 520355840 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006806499498495487, + "loss": 1.4961, + "theoretical_loss": 3.9265386465219936, + "tokens_seen": 520421376 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006806339017051155, + "loss": 1.4807, + "theoretical_loss": 3.9264861424359516, + "tokens_seen": 520486912 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006806178535606821, + "loss": 1.573, + "theoretical_loss": 3.9264336468112315, + "tokens_seen": 520552448 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006806018054162489, + "loss": 1.4342, + "theoretical_loss": 3.9263811596454037, + "tokens_seen": 520617984 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006805857572718155, + "loss": 1.5373, + "theoretical_loss": 3.9263286809360407, + "tokens_seen": 520683520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006805697091273823, + "loss": 1.5055, + "theoretical_loss": 3.9262762106807174, + "tokens_seen": 520749056 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006805536609829489, + "loss": 1.4692, + "theoretical_loss": 3.926223748877007, + "tokens_seen": 520814592 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006805376128385156, + "loss": 1.45, + "theoretical_loss": 3.926171295522485, + "tokens_seen": 520880128 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006805215646940823, + "loss": 1.5401, + "theoretical_loss": 3.9261188506147278, + "tokens_seen": 520945664 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.9877818822860718, + "objective/train/avg_token_score": 0.012218118645250797, + "objective/train/docs_used": 195804, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6306140422821045, + "objective/train/original_loss": 1.6508474349975586, + "objective/train/theoretical_loss": 3.926066414151313, + "objective/train/tokens_used": 541471200, + "objective/train/unlikelihood_loss": 0.024284150451421738, + "theoretical_loss": 3.926066414151313, + "tokens_seen": 521011200 + }, + { + "epoch": 0.16, + "learning_rate": 0.000680505516549649, + "loss": 1.5824, + "theoretical_loss": 3.926066414151313, + "tokens_seen": 521011200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006804894684052157, + "loss": 1.5857, + "theoretical_loss": 3.926013986129819, + "tokens_seen": 521076736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006804734202607824, + "loss": 1.5613, + "theoretical_loss": 3.9259615665478256, + "tokens_seen": 521142272 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006804573721163491, + "loss": 1.4618, + "theoretical_loss": 3.9259091554029126, + "tokens_seen": 521207808 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006804413239719158, + "loss": 1.5686, + "theoretical_loss": 3.9258567526926615, + "tokens_seen": 521273344 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006804252758274825, + "loss": 1.4676, + "theoretical_loss": 3.9258043584146556, + "tokens_seen": 521338880 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006804092276830492, + "loss": 1.5374, + "theoretical_loss": 3.925751972566477, + "tokens_seen": 521404416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006803931795386158, + "loss": 1.653, + "theoretical_loss": 3.9256995951457103, + "tokens_seen": 521469952 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006803771313941826, + "loss": 1.5016, + "theoretical_loss": 3.925647226149942, + "tokens_seen": 521535488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006803610832497492, + "loss": 1.7053, + "theoretical_loss": 3.925594865576758, + "tokens_seen": 521601024 + }, + { + "epoch": 0.16, + "learning_rate": 0.000680345035105316, + "loss": 1.4565, + "theoretical_loss": 3.9255425134237445, + "tokens_seen": 521666560 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006803289869608826, + "loss": 1.627, + "theoretical_loss": 3.9254901696884916, + "tokens_seen": 521732096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006803129388164494, + "loss": 1.4554, + "theoretical_loss": 3.925437834368588, + "tokens_seen": 521797632 + }, + { + "epoch": 0.16, + "learning_rate": 0.000680296890672016, + "loss": 1.5259, + "theoretical_loss": 3.925385507461624, + "tokens_seen": 521863168 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006802808425275828, + "loss": 1.4624, + "theoretical_loss": 3.9253331889651912, + "tokens_seen": 521928704 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006802647943831494, + "loss": 1.464, + "theoretical_loss": 3.9252808788768814, + "tokens_seen": 521994240 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006802487462387162, + "loss": 1.4925, + "theoretical_loss": 3.9252285771942876, + "tokens_seen": 522059776 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006802326980942828, + "loss": 1.6095, + "theoretical_loss": 3.9251762839150057, + "tokens_seen": 522125312 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006802166499498496, + "loss": 1.6268, + "theoretical_loss": 3.9251239990366305, + "tokens_seen": 522190848 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006802006018054162, + "loss": 1.6015, + "theoretical_loss": 3.925071722556757, + "tokens_seen": 522256384 + }, + { + "epoch": 0.16, + "learning_rate": 0.000680184553660983, + "loss": 1.5093, + "theoretical_loss": 3.9250194544729844, + "tokens_seen": 522321920 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006801685055165496, + "loss": 1.6027, + "theoretical_loss": 3.9249671947829095, + "tokens_seen": 522387456 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006801524573721164, + "loss": 1.4919, + "theoretical_loss": 3.924914943484133, + "tokens_seen": 522452992 + }, + { + "epoch": 0.16, + "learning_rate": 0.000680136409227683, + "loss": 1.4107, + "theoretical_loss": 3.9248627005742533, + "tokens_seen": 522518528 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006801203610832498, + "loss": 1.4943, + "theoretical_loss": 3.9248104660508734, + "tokens_seen": 522584064 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006801043129388164, + "loss": 1.6131, + "theoretical_loss": 3.9247582399115943, + "tokens_seen": 522649600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006800882647943832, + "loss": 1.5676, + "theoretical_loss": 3.92470602215402, + "tokens_seen": 522715136 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006800722166499498, + "loss": 1.5873, + "theoretical_loss": 3.9246538127757544, + "tokens_seen": 522780672 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006800561685055166, + "loss": 1.6046, + "theoretical_loss": 3.9246016117744036, + "tokens_seen": 522846208 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006800401203610832, + "loss": 1.5064, + "theoretical_loss": 3.924549419147572, + "tokens_seen": 522911744 + }, + { + "epoch": 0.16, + "learning_rate": 0.00068002407221665, + "loss": 1.4072, + "theoretical_loss": 3.9244972348928684, + "tokens_seen": 522977280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006800080240722166, + "loss": 1.5959, + "theoretical_loss": 3.9244450590079003, + "tokens_seen": 523042816 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006799919759277834, + "loss": 1.5423, + "theoretical_loss": 3.9243928914902764, + "tokens_seen": 523108352 + }, + { + "epoch": 0.16, + "learning_rate": 0.00067997592778335, + "loss": 1.583, + "theoretical_loss": 3.924340732337608, + "tokens_seen": 523173888 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006799598796389168, + "loss": 1.5743, + "theoretical_loss": 3.924288581547505, + "tokens_seen": 523239424 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006799438314944834, + "loss": 1.4957, + "theoretical_loss": 3.9242364391175797, + "tokens_seen": 523304960 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006799277833500502, + "loss": 1.4978, + "theoretical_loss": 3.924184305045446, + "tokens_seen": 523370496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006799117352056168, + "loss": 1.4558, + "theoretical_loss": 3.9241321793287174, + "tokens_seen": 523436032 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006798956870611836, + "loss": 1.5677, + "theoretical_loss": 3.9240800619650082, + "tokens_seen": 523501568 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006798796389167503, + "loss": 1.5414, + "theoretical_loss": 3.9240279529519357, + "tokens_seen": 523567104 + }, + { + "epoch": 0.16, + "learning_rate": 0.000679863590772317, + "loss": 1.6672, + "theoretical_loss": 3.9239758522871164, + "tokens_seen": 523632640 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006798475426278837, + "loss": 1.473, + "theoretical_loss": 3.923923759968168, + "tokens_seen": 523698176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006798314944834504, + "loss": 1.4975, + "theoretical_loss": 3.9238716759927095, + "tokens_seen": 523763712 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006798154463390171, + "loss": 1.5757, + "theoretical_loss": 3.923819600358361, + "tokens_seen": 523829248 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006797993981945838, + "loss": 1.6787, + "theoretical_loss": 3.923767533062743, + "tokens_seen": 523894784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006797833500501505, + "loss": 1.4528, + "theoretical_loss": 3.9237154741034783, + "tokens_seen": 523960320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006797673019057172, + "loss": 1.5505, + "theoretical_loss": 3.923663423478189, + "tokens_seen": 524025856 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006797512537612839, + "loss": 1.5944, + "theoretical_loss": 3.9236113811844984, + "tokens_seen": 524091392 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006797352056168506, + "loss": 1.523, + "theoretical_loss": 3.9235593472200323, + "tokens_seen": 524156928 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006797191574724173, + "loss": 1.5185, + "theoretical_loss": 3.9235073215824166, + "tokens_seen": 524222464 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.9930202960968018, + "objective/train/avg_token_score": 0.006979662459343672, + "objective/train/docs_used": 196964, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4329326152801514, + "objective/train/original_loss": 1.441577672958374, + "objective/train/theoretical_loss": 3.9234553042692766, + "objective/train/tokens_used": 544748000, + "objective/train/unlikelihood_loss": 0.02277975156903267, + "theoretical_loss": 3.9234553042692766, + "tokens_seen": 524288000 + }, + { + "epoch": 0.16, + "learning_rate": 0.000679703109327984, + "loss": 1.4666, + "theoretical_loss": 3.9234553042692766, + "tokens_seen": 524288000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006796870611835507, + "loss": 1.5156, + "theoretical_loss": 3.9234032952782414, + "tokens_seen": 524353536 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006796710130391174, + "loss": 1.5986, + "theoretical_loss": 3.923351294606939, + "tokens_seen": 524419072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006796549648946841, + "loss": 1.5336, + "theoretical_loss": 3.923299302253, + "tokens_seen": 524484608 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006796389167502508, + "loss": 1.4997, + "theoretical_loss": 3.923247318214054, + "tokens_seen": 524550144 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006796228686058175, + "loss": 1.4351, + "theoretical_loss": 3.923195342487733, + "tokens_seen": 524615680 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006796068204613842, + "loss": 1.5432, + "theoretical_loss": 3.9231433750716693, + "tokens_seen": 524681216 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006795907723169509, + "loss": 1.5555, + "theoretical_loss": 3.9230914159634964, + "tokens_seen": 524746752 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006795747241725176, + "loss": 1.3913, + "theoretical_loss": 3.9230394651608496, + "tokens_seen": 524812288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006795586760280843, + "loss": 1.6815, + "theoretical_loss": 3.922987522661363, + "tokens_seen": 524877824 + }, + { + "epoch": 0.16, + "learning_rate": 0.000679542627883651, + "loss": 1.5386, + "theoretical_loss": 3.922935588462675, + "tokens_seen": 524943360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006795265797392177, + "loss": 1.3855, + "theoretical_loss": 3.9228836625624206, + "tokens_seen": 525008896 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006795105315947844, + "loss": 1.5284, + "theoretical_loss": 3.9228317449582404, + "tokens_seen": 525074432 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006794944834503511, + "loss": 1.4905, + "theoretical_loss": 3.9227798356477726, + "tokens_seen": 525139968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006794784353059178, + "loss": 1.5156, + "theoretical_loss": 3.922727934628658, + "tokens_seen": 525205504 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006794623871614845, + "loss": 1.5494, + "theoretical_loss": 3.922676041898537, + "tokens_seen": 525271040 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006794463390170512, + "loss": 1.5924, + "theoretical_loss": 3.9226241574550533, + "tokens_seen": 525336576 + }, + { + "epoch": 0.16, + "learning_rate": 0.000679430290872618, + "loss": 1.5692, + "theoretical_loss": 3.9225722812958486, + "tokens_seen": 525402112 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006794142427281846, + "loss": 1.6463, + "theoretical_loss": 3.922520413418568, + "tokens_seen": 525467648 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006793981945837513, + "loss": 1.5726, + "theoretical_loss": 3.922468553820856, + "tokens_seen": 525533184 + }, + { + "epoch": 0.16, + "learning_rate": 0.000679382146439318, + "loss": 1.5881, + "theoretical_loss": 3.9224167025003593, + "tokens_seen": 525598720 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006793660982948848, + "loss": 1.5717, + "theoretical_loss": 3.922364859454725, + "tokens_seen": 525664256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006793500501504514, + "loss": 1.4797, + "theoretical_loss": 3.9223130246816007, + "tokens_seen": 525729792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006793340020060182, + "loss": 1.4951, + "theoretical_loss": 3.9222611981786355, + "tokens_seen": 525795328 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006793179538615848, + "loss": 1.5442, + "theoretical_loss": 3.92220937994348, + "tokens_seen": 525860864 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006793019057171516, + "loss": 1.4764, + "theoretical_loss": 3.9221575699737836, + "tokens_seen": 525926400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006792858575727182, + "loss": 1.5438, + "theoretical_loss": 3.9221057682672, + "tokens_seen": 525991936 + }, + { + "epoch": 0.16, + "learning_rate": 0.000679269809428285, + "loss": 1.5672, + "theoretical_loss": 3.92205397482138, + "tokens_seen": 526057472 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006792537612838516, + "loss": 1.4317, + "theoretical_loss": 3.9220021896339796, + "tokens_seen": 526123008 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006792377131394184, + "loss": 1.6038, + "theoretical_loss": 3.921950412702652, + "tokens_seen": 526188544 + }, + { + "epoch": 0.16, + "learning_rate": 0.000679221664994985, + "loss": 1.5227, + "theoretical_loss": 3.9218986440250534, + "tokens_seen": 526254080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006792056168505518, + "loss": 1.5907, + "theoretical_loss": 3.9218468835988403, + "tokens_seen": 526319616 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006791895687061184, + "loss": 1.5745, + "theoretical_loss": 3.9217951314216704, + "tokens_seen": 526385152 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006791735205616852, + "loss": 1.5811, + "theoretical_loss": 3.9217433874912024, + "tokens_seen": 526450688 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006791574724172518, + "loss": 1.4677, + "theoretical_loss": 3.921691651805096, + "tokens_seen": 526516224 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006791414242728185, + "loss": 1.5547, + "theoretical_loss": 3.9216399243610116, + "tokens_seen": 526581760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006791253761283852, + "loss": 1.5296, + "theoretical_loss": 3.9215882051566098, + "tokens_seen": 526647296 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006791093279839519, + "loss": 1.5419, + "theoretical_loss": 3.921536494189554, + "tokens_seen": 526712832 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006790932798395186, + "loss": 1.6052, + "theoretical_loss": 3.921484791457507, + "tokens_seen": 526778368 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006790772316950853, + "loss": 1.4682, + "theoretical_loss": 3.921433096958134, + "tokens_seen": 526843904 + }, + { + "epoch": 0.16, + "learning_rate": 0.000679061183550652, + "loss": 1.4967, + "theoretical_loss": 3.921381410689099, + "tokens_seen": 526909440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006790451354062187, + "loss": 1.5451, + "theoretical_loss": 3.9213297326480694, + "tokens_seen": 526974976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006790290872617854, + "loss": 1.4597, + "theoretical_loss": 3.9212780628327115, + "tokens_seen": 527040512 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006790130391173521, + "loss": 1.5892, + "theoretical_loss": 3.921226401240693, + "tokens_seen": 527106048 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006789969909729187, + "loss": 1.4723, + "theoretical_loss": 3.9211747478696846, + "tokens_seen": 527171584 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006789809428284855, + "loss": 1.4891, + "theoretical_loss": 3.9211231027173548, + "tokens_seen": 527237120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006789648946840521, + "loss": 1.5718, + "theoretical_loss": 3.921071465781375, + "tokens_seen": 527302656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006789488465396189, + "loss": 1.5704, + "theoretical_loss": 3.9210198370594176, + "tokens_seen": 527368192 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006789327983951855, + "loss": 1.5069, + "theoretical_loss": 3.920968216549155, + "tokens_seen": 527433728 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006789167502507523, + "loss": 1.5436, + "theoretical_loss": 3.9209166042482613, + "tokens_seen": 527499264 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.9926773309707642, + "objective/train/avg_token_score": 0.00732271047309041, + "objective/train/docs_used": 198271, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.394217848777771, + "objective/train/original_loss": 1.4063340425491333, + "objective/train/theoretical_loss": 3.9208650001544108, + "objective/train/tokens_used": 548024800, + "objective/train/unlikelihood_loss": 0.016799025237560272, + "theoretical_loss": 3.9208650001544108, + "tokens_seen": 527564800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006789007021063189, + "loss": 1.4683, + "theoretical_loss": 3.9208650001544108, + "tokens_seen": 527564800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006788846539618857, + "loss": 1.5597, + "theoretical_loss": 3.920813404265279, + "tokens_seen": 527630336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006788686058174523, + "loss": 1.5209, + "theoretical_loss": 3.920761816578543, + "tokens_seen": 527695872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006788525576730191, + "loss": 1.5761, + "theoretical_loss": 3.920710237091881, + "tokens_seen": 527761408 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006788365095285857, + "loss": 1.4631, + "theoretical_loss": 3.9206586658029714, + "tokens_seen": 527826944 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006788204613841525, + "loss": 1.6268, + "theoretical_loss": 3.920607102709493, + "tokens_seen": 527892480 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006788044132397191, + "loss": 1.5279, + "theoretical_loss": 3.9205555478091254, + "tokens_seen": 527958016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006787883650952859, + "loss": 1.5898, + "theoretical_loss": 3.920504001099552, + "tokens_seen": 528023552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006787723169508525, + "loss": 1.522, + "theoretical_loss": 3.920452462578454, + "tokens_seen": 528089088 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006787562688064193, + "loss": 1.5673, + "theoretical_loss": 3.920400932243515, + "tokens_seen": 528154624 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006787402206619859, + "loss": 1.452, + "theoretical_loss": 3.920349410092419, + "tokens_seen": 528220160 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006787241725175527, + "loss": 1.5464, + "theoretical_loss": 3.920297896122851, + "tokens_seen": 528285696 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006787081243731193, + "loss": 1.5646, + "theoretical_loss": 3.9202463903324976, + "tokens_seen": 528351232 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006786920762286861, + "loss": 1.5287, + "theoretical_loss": 3.920194892719045, + "tokens_seen": 528416768 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006786760280842528, + "loss": 1.5273, + "theoretical_loss": 3.920143403280182, + "tokens_seen": 528482304 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006786599799398195, + "loss": 1.473, + "theoretical_loss": 3.9200919220135972, + "tokens_seen": 528547840 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006786439317953862, + "loss": 1.5717, + "theoretical_loss": 3.920040448916981, + "tokens_seen": 528613376 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006786278836509529, + "loss": 1.5507, + "theoretical_loss": 3.919988983988023, + "tokens_seen": 528678912 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006786118355065196, + "loss": 1.4356, + "theoretical_loss": 3.919937527224416, + "tokens_seen": 528744448 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006785957873620863, + "loss": 1.5093, + "theoretical_loss": 3.9198860786238523, + "tokens_seen": 528809984 + }, + { + "epoch": 0.16, + "learning_rate": 0.000678579739217653, + "loss": 1.6489, + "theoretical_loss": 3.919834638184025, + "tokens_seen": 528875520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006785636910732197, + "loss": 1.557, + "theoretical_loss": 3.91978320590263, + "tokens_seen": 528941056 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006785476429287864, + "loss": 1.569, + "theoretical_loss": 3.919731781777361, + "tokens_seen": 529006592 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006785315947843531, + "loss": 1.552, + "theoretical_loss": 3.919680365805916, + "tokens_seen": 529072128 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006785155466399198, + "loss": 1.4769, + "theoretical_loss": 3.9196289579859913, + "tokens_seen": 529137664 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006784994984954865, + "loss": 1.5035, + "theoretical_loss": 3.919577558315286, + "tokens_seen": 529203200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006784834503510532, + "loss": 1.5665, + "theoretical_loss": 3.9195261667914982, + "tokens_seen": 529268736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006784674022066199, + "loss": 1.4731, + "theoretical_loss": 3.91947478341233, + "tokens_seen": 529334272 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006784513540621866, + "loss": 1.5231, + "theoretical_loss": 3.9194234081754797, + "tokens_seen": 529399808 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006784353059177533, + "loss": 1.5011, + "theoretical_loss": 3.9193720410786517, + "tokens_seen": 529465344 + }, + { + "epoch": 0.16, + "learning_rate": 0.00067841925777332, + "loss": 1.6981, + "theoretical_loss": 3.9193206821195483, + "tokens_seen": 529530880 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006784032096288867, + "loss": 1.4491, + "theoretical_loss": 3.9192693312958733, + "tokens_seen": 529596416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006783871614844534, + "loss": 1.5467, + "theoretical_loss": 3.919217988605331, + "tokens_seen": 529661952 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006783711133400201, + "loss": 1.5573, + "theoretical_loss": 3.919166654045628, + "tokens_seen": 529727488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006783550651955868, + "loss": 1.589, + "theoretical_loss": 3.919115327614471, + "tokens_seen": 529793024 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006783390170511535, + "loss": 1.5927, + "theoretical_loss": 3.9190640093095666, + "tokens_seen": 529858560 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006783229689067202, + "loss": 1.6124, + "theoretical_loss": 3.9190126991286243, + "tokens_seen": 529924096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006783069207622869, + "loss": 1.5274, + "theoretical_loss": 3.918961397069353, + "tokens_seen": 529989632 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006782908726178536, + "loss": 1.6377, + "theoretical_loss": 3.9189101031294635, + "tokens_seen": 530055168 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006782748244734203, + "loss": 1.4066, + "theoretical_loss": 3.918858817306667, + "tokens_seen": 530120704 + }, + { + "epoch": 0.16, + "learning_rate": 0.000678258776328987, + "loss": 1.4025, + "theoretical_loss": 3.918807539598676, + "tokens_seen": 530186240 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006782427281845537, + "loss": 1.4937, + "theoretical_loss": 3.918756270003203, + "tokens_seen": 530251776 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006782266800401204, + "loss": 1.54, + "theoretical_loss": 3.9187050085179633, + "tokens_seen": 530317312 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006782106318956871, + "loss": 1.5107, + "theoretical_loss": 3.918653755140671, + "tokens_seen": 530382848 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006781945837512538, + "loss": 1.4575, + "theoretical_loss": 3.918602509869042, + "tokens_seen": 530448384 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006781785356068205, + "loss": 1.5517, + "theoretical_loss": 3.9185512727007934, + "tokens_seen": 530513920 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006781624874623872, + "loss": 1.5218, + "theoretical_loss": 3.9185000436336432, + "tokens_seen": 530579456 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006781464393179539, + "loss": 1.562, + "theoretical_loss": 3.9184488226653102, + "tokens_seen": 530644992 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006781303911735206, + "loss": 1.5196, + "theoretical_loss": 3.918397609793514, + "tokens_seen": 530710528 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006781143430290873, + "loss": 1.5177, + "theoretical_loss": 3.9183464050159746, + "tokens_seen": 530776064 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.9919033646583557, + "objective/train/avg_token_score": 0.008096570149064064, + "objective/train/docs_used": 199409, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5678794384002686, + "objective/train/original_loss": 1.5803380012512207, + "objective/train/theoretical_loss": 3.918295208330415, + "objective/train/tokens_used": 551301600, + "objective/train/unlikelihood_loss": 0.02289610356092453, + "theoretical_loss": 3.918295208330415, + "tokens_seen": 530841600 + }, + { + "epoch": 0.16, + "learning_rate": 0.000678098294884654, + "loss": 1.5062, + "theoretical_loss": 3.918295208330415, + "tokens_seen": 530841600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006780822467402207, + "loss": 1.5401, + "theoretical_loss": 3.918244019734556, + "tokens_seen": 530907136 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006780661985957874, + "loss": 1.5156, + "theoretical_loss": 3.9181928392261214, + "tokens_seen": 530972672 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006780501504513541, + "loss": 1.4202, + "theoretical_loss": 3.9181416668028355, + "tokens_seen": 531038208 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006780341023069208, + "loss": 1.4515, + "theoretical_loss": 3.918090502462425, + "tokens_seen": 531103744 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006780180541624875, + "loss": 1.6298, + "theoretical_loss": 3.9180393462026135, + "tokens_seen": 531169280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006780020060180543, + "loss": 1.5941, + "theoretical_loss": 3.9179881980211295, + "tokens_seen": 531234816 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006779859578736209, + "loss": 1.58, + "theoretical_loss": 3.917937057915701, + "tokens_seen": 531300352 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006779699097291877, + "loss": 1.4885, + "theoretical_loss": 3.9178859258840566, + "tokens_seen": 531365888 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006779538615847543, + "loss": 1.5035, + "theoretical_loss": 3.9178348019239264, + "tokens_seen": 531431424 + }, + { + "epoch": 0.16, + "learning_rate": 0.000677937813440321, + "loss": 1.5714, + "theoretical_loss": 3.91778368603304, + "tokens_seen": 531496960 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006779217652958877, + "loss": 1.6706, + "theoretical_loss": 3.9177325782091303, + "tokens_seen": 531562496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006779057171514545, + "loss": 1.5649, + "theoretical_loss": 3.9176814784499303, + "tokens_seen": 531628032 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006778896690070211, + "loss": 1.5415, + "theoretical_loss": 3.9176303867531717, + "tokens_seen": 531693568 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006778736208625879, + "loss": 1.6175, + "theoretical_loss": 3.9175793031165895, + "tokens_seen": 531759104 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006778575727181545, + "loss": 1.6213, + "theoretical_loss": 3.91752822753792, + "tokens_seen": 531824640 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006778415245737213, + "loss": 1.442, + "theoretical_loss": 3.917477160014898, + "tokens_seen": 531890176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006778254764292879, + "loss": 1.4851, + "theoretical_loss": 3.9174261005452617, + "tokens_seen": 531955712 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006778094282848547, + "loss": 1.5491, + "theoretical_loss": 3.917375049126749, + "tokens_seen": 532021248 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006777933801404213, + "loss": 1.449, + "theoretical_loss": 3.9173240057570986, + "tokens_seen": 532086784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006777773319959881, + "loss": 1.4732, + "theoretical_loss": 3.91727297043405, + "tokens_seen": 532152320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006777612838515547, + "loss": 1.4136, + "theoretical_loss": 3.917221943155345, + "tokens_seen": 532217856 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006777452357071214, + "loss": 1.4739, + "theoretical_loss": 3.9171709239187247, + "tokens_seen": 532283392 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006777291875626881, + "loss": 1.7262, + "theoretical_loss": 3.9171199127219314, + "tokens_seen": 532348928 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006777131394182548, + "loss": 1.505, + "theoretical_loss": 3.917068909562709, + "tokens_seen": 532414464 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006776970912738215, + "loss": 1.5971, + "theoretical_loss": 3.917017914438802, + "tokens_seen": 532480000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006776810431293882, + "loss": 1.6364, + "theoretical_loss": 3.9169669273479553, + "tokens_seen": 532545536 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006776649949849549, + "loss": 1.578, + "theoretical_loss": 3.916915948287916, + "tokens_seen": 532611072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006776489468405216, + "loss": 1.5222, + "theoretical_loss": 3.91686497725643, + "tokens_seen": 532676608 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006776328986960883, + "loss": 1.4429, + "theoretical_loss": 3.916814014251247, + "tokens_seen": 532742144 + }, + { + "epoch": 0.16, + "learning_rate": 0.000677616850551655, + "loss": 1.5493, + "theoretical_loss": 3.9167630592701146, + "tokens_seen": 532807680 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006776008024072216, + "loss": 1.5755, + "theoretical_loss": 3.9167121123107833, + "tokens_seen": 532873216 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006775847542627884, + "loss": 1.5259, + "theoretical_loss": 3.9166611733710037, + "tokens_seen": 532938752 + }, + { + "epoch": 0.16, + "learning_rate": 0.000677568706118355, + "loss": 1.5707, + "theoretical_loss": 3.9166102424485274, + "tokens_seen": 533004288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006775526579739218, + "loss": 1.5247, + "theoretical_loss": 3.9165593195411077, + "tokens_seen": 533069824 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006775366098294884, + "loss": 1.5659, + "theoretical_loss": 3.916508404646497, + "tokens_seen": 533135360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006775205616850552, + "loss": 1.5078, + "theoretical_loss": 3.9164574977624507, + "tokens_seen": 533200896 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006775045135406218, + "loss": 1.5493, + "theoretical_loss": 3.916406598886724, + "tokens_seen": 533266432 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006774884653961886, + "loss": 1.5106, + "theoretical_loss": 3.916355708017073, + "tokens_seen": 533331968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006774724172517553, + "loss": 1.5334, + "theoretical_loss": 3.916304825151254, + "tokens_seen": 533397504 + }, + { + "epoch": 0.16, + "learning_rate": 0.000677456369107322, + "loss": 1.5337, + "theoretical_loss": 3.916253950287026, + "tokens_seen": 533463040 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006774403209628887, + "loss": 1.5196, + "theoretical_loss": 3.916203083422148, + "tokens_seen": 533528576 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006774242728184554, + "loss": 1.4959, + "theoretical_loss": 3.91615222455438, + "tokens_seen": 533594112 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006774082246740221, + "loss": 1.5808, + "theoretical_loss": 3.9161013736814816, + "tokens_seen": 533659648 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006773921765295888, + "loss": 1.5187, + "theoretical_loss": 3.916050530801215, + "tokens_seen": 533725184 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006773761283851555, + "loss": 1.5335, + "theoretical_loss": 3.915999695911343, + "tokens_seen": 533790720 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006773600802407222, + "loss": 1.4735, + "theoretical_loss": 3.91594886900963, + "tokens_seen": 533856256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006773440320962889, + "loss": 1.6058, + "theoretical_loss": 3.9158980500938383, + "tokens_seen": 533921792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006773279839518556, + "loss": 1.5409, + "theoretical_loss": 3.9158472391617343, + "tokens_seen": 533987328 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006773119358074223, + "loss": 1.5697, + "theoretical_loss": 3.9157964362110844, + "tokens_seen": 534052864 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.9939670562744141, + "objective/train/avg_token_score": 0.006032905541360378, + "objective/train/docs_used": 200516, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5747506618499756, + "objective/train/original_loss": 1.5854096412658691, + "objective/train/theoretical_loss": 3.9157456412396554, + "objective/train/tokens_used": 554578400, + "objective/train/unlikelihood_loss": 0.013583440333604813, + "theoretical_loss": 3.9157456412396554, + "tokens_seen": 534118400 + }, + { + "epoch": 0.16, + "learning_rate": 0.000677295887662989, + "loss": 1.5098, + "theoretical_loss": 3.9157456412396554, + "tokens_seen": 534118400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006772798395185557, + "loss": 1.635, + "theoretical_loss": 3.9156948542452144, + "tokens_seen": 534183936 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006772637913741224, + "loss": 1.6146, + "theoretical_loss": 3.915644075225532, + "tokens_seen": 534249472 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006772477432296891, + "loss": 1.5488, + "theoretical_loss": 3.915593304178376, + "tokens_seen": 534315008 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006772316950852558, + "loss": 1.5556, + "theoretical_loss": 3.9155425411015186, + "tokens_seen": 534380544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006772156469408225, + "loss": 1.5789, + "theoretical_loss": 3.9154917859927307, + "tokens_seen": 534446080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006771995987963892, + "loss": 1.5621, + "theoretical_loss": 3.9154410388497842, + "tokens_seen": 534511616 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006771835506519559, + "loss": 1.5395, + "theoretical_loss": 3.9153902996704533, + "tokens_seen": 534577152 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006771675025075226, + "loss": 1.4714, + "theoretical_loss": 3.915339568452512, + "tokens_seen": 534642688 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006771514543630893, + "loss": 1.5256, + "theoretical_loss": 3.9152888451937358, + "tokens_seen": 534708224 + }, + { + "epoch": 0.16, + "learning_rate": 0.000677135406218656, + "loss": 1.5201, + "theoretical_loss": 3.9152381298919, + "tokens_seen": 534773760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006771193580742227, + "loss": 1.6521, + "theoretical_loss": 3.915187422544782, + "tokens_seen": 534839296 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006771033099297894, + "loss": 1.4522, + "theoretical_loss": 3.915136723150159, + "tokens_seen": 534904832 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006770872617853561, + "loss": 1.5389, + "theoretical_loss": 3.91508603170581, + "tokens_seen": 534970368 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006770712136409228, + "loss": 1.5085, + "theoretical_loss": 3.915035348209514, + "tokens_seen": 535035904 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006770551654964895, + "loss": 1.5008, + "theoretical_loss": 3.9149846726590534, + "tokens_seen": 535101440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006770391173520562, + "loss": 1.5617, + "theoretical_loss": 3.914934005052208, + "tokens_seen": 535166976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006770230692076229, + "loss": 1.5975, + "theoretical_loss": 3.91488334538676, + "tokens_seen": 535232512 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006770070210631896, + "loss": 1.4839, + "theoretical_loss": 3.914832693660493, + "tokens_seen": 535298048 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006769909729187563, + "loss": 1.4577, + "theoretical_loss": 3.9147820498711914, + "tokens_seen": 535363584 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676974924774323, + "loss": 1.4591, + "theoretical_loss": 3.91473141401664, + "tokens_seen": 535429120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006769588766298897, + "loss": 1.5264, + "theoretical_loss": 3.9146807860946238, + "tokens_seen": 535494656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006769428284854564, + "loss": 1.4996, + "theoretical_loss": 3.91463016610293, + "tokens_seen": 535560192 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006769267803410231, + "loss": 1.5619, + "theoretical_loss": 3.9145795540393467, + "tokens_seen": 535625728 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006769107321965898, + "loss": 1.5322, + "theoretical_loss": 3.9145289499016616, + "tokens_seen": 535691264 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006768946840521565, + "loss": 1.5471, + "theoretical_loss": 3.9144783536876644, + "tokens_seen": 535756800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006768786359077232, + "loss": 1.4634, + "theoretical_loss": 3.9144277653951454, + "tokens_seen": 535822336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006768625877632899, + "loss": 1.6186, + "theoretical_loss": 3.914377185021896, + "tokens_seen": 535887872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006768465396188566, + "loss": 1.5894, + "theoretical_loss": 3.914326612565708, + "tokens_seen": 535953408 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006768304914744233, + "loss": 1.5661, + "theoretical_loss": 3.914276048024374, + "tokens_seen": 536018944 + }, + { + "epoch": 0.16, + "learning_rate": 0.00067681444332999, + "loss": 1.517, + "theoretical_loss": 3.9142254913956886, + "tokens_seen": 536084480 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006767983951855567, + "loss": 1.5118, + "theoretical_loss": 3.914174942677446, + "tokens_seen": 536150016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006767823470411234, + "loss": 1.4837, + "theoretical_loss": 3.9141244018674413, + "tokens_seen": 536215552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006767662988966901, + "loss": 1.573, + "theoretical_loss": 3.9140738689634715, + "tokens_seen": 536281088 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006767502507522568, + "loss": 1.5116, + "theoretical_loss": 3.914023343963334, + "tokens_seen": 536346624 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006767342026078235, + "loss": 1.6109, + "theoretical_loss": 3.9139728268648266, + "tokens_seen": 536412160 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006767181544633902, + "loss": 1.5085, + "theoretical_loss": 3.913922317665749, + "tokens_seen": 536477696 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676702106318957, + "loss": 1.4495, + "theoretical_loss": 3.9138718163639012, + "tokens_seen": 536543232 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006766860581745236, + "loss": 1.4396, + "theoretical_loss": 3.9138213229570837, + "tokens_seen": 536608768 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006766700100300903, + "loss": 1.5308, + "theoretical_loss": 3.9137708374430984, + "tokens_seen": 536674304 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676653961885657, + "loss": 1.6088, + "theoretical_loss": 3.913720359819748, + "tokens_seen": 536739840 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006766379137412238, + "loss": 1.5019, + "theoretical_loss": 3.913669890084835, + "tokens_seen": 536805376 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006766218655967904, + "loss": 1.5423, + "theoretical_loss": 3.9136194282361654, + "tokens_seen": 536870912 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006766058174523572, + "loss": 1.4445, + "theoretical_loss": 3.913568974271543, + "tokens_seen": 536936448 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006765897693079238, + "loss": 1.502, + "theoretical_loss": 3.913518528188775, + "tokens_seen": 537001984 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006765737211634906, + "loss": 1.4768, + "theoretical_loss": 3.9134680899856686, + "tokens_seen": 537067520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006765576730190572, + "loss": 1.501, + "theoretical_loss": 3.913417659660031, + "tokens_seen": 537133056 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676541624874624, + "loss": 1.4784, + "theoretical_loss": 3.913367237209671, + "tokens_seen": 537198592 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006765255767301906, + "loss": 1.5501, + "theoretical_loss": 3.9133168226323987, + "tokens_seen": 537264128 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006765095285857574, + "loss": 1.5368, + "theoretical_loss": 3.913266415926024, + "tokens_seen": 537329664 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.9932007193565369, + "objective/train/avg_token_score": 0.0067993332631886005, + "objective/train/docs_used": 201641, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.437907338142395, + "objective/train/original_loss": 1.4490562677383423, + "objective/train/theoretical_loss": 3.9132160170883585, + "objective/train/tokens_used": 557855200, + "objective/train/unlikelihood_loss": 0.016128122806549072, + "theoretical_loss": 3.9132160170883585, + "tokens_seen": 537395200 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676493480441324, + "loss": 1.4362, + "theoretical_loss": 3.9132160170883585, + "tokens_seen": 537395200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006764774322968908, + "loss": 1.5591, + "theoretical_loss": 3.913165626117215, + "tokens_seen": 537460736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006764613841524574, + "loss": 1.5817, + "theoretical_loss": 3.913115243010407, + "tokens_seen": 537526272 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006764453360080242, + "loss": 1.4458, + "theoretical_loss": 3.9130648677657476, + "tokens_seen": 537591808 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006764292878635908, + "loss": 1.4854, + "theoretical_loss": 3.9130145003810517, + "tokens_seen": 537657344 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006764132397191576, + "loss": 1.578, + "theoretical_loss": 3.912964140854135, + "tokens_seen": 537722880 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006763971915747242, + "loss": 1.5742, + "theoretical_loss": 3.9129137891828147, + "tokens_seen": 537788416 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676381143430291, + "loss": 1.5399, + "theoretical_loss": 3.9128634453649083, + "tokens_seen": 537853952 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006763650952858576, + "loss": 1.5722, + "theoretical_loss": 3.9128131093982343, + "tokens_seen": 537919488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006763490471414244, + "loss": 1.6003, + "theoretical_loss": 3.9127627812806116, + "tokens_seen": 537985024 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676332998996991, + "loss": 1.544, + "theoretical_loss": 3.91271246100986, + "tokens_seen": 538050560 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006763169508525577, + "loss": 1.5618, + "theoretical_loss": 3.9126621485838013, + "tokens_seen": 538116096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006763009027081244, + "loss": 1.4138, + "theoretical_loss": 3.9126118440002573, + "tokens_seen": 538181632 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006762848545636911, + "loss": 1.5417, + "theoretical_loss": 3.9125615472570505, + "tokens_seen": 538247168 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006762688064192578, + "loss": 1.6367, + "theoretical_loss": 3.912511258352004, + "tokens_seen": 538312704 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006762527582748245, + "loss": 1.6241, + "theoretical_loss": 3.9124609772829437, + "tokens_seen": 538378240 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006762367101303912, + "loss": 1.4628, + "theoretical_loss": 3.9124107040476934, + "tokens_seen": 538443776 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006762206619859579, + "loss": 1.497, + "theoretical_loss": 3.91236043864408, + "tokens_seen": 538509312 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006762046138415246, + "loss": 1.5126, + "theoretical_loss": 3.912310181069931, + "tokens_seen": 538574848 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006761885656970913, + "loss": 1.6726, + "theoretical_loss": 3.9122599313230735, + "tokens_seen": 538640384 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676172517552658, + "loss": 1.4931, + "theoretical_loss": 3.9122096894013376, + "tokens_seen": 538705920 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006761564694082247, + "loss": 1.5969, + "theoretical_loss": 3.9121594553025516, + "tokens_seen": 538771456 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006761404212637914, + "loss": 1.5076, + "theoretical_loss": 3.912109229024547, + "tokens_seen": 538836992 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006761243731193581, + "loss": 1.507, + "theoretical_loss": 3.9120590105651547, + "tokens_seen": 538902528 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006761083249749248, + "loss": 1.5704, + "theoretical_loss": 3.9120087999222073, + "tokens_seen": 538968064 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006760922768304915, + "loss": 1.4131, + "theoretical_loss": 3.9119585970935375, + "tokens_seen": 539033600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006760762286860582, + "loss": 1.4475, + "theoretical_loss": 3.91190840207698, + "tokens_seen": 539099136 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006760601805416249, + "loss": 1.4955, + "theoretical_loss": 3.9118582148703696, + "tokens_seen": 539164672 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006760441323971916, + "loss": 1.5815, + "theoretical_loss": 3.9118080354715414, + "tokens_seen": 539230208 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006760280842527583, + "loss": 1.6076, + "theoretical_loss": 3.911757863878333, + "tokens_seen": 539295744 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676012036108325, + "loss": 1.6641, + "theoretical_loss": 3.9117077000885807, + "tokens_seen": 539361280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006759959879638917, + "loss": 1.5275, + "theoretical_loss": 3.911657544100124, + "tokens_seen": 539426816 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006759799398194584, + "loss": 1.4477, + "theoretical_loss": 3.911607395910801, + "tokens_seen": 539492352 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006759638916750251, + "loss": 1.5347, + "theoretical_loss": 3.911557255518453, + "tokens_seen": 539557888 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006759478435305918, + "loss": 1.5669, + "theoretical_loss": 3.9115071229209195, + "tokens_seen": 539623424 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006759317953861585, + "loss": 1.5545, + "theoretical_loss": 3.9114569981160434, + "tokens_seen": 539688960 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006759157472417252, + "loss": 1.5159, + "theoretical_loss": 3.9114068811016662, + "tokens_seen": 539754496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006758996990972919, + "loss": 1.6027, + "theoretical_loss": 3.9113567718756332, + "tokens_seen": 539820032 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006758836509528586, + "loss": 1.5372, + "theoretical_loss": 3.9113066704357866, + "tokens_seen": 539885568 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006758676028084253, + "loss": 1.612, + "theoretical_loss": 3.9112565767799734, + "tokens_seen": 539951104 + }, + { + "epoch": 0.16, + "learning_rate": 0.000675851554663992, + "loss": 1.596, + "theoretical_loss": 3.9112064909060393, + "tokens_seen": 540016640 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006758355065195587, + "loss": 1.5475, + "theoretical_loss": 3.91115641281183, + "tokens_seen": 540082176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006758194583751254, + "loss": 1.4662, + "theoretical_loss": 3.911106342495195, + "tokens_seen": 540147712 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006758034102306921, + "loss": 1.7235, + "theoretical_loss": 3.911056279953981, + "tokens_seen": 540213248 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006757873620862588, + "loss": 1.4688, + "theoretical_loss": 3.91100622518604, + "tokens_seen": 540278784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006757713139418255, + "loss": 1.643, + "theoretical_loss": 3.9109561781892204, + "tokens_seen": 540344320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006757552657973922, + "loss": 1.5624, + "theoretical_loss": 3.9109061389613737, + "tokens_seen": 540409856 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006757392176529589, + "loss": 1.4811, + "theoretical_loss": 3.9108561075003525, + "tokens_seen": 540475392 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006757231695085256, + "loss": 1.6177, + "theoretical_loss": 3.9108060838040095, + "tokens_seen": 540540928 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006757071213640923, + "loss": 1.4686, + "theoretical_loss": 3.9107560678701985, + "tokens_seen": 540606464 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.9921212792396545, + "objective/train/avg_token_score": 0.007878746837377548, + "objective/train/docs_used": 202818, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4836454391479492, + "objective/train/original_loss": 1.496475100517273, + "objective/train/theoretical_loss": 3.910706059696774, + "objective/train/tokens_used": 561132000, + "objective/train/unlikelihood_loss": 0.018754584714770317, + "theoretical_loss": 3.910706059696774, + "tokens_seen": 540672000 + }, + { + "epoch": 0.16, + "learning_rate": 0.000675691073219659, + "loss": 1.524, + "theoretical_loss": 3.910706059696774, + "tokens_seen": 540672000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006756750250752257, + "loss": 1.6002, + "theoretical_loss": 3.910656059281591, + "tokens_seen": 540737536 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006756589769307924, + "loss": 1.4989, + "theoretical_loss": 3.910606066622507, + "tokens_seen": 540803072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006756429287863591, + "loss": 1.5061, + "theoretical_loss": 3.9105560817173783, + "tokens_seen": 540868608 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006756268806419258, + "loss": 1.5122, + "theoretical_loss": 3.910506104564063, + "tokens_seen": 540934144 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006756108324974925, + "loss": 1.4927, + "theoretical_loss": 3.9104561351604206, + "tokens_seen": 540999680 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006755947843530592, + "loss": 1.4459, + "theoretical_loss": 3.9104061735043105, + "tokens_seen": 541065216 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006755787362086259, + "loss": 1.4923, + "theoretical_loss": 3.9103562195935933, + "tokens_seen": 541130752 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006755626880641926, + "loss": 1.6563, + "theoretical_loss": 3.91030627342613, + "tokens_seen": 541196288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006755466399197593, + "loss": 1.6458, + "theoretical_loss": 3.910256334999783, + "tokens_seen": 541261824 + }, + { + "epoch": 0.16, + "learning_rate": 0.000675530591775326, + "loss": 1.5889, + "theoretical_loss": 3.910206404312416, + "tokens_seen": 541327360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006755145436308927, + "loss": 1.554, + "theoretical_loss": 3.9101564813618923, + "tokens_seen": 541392896 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006754984954864594, + "loss": 1.555, + "theoretical_loss": 3.9101065661460774, + "tokens_seen": 541458432 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006754824473420261, + "loss": 1.5437, + "theoretical_loss": 3.9100566586628362, + "tokens_seen": 541523968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006754663991975928, + "loss": 1.6377, + "theoretical_loss": 3.9100067589100362, + "tokens_seen": 541589504 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006754503510531595, + "loss": 1.6353, + "theoretical_loss": 3.909956866885544, + "tokens_seen": 541655040 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006754343029087262, + "loss": 1.5299, + "theoretical_loss": 3.909906982587228, + "tokens_seen": 541720576 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006754182547642929, + "loss": 1.5558, + "theoretical_loss": 3.909857106012957, + "tokens_seen": 541786112 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006754022066198596, + "loss": 1.443, + "theoretical_loss": 3.9098072371606016, + "tokens_seen": 541851648 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006753861584754263, + "loss": 1.6684, + "theoretical_loss": 3.9097573760280326, + "tokens_seen": 541917184 + }, + { + "epoch": 0.16, + "learning_rate": 0.000675370110330993, + "loss": 1.5744, + "theoretical_loss": 3.90970752261312, + "tokens_seen": 541982720 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006753540621865597, + "loss": 1.5623, + "theoretical_loss": 3.9096576769137386, + "tokens_seen": 542048256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006753380140421264, + "loss": 1.4574, + "theoretical_loss": 3.90960783892776, + "tokens_seen": 542113792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006753219658976931, + "loss": 1.5085, + "theoretical_loss": 3.9095580086530592, + "tokens_seen": 542179328 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006753059177532598, + "loss": 1.5654, + "theoretical_loss": 3.90950818608751, + "tokens_seen": 542244864 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006752898696088265, + "loss": 1.5411, + "theoretical_loss": 3.9094583712289896, + "tokens_seen": 542310400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006752738214643933, + "loss": 1.5281, + "theoretical_loss": 3.909408564075374, + "tokens_seen": 542375936 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006752577733199599, + "loss": 1.4972, + "theoretical_loss": 3.9093587646245407, + "tokens_seen": 542441472 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006752417251755267, + "loss": 1.6179, + "theoretical_loss": 3.9093089728743684, + "tokens_seen": 542507008 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006752256770310933, + "loss": 1.5791, + "theoretical_loss": 3.9092591888227357, + "tokens_seen": 542572544 + }, + { + "epoch": 0.16, + "learning_rate": 0.00067520962888666, + "loss": 1.5638, + "theoretical_loss": 3.909209412467523, + "tokens_seen": 542638080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006751935807422267, + "loss": 1.5773, + "theoretical_loss": 3.9091596438066114, + "tokens_seen": 542703616 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006751775325977935, + "loss": 1.5819, + "theoretical_loss": 3.909109882837882, + "tokens_seen": 542769152 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006751614844533601, + "loss": 1.5461, + "theoretical_loss": 3.9090601295592178, + "tokens_seen": 542834688 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006751454363089269, + "loss": 1.4034, + "theoretical_loss": 3.9090103839685013, + "tokens_seen": 542900224 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006751293881644935, + "loss": 1.7304, + "theoretical_loss": 3.908960646063618, + "tokens_seen": 542965760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006751133400200603, + "loss": 1.5214, + "theoretical_loss": 3.908910915842452, + "tokens_seen": 543031296 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006750972918756269, + "loss": 1.4705, + "theoretical_loss": 3.90886119330289, + "tokens_seen": 543096832 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006750812437311937, + "loss": 1.4561, + "theoretical_loss": 3.908811478442818, + "tokens_seen": 543162368 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006750651955867603, + "loss": 1.5417, + "theoretical_loss": 3.908761771260124, + "tokens_seen": 543227904 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006750491474423271, + "loss": 1.4487, + "theoretical_loss": 3.908712071752696, + "tokens_seen": 543293440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006750330992978937, + "loss": 1.58, + "theoretical_loss": 3.908662379918424, + "tokens_seen": 543358976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006750170511534605, + "loss": 1.5634, + "theoretical_loss": 3.908612695755197, + "tokens_seen": 543424512 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006750010030090271, + "loss": 1.5591, + "theoretical_loss": 3.9085630192609067, + "tokens_seen": 543490048 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006749849548645939, + "loss": 1.5673, + "theoretical_loss": 3.908513350433444, + "tokens_seen": 543555584 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006749689067201605, + "loss": 1.5685, + "theoretical_loss": 3.908463689270703, + "tokens_seen": 543621120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006749528585757273, + "loss": 1.516, + "theoretical_loss": 3.9084140357705754, + "tokens_seen": 543686656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006749368104312939, + "loss": 1.5669, + "theoretical_loss": 3.9083643899309566, + "tokens_seen": 543752192 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006749207622868606, + "loss": 1.5921, + "theoretical_loss": 3.9083147517497414, + "tokens_seen": 543817728 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006749047141424273, + "loss": 1.5007, + "theoretical_loss": 3.908265121224825, + "tokens_seen": 543883264 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.9907113313674927, + "objective/train/avg_token_score": 0.009288685396313667, + "objective/train/docs_used": 204061, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.569307565689087, + "objective/train/original_loss": 1.584686040878296, + "objective/train/theoretical_loss": 3.908215498354105, + "objective/train/tokens_used": 564408800, + "objective/train/unlikelihood_loss": 0.019943062216043472, + "theoretical_loss": 3.908215498354105, + "tokens_seen": 543948800 + }, + { + "epoch": 0.16, + "learning_rate": 0.000674888665997994, + "loss": 1.5888, + "theoretical_loss": 3.908215498354105, + "tokens_seen": 543948800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006748726178535607, + "loss": 1.5349, + "theoretical_loss": 3.9081658831354784, + "tokens_seen": 544014336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006748565697091274, + "loss": 1.538, + "theoretical_loss": 3.908116275566844, + "tokens_seen": 544079872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006748405215646941, + "loss": 1.5256, + "theoretical_loss": 3.9080666756461007, + "tokens_seen": 544145408 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006748244734202608, + "loss": 1.5849, + "theoretical_loss": 3.908017083371149, + "tokens_seen": 544210944 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006748084252758275, + "loss": 1.5844, + "theoretical_loss": 3.9079674987398896, + "tokens_seen": 544276480 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006747923771313942, + "loss": 1.5788, + "theoretical_loss": 3.907917921750224, + "tokens_seen": 544342016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006747763289869609, + "loss": 1.3664, + "theoretical_loss": 3.9078683524000546, + "tokens_seen": 544407552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006747602808425276, + "loss": 1.6047, + "theoretical_loss": 3.907818790687285, + "tokens_seen": 544473088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006747442326980943, + "loss": 1.5419, + "theoretical_loss": 3.90776923660982, + "tokens_seen": 544538624 + }, + { + "epoch": 0.17, + "learning_rate": 0.000674728184553661, + "loss": 1.4816, + "theoretical_loss": 3.907719690165564, + "tokens_seen": 544604160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006747121364092277, + "loss": 1.4927, + "theoretical_loss": 3.9076701513524226, + "tokens_seen": 544669696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006746960882647944, + "loss": 1.5462, + "theoretical_loss": 3.9076206201683026, + "tokens_seen": 544735232 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006746800401203611, + "loss": 1.5047, + "theoretical_loss": 3.907571096611112, + "tokens_seen": 544800768 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006746639919759278, + "loss": 1.5322, + "theoretical_loss": 3.9075215806787584, + "tokens_seen": 544866304 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006746479438314945, + "loss": 1.5477, + "theoretical_loss": 3.907472072369152, + "tokens_seen": 544931840 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006746318956870612, + "loss": 1.6947, + "theoretical_loss": 3.907422571680202, + "tokens_seen": 544997376 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006746158475426279, + "loss": 1.5539, + "theoretical_loss": 3.907373078609819, + "tokens_seen": 545062912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006745997993981946, + "loss": 1.6296, + "theoretical_loss": 3.9073235931559154, + "tokens_seen": 545128448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006745837512537613, + "loss": 1.5659, + "theoretical_loss": 3.907274115316403, + "tokens_seen": 545193984 + }, + { + "epoch": 0.17, + "learning_rate": 0.000674567703109328, + "loss": 1.4739, + "theoretical_loss": 3.907224645089195, + "tokens_seen": 545259520 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006745516549648947, + "loss": 1.6636, + "theoretical_loss": 3.907175182472206, + "tokens_seen": 545325056 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006745356068204614, + "loss": 1.5035, + "theoretical_loss": 3.9071257274633506, + "tokens_seen": 545390592 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006745195586760281, + "loss": 1.6475, + "theoretical_loss": 3.9070762800605454, + "tokens_seen": 545456128 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006745035105315948, + "loss": 1.5962, + "theoretical_loss": 3.9070268402617057, + "tokens_seen": 545521664 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006744874623871615, + "loss": 1.5666, + "theoretical_loss": 3.9069774080647495, + "tokens_seen": 545587200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006744714142427282, + "loss": 1.6411, + "theoretical_loss": 3.9069279834675945, + "tokens_seen": 545652736 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006744553660982949, + "loss": 1.5898, + "theoretical_loss": 3.9068785664681602, + "tokens_seen": 545718272 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006744393179538616, + "loss": 1.5341, + "theoretical_loss": 3.906829157064367, + "tokens_seen": 545783808 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006744232698094283, + "loss": 1.5421, + "theoretical_loss": 3.906779755254134, + "tokens_seen": 545849344 + }, + { + "epoch": 0.17, + "learning_rate": 0.000674407221664995, + "loss": 1.5455, + "theoretical_loss": 3.9067303610353847, + "tokens_seen": 545914880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006743911735205617, + "loss": 1.591, + "theoretical_loss": 3.9066809744060396, + "tokens_seen": 545980416 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006743751253761284, + "loss": 1.5097, + "theoretical_loss": 3.9066315953640225, + "tokens_seen": 546045952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006743590772316951, + "loss": 1.6075, + "theoretical_loss": 3.9065822239072574, + "tokens_seen": 546111488 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006743430290872618, + "loss": 1.5872, + "theoretical_loss": 3.9065328600336695, + "tokens_seen": 546177024 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006743269809428285, + "loss": 1.519, + "theoretical_loss": 3.9064835037411836, + "tokens_seen": 546242560 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006743109327983952, + "loss": 1.5318, + "theoretical_loss": 3.9064341550277266, + "tokens_seen": 546308096 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006742948846539619, + "loss": 1.6079, + "theoretical_loss": 3.9063848138912247, + "tokens_seen": 546373632 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006742788365095286, + "loss": 1.5136, + "theoretical_loss": 3.9063354803296075, + "tokens_seen": 546439168 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006742627883650953, + "loss": 1.5401, + "theoretical_loss": 3.906286154340802, + "tokens_seen": 546504704 + }, + { + "epoch": 0.17, + "learning_rate": 0.000674246740220662, + "loss": 1.5752, + "theoretical_loss": 3.9062368359227397, + "tokens_seen": 546570240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006742306920762287, + "loss": 1.5212, + "theoretical_loss": 3.90618752507335, + "tokens_seen": 546635776 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006742146439317954, + "loss": 1.5862, + "theoretical_loss": 3.906138221790565, + "tokens_seen": 546701312 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006741985957873621, + "loss": 1.4942, + "theoretical_loss": 3.9060889260723153, + "tokens_seen": 546766848 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006741825476429288, + "loss": 1.5792, + "theoretical_loss": 3.906039637916535, + "tokens_seen": 546832384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006741664994984955, + "loss": 1.5778, + "theoretical_loss": 3.9059903573211576, + "tokens_seen": 546897920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006741504513540622, + "loss": 1.5536, + "theoretical_loss": 3.905941084284117, + "tokens_seen": 546963456 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006741344032096289, + "loss": 1.3449, + "theoretical_loss": 3.9058918188033496, + "tokens_seen": 547028992 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006741183550651956, + "loss": 1.6056, + "theoretical_loss": 3.905842560876791, + "tokens_seen": 547094528 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006741023069207623, + "loss": 1.5735, + "theoretical_loss": 3.9057933105023777, + "tokens_seen": 547160064 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.9929946064949036, + "objective/train/avg_token_score": 0.007005331106483936, + "objective/train/docs_used": 204823, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4288746118545532, + "objective/train/original_loss": 1.440773844718933, + "objective/train/theoretical_loss": 3.9057440676780484, + "objective/train/tokens_used": 567685600, + "objective/train/unlikelihood_loss": 0.01432681456208229, + "theoretical_loss": 3.9057440676780484, + "tokens_seen": 547225600 + }, + { + "epoch": 0.17, + "learning_rate": 0.000674086258776329, + "loss": 1.4941, + "theoretical_loss": 3.9057440676780484, + "tokens_seen": 547225600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006740702106318957, + "loss": 1.5183, + "theoretical_loss": 3.905694832401741, + "tokens_seen": 547291136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006740541624874624, + "loss": 1.6219, + "theoretical_loss": 3.905645604671395, + "tokens_seen": 547356672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006740381143430291, + "loss": 1.5096, + "theoretical_loss": 3.9055963844849506, + "tokens_seen": 547422208 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006740220661985958, + "loss": 1.4402, + "theoretical_loss": 3.9055471718403494, + "tokens_seen": 547487744 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006740060180541625, + "loss": 1.5212, + "theoretical_loss": 3.905497966735532, + "tokens_seen": 547553280 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006739899699097292, + "loss": 1.4971, + "theoretical_loss": 3.9054487691684425, + "tokens_seen": 547618816 + }, + { + "epoch": 0.17, + "learning_rate": 0.000673973921765296, + "loss": 1.4999, + "theoretical_loss": 3.905399579137023, + "tokens_seen": 547684352 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006739578736208626, + "loss": 1.5207, + "theoretical_loss": 3.9053503966392187, + "tokens_seen": 547749888 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006739418254764293, + "loss": 1.5246, + "theoretical_loss": 3.905301221672974, + "tokens_seen": 547815424 + }, + { + "epoch": 0.17, + "learning_rate": 0.000673925777331996, + "loss": 1.5198, + "theoretical_loss": 3.9052520542362346, + "tokens_seen": 547880960 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006739097291875627, + "loss": 1.604, + "theoretical_loss": 3.9052028943269477, + "tokens_seen": 547946496 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006738936810431294, + "loss": 1.4732, + "theoretical_loss": 3.905153741943061, + "tokens_seen": 548012032 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006738776328986962, + "loss": 1.5542, + "theoretical_loss": 3.905104597082522, + "tokens_seen": 548077568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006738615847542629, + "loss": 1.5655, + "theoretical_loss": 3.9050554597432807, + "tokens_seen": 548143104 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006738455366098296, + "loss": 1.5679, + "theoretical_loss": 3.905006329923286, + "tokens_seen": 548208640 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006738294884653963, + "loss": 1.6081, + "theoretical_loss": 3.904957207620489, + "tokens_seen": 548274176 + }, + { + "epoch": 0.17, + "learning_rate": 0.000673813440320963, + "loss": 1.5282, + "theoretical_loss": 3.904908092832841, + "tokens_seen": 548339712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006737973921765297, + "loss": 1.5029, + "theoretical_loss": 3.904858985558295, + "tokens_seen": 548405248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006737813440320964, + "loss": 1.6055, + "theoretical_loss": 3.904809885794803, + "tokens_seen": 548470784 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006737652958876631, + "loss": 1.4386, + "theoretical_loss": 3.9047607935403197, + "tokens_seen": 548536320 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006737492477432298, + "loss": 1.4549, + "theoretical_loss": 3.9047117087927994, + "tokens_seen": 548601856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006737331995987965, + "loss": 1.3757, + "theoretical_loss": 3.904662631550198, + "tokens_seen": 548667392 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006737171514543632, + "loss": 1.3786, + "theoretical_loss": 3.904613561810471, + "tokens_seen": 548732928 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006737011033099299, + "loss": 1.5382, + "theoretical_loss": 3.9045644995715767, + "tokens_seen": 548798464 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006736850551654966, + "loss": 1.6389, + "theoretical_loss": 3.9045154448314716, + "tokens_seen": 548864000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006736690070210632, + "loss": 1.5701, + "theoretical_loss": 3.9044663975881155, + "tokens_seen": 548929536 + }, + { + "epoch": 0.17, + "learning_rate": 0.00067365295887663, + "loss": 1.5302, + "theoretical_loss": 3.9044173578394674, + "tokens_seen": 548995072 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006736369107321966, + "loss": 1.5295, + "theoretical_loss": 3.904368325583488, + "tokens_seen": 549060608 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006736208625877634, + "loss": 1.5746, + "theoretical_loss": 3.9043193008181376, + "tokens_seen": 549126144 + }, + { + "epoch": 0.17, + "learning_rate": 0.00067360481444333, + "loss": 1.5188, + "theoretical_loss": 3.9042702835413787, + "tokens_seen": 549191680 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006735887662988968, + "loss": 1.5634, + "theoretical_loss": 3.9042212737511743, + "tokens_seen": 549257216 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006735727181544634, + "loss": 1.5617, + "theoretical_loss": 3.904172271445487, + "tokens_seen": 549322752 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006735566700100302, + "loss": 1.5512, + "theoretical_loss": 3.9041232766222813, + "tokens_seen": 549388288 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006735406218655968, + "loss": 1.5187, + "theoretical_loss": 3.9040742892795226, + "tokens_seen": 549453824 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006735245737211635, + "loss": 1.5813, + "theoretical_loss": 3.9040253094151773, + "tokens_seen": 549519360 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006735085255767302, + "loss": 1.6579, + "theoretical_loss": 3.9039763370272107, + "tokens_seen": 549584896 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006734924774322969, + "loss": 1.4079, + "theoretical_loss": 3.903927372113592, + "tokens_seen": 549650432 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006734764292878636, + "loss": 1.5444, + "theoretical_loss": 3.9038784146722874, + "tokens_seen": 549715968 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006734603811434303, + "loss": 1.5856, + "theoretical_loss": 3.9038294647012672, + "tokens_seen": 549781504 + }, + { + "epoch": 0.17, + "learning_rate": 0.000673444332998997, + "loss": 1.5297, + "theoretical_loss": 3.9037805221985016, + "tokens_seen": 549847040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006734282848545637, + "loss": 1.4792, + "theoretical_loss": 3.90373158716196, + "tokens_seen": 549912576 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006734122367101304, + "loss": 1.5892, + "theoretical_loss": 3.903682659589615, + "tokens_seen": 549978112 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006733961885656971, + "loss": 1.4859, + "theoretical_loss": 3.903633739479438, + "tokens_seen": 550043648 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006733801404212638, + "loss": 1.4995, + "theoretical_loss": 3.9035848268294027, + "tokens_seen": 550109184 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006733640922768305, + "loss": 1.4442, + "theoretical_loss": 3.9035359216374825, + "tokens_seen": 550174720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006733480441323972, + "loss": 1.6507, + "theoretical_loss": 3.9034870239016524, + "tokens_seen": 550240256 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006733319959879639, + "loss": 1.5625, + "theoretical_loss": 3.903438133619887, + "tokens_seen": 550305792 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006733159478435306, + "loss": 1.3799, + "theoretical_loss": 3.9033892507901635, + "tokens_seen": 550371328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006732998996990973, + "loss": 1.5829, + "theoretical_loss": 3.9033403754104583, + "tokens_seen": 550436864 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.9924510717391968, + "objective/train/avg_token_score": 0.007548890542238951, + "objective/train/docs_used": 206347, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6451343297958374, + "objective/train/original_loss": 1.657411813735962, + "objective/train/theoretical_loss": 3.9032915074787486, + "objective/train/tokens_used": 570962400, + "objective/train/unlikelihood_loss": 0.016790827736258507, + "theoretical_loss": 3.9032915074787486, + "tokens_seen": 550502400 + }, + { + "epoch": 0.17, + "learning_rate": 0.000673283851554664, + "loss": 1.5657, + "theoretical_loss": 3.9032915074787486, + "tokens_seen": 550502400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006732678034102307, + "loss": 1.5647, + "theoretical_loss": 3.903242646993014, + "tokens_seen": 550567936 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006732517552657974, + "loss": 1.542, + "theoretical_loss": 3.9031937939512336, + "tokens_seen": 550633472 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006732357071213641, + "loss": 1.5031, + "theoretical_loss": 3.903144948351387, + "tokens_seen": 550699008 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006732196589769308, + "loss": 1.4346, + "theoretical_loss": 3.9030961101914556, + "tokens_seen": 550764544 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006732036108324975, + "loss": 1.584, + "theoretical_loss": 3.903047279469421, + "tokens_seen": 550830080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006731875626880642, + "loss": 1.569, + "theoretical_loss": 3.9029984561832656, + "tokens_seen": 550895616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006731715145436309, + "loss": 1.5154, + "theoretical_loss": 3.9029496403309727, + "tokens_seen": 550961152 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006731554663991976, + "loss": 1.6885, + "theoretical_loss": 3.902900831910527, + "tokens_seen": 551026688 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006731394182547643, + "loss": 1.5857, + "theoretical_loss": 3.902852030919912, + "tokens_seen": 551092224 + }, + { + "epoch": 0.17, + "learning_rate": 0.000673123370110331, + "loss": 1.5165, + "theoretical_loss": 3.902803237357115, + "tokens_seen": 551157760 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006731073219658977, + "loss": 1.566, + "theoretical_loss": 3.9027544512201207, + "tokens_seen": 551223296 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006730912738214644, + "loss": 1.5478, + "theoretical_loss": 3.902705672506918, + "tokens_seen": 551288832 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006730752256770311, + "loss": 1.5414, + "theoretical_loss": 3.9026569012154932, + "tokens_seen": 551354368 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006730591775325978, + "loss": 1.5237, + "theoretical_loss": 3.9026081373438366, + "tokens_seen": 551419904 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006730431293881645, + "loss": 1.5056, + "theoretical_loss": 3.9025593808899366, + "tokens_seen": 551485440 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006730270812437312, + "loss": 1.6428, + "theoretical_loss": 3.9025106318517846, + "tokens_seen": 551550976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006730110330992979, + "loss": 1.4666, + "theoretical_loss": 3.902461890227371, + "tokens_seen": 551616512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006729949849548646, + "loss": 1.4396, + "theoretical_loss": 3.902413156014688, + "tokens_seen": 551682048 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006729789368104313, + "loss": 1.5201, + "theoretical_loss": 3.902364429211728, + "tokens_seen": 551747584 + }, + { + "epoch": 0.17, + "learning_rate": 0.000672962888665998, + "loss": 1.5449, + "theoretical_loss": 3.9023157098164853, + "tokens_seen": 551813120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006729468405215647, + "loss": 1.5782, + "theoretical_loss": 3.902266997826953, + "tokens_seen": 551878656 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006729307923771314, + "loss": 1.4442, + "theoretical_loss": 3.902218293241127, + "tokens_seen": 551944192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006729147442326981, + "loss": 1.4834, + "theoretical_loss": 3.9021695960570026, + "tokens_seen": 552009728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006728986960882648, + "loss": 1.5511, + "theoretical_loss": 3.902120906272577, + "tokens_seen": 552075264 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006728826479438315, + "loss": 1.5168, + "theoretical_loss": 3.9020722238858467, + "tokens_seen": 552140800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006728665997993982, + "loss": 1.4986, + "theoretical_loss": 3.902023548894811, + "tokens_seen": 552206336 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006728505516549649, + "loss": 1.6469, + "theoretical_loss": 3.901974881297468, + "tokens_seen": 552271872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006728345035105316, + "loss": 1.5227, + "theoretical_loss": 3.9019262210918173, + "tokens_seen": 552337408 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006728184553660983, + "loss": 1.405, + "theoretical_loss": 3.90187756827586, + "tokens_seen": 552402944 + }, + { + "epoch": 0.17, + "learning_rate": 0.000672802407221665, + "loss": 1.5097, + "theoretical_loss": 3.9018289228475975, + "tokens_seen": 552468480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006727863590772317, + "loss": 1.6409, + "theoretical_loss": 3.9017802848050307, + "tokens_seen": 552534016 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006727703109327984, + "loss": 1.4384, + "theoretical_loss": 3.9017316541461637, + "tokens_seen": 552599552 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006727542627883651, + "loss": 1.6332, + "theoretical_loss": 3.9016830308689996, + "tokens_seen": 552665088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006727382146439318, + "loss": 1.5643, + "theoretical_loss": 3.901634414971543, + "tokens_seen": 552730624 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006727221664994985, + "loss": 1.474, + "theoretical_loss": 3.901585806451799, + "tokens_seen": 552796160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006727061183550652, + "loss": 1.4355, + "theoretical_loss": 3.901537205307773, + "tokens_seen": 552861696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006726900702106319, + "loss": 1.3742, + "theoretical_loss": 3.9014886115374723, + "tokens_seen": 552927232 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006726740220661986, + "loss": 1.5925, + "theoretical_loss": 3.9014400251389043, + "tokens_seen": 552992768 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006726579739217654, + "loss": 1.5209, + "theoretical_loss": 3.901391446110077, + "tokens_seen": 553058304 + }, + { + "epoch": 0.17, + "learning_rate": 0.000672641925777332, + "loss": 1.6008, + "theoretical_loss": 3.901342874449, + "tokens_seen": 553123840 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006726258776328988, + "loss": 1.4576, + "theoretical_loss": 3.9012943101536823, + "tokens_seen": 553189376 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006726098294884654, + "loss": 1.5314, + "theoretical_loss": 3.901245753222135, + "tokens_seen": 553254912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006725937813440322, + "loss": 1.5619, + "theoretical_loss": 3.90119720365237, + "tokens_seen": 553320448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006725777331995988, + "loss": 1.5045, + "theoretical_loss": 3.9011486614423982, + "tokens_seen": 553385984 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006725616850551656, + "loss": 1.5567, + "theoretical_loss": 3.901100126590233, + "tokens_seen": 553451520 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006725456369107322, + "loss": 1.4754, + "theoretical_loss": 3.901051599093889, + "tokens_seen": 553517056 + }, + { + "epoch": 0.17, + "learning_rate": 0.000672529588766299, + "loss": 1.6093, + "theoretical_loss": 3.901003078951379, + "tokens_seen": 553582592 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006725135406218657, + "loss": 1.4907, + "theoretical_loss": 3.9009545661607192, + "tokens_seen": 553648128 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006724974924774324, + "loss": 1.6402, + "theoretical_loss": 3.900906060719925, + "tokens_seen": 553713664 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.9914413690567017, + "objective/train/avg_token_score": 0.00855861883610487, + "objective/train/docs_used": 207589, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.7540210485458374, + "objective/train/original_loss": 1.7703471183776855, + "objective/train/theoretical_loss": 3.9008575626270146, + "objective/train/tokens_used": 574239200, + "objective/train/unlikelihood_loss": 0.01722062937915325, + "theoretical_loss": 3.9008575626270146, + "tokens_seen": 553779200 + }, + { + "epoch": 0.17, + "learning_rate": 0.000672481444332999, + "loss": 1.5854, + "theoretical_loss": 3.9008575626270146, + "tokens_seen": 553779200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006724653961885658, + "loss": 1.4872, + "theoretical_loss": 3.900809071880004, + "tokens_seen": 553844736 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006724493480441325, + "loss": 1.5636, + "theoretical_loss": 3.900760588476912, + "tokens_seen": 553910272 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006724332998996992, + "loss": 1.6263, + "theoretical_loss": 3.900712112415758, + "tokens_seen": 553975808 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006724172517552659, + "loss": 1.5383, + "theoretical_loss": 3.900663643694561, + "tokens_seen": 554041344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006724012036108326, + "loss": 1.5127, + "theoretical_loss": 3.900615182311342, + "tokens_seen": 554106880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006723851554663993, + "loss": 1.5291, + "theoretical_loss": 3.900566728264123, + "tokens_seen": 554172416 + }, + { + "epoch": 0.17, + "learning_rate": 0.000672369107321966, + "loss": 1.492, + "theoretical_loss": 3.9005182815509256, + "tokens_seen": 554237952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006723530591775327, + "loss": 1.4807, + "theoretical_loss": 3.9004698421697723, + "tokens_seen": 554303488 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006723370110330994, + "loss": 1.6689, + "theoretical_loss": 3.900421410118688, + "tokens_seen": 554369024 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006723209628886661, + "loss": 1.4483, + "theoretical_loss": 3.9003729853956957, + "tokens_seen": 554434560 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006723049147442328, + "loss": 1.6067, + "theoretical_loss": 3.9003245679988217, + "tokens_seen": 554500096 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006722888665997995, + "loss": 1.564, + "theoretical_loss": 3.9002761579260907, + "tokens_seen": 554565632 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006722728184553661, + "loss": 1.4883, + "theoretical_loss": 3.9002277551755307, + "tokens_seen": 554631168 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006722567703109329, + "loss": 1.5093, + "theoretical_loss": 3.9001793597451684, + "tokens_seen": 554696704 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006722407221664995, + "loss": 1.5467, + "theoretical_loss": 3.900130971633033, + "tokens_seen": 554762240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006722246740220663, + "loss": 1.5067, + "theoretical_loss": 3.9000825908371524, + "tokens_seen": 554827776 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006722086258776329, + "loss": 1.4739, + "theoretical_loss": 3.9000342173555573, + "tokens_seen": 554893312 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006721925777331997, + "loss": 1.5899, + "theoretical_loss": 3.899985851186277, + "tokens_seen": 554958848 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006721765295887663, + "loss": 1.4702, + "theoretical_loss": 3.8999374923273447, + "tokens_seen": 555024384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006721604814443331, + "loss": 1.5901, + "theoretical_loss": 3.899889140776791, + "tokens_seen": 555089920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006721444332998997, + "loss": 1.5042, + "theoretical_loss": 3.8998407965326494, + "tokens_seen": 555155456 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006721283851554664, + "loss": 1.4244, + "theoretical_loss": 3.899792459592953, + "tokens_seen": 555220992 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006721123370110331, + "loss": 1.4341, + "theoretical_loss": 3.899744129955737, + "tokens_seen": 555286528 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006720962888665998, + "loss": 1.4849, + "theoretical_loss": 3.899695807619035, + "tokens_seen": 555352064 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006720802407221665, + "loss": 1.5648, + "theoretical_loss": 3.899647492580885, + "tokens_seen": 555417600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006720641925777332, + "loss": 1.5911, + "theoretical_loss": 3.899599184839322, + "tokens_seen": 555483136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006720481444332999, + "loss": 1.4497, + "theoretical_loss": 3.899550884392384, + "tokens_seen": 555548672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006720320962888666, + "loss": 1.4076, + "theoretical_loss": 3.8995025912381087, + "tokens_seen": 555614208 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006720160481444333, + "loss": 1.4991, + "theoretical_loss": 3.899454305374536, + "tokens_seen": 555679744 + }, + { + "epoch": 0.17, + "learning_rate": 0.000672, + "loss": 1.4822, + "theoretical_loss": 3.899406026799704, + "tokens_seen": 555745280 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006719839518555667, + "loss": 1.4862, + "theoretical_loss": 3.899357755511655, + "tokens_seen": 555810816 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006719679037111334, + "loss": 1.6418, + "theoretical_loss": 3.899309491508429, + "tokens_seen": 555876352 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006719518555667001, + "loss": 1.6046, + "theoretical_loss": 3.899261234788068, + "tokens_seen": 555941888 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006719358074222668, + "loss": 1.3012, + "theoretical_loss": 3.8992129853486155, + "tokens_seen": 556007424 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006719197592778335, + "loss": 1.508, + "theoretical_loss": 3.8991647431881145, + "tokens_seen": 556072960 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006719037111334002, + "loss": 1.6292, + "theoretical_loss": 3.899116508304609, + "tokens_seen": 556138496 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006718876629889669, + "loss": 1.6128, + "theoretical_loss": 3.899068280696144, + "tokens_seen": 556204032 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006718716148445336, + "loss": 1.4082, + "theoretical_loss": 3.8990200603607654, + "tokens_seen": 556269568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006718555667001003, + "loss": 1.5812, + "theoretical_loss": 3.8989718472965196, + "tokens_seen": 556335104 + }, + { + "epoch": 0.17, + "learning_rate": 0.000671839518555667, + "loss": 1.5502, + "theoretical_loss": 3.898923641501454, + "tokens_seen": 556400640 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006718234704112337, + "loss": 1.503, + "theoretical_loss": 3.898875442973617, + "tokens_seen": 556466176 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006718074222668004, + "loss": 1.4774, + "theoretical_loss": 3.8988272517110563, + "tokens_seen": 556531712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006717913741223671, + "loss": 1.4672, + "theoretical_loss": 3.898779067711822, + "tokens_seen": 556597248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006717753259779338, + "loss": 1.4965, + "theoretical_loss": 3.8987308909739644, + "tokens_seen": 556662784 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006717592778335005, + "loss": 1.6359, + "theoretical_loss": 3.898682721495535, + "tokens_seen": 556728320 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006717432296890672, + "loss": 1.5398, + "theoretical_loss": 3.8986345592745844, + "tokens_seen": 556793856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006717271815446339, + "loss": 1.5305, + "theoretical_loss": 3.898586404309166, + "tokens_seen": 556859392 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006717111334002006, + "loss": 1.4502, + "theoretical_loss": 3.8985382565973326, + "tokens_seen": 556924928 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006716950852557673, + "loss": 1.4685, + "theoretical_loss": 3.898490116137139, + "tokens_seen": 556990464 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.9942537546157837, + "objective/train/avg_token_score": 0.0057462709955871105, + "objective/train/docs_used": 208740, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5766122341156006, + "objective/train/original_loss": 1.585820198059082, + "objective/train/theoretical_loss": 3.898441982926639, + "objective/train/tokens_used": 577516000, + "objective/train/unlikelihood_loss": 0.014643192291259766, + "theoretical_loss": 3.898441982926639, + "tokens_seen": 557056000 + }, + { + "epoch": 0.17, + "learning_rate": 0.000671679037111334, + "loss": 1.4873, + "theoretical_loss": 3.898441982926639, + "tokens_seen": 557056000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006716629889669007, + "loss": 1.4233, + "theoretical_loss": 3.8983938569638883, + "tokens_seen": 557121536 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006716469408224674, + "loss": 1.5699, + "theoretical_loss": 3.8983457382469435, + "tokens_seen": 557187072 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006716308926780341, + "loss": 1.4693, + "theoretical_loss": 3.8982976267738616, + "tokens_seen": 557252608 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006716148445336008, + "loss": 1.4154, + "theoretical_loss": 3.8982495225427005, + "tokens_seen": 557318144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006715987963891675, + "loss": 1.4663, + "theoretical_loss": 3.8982014255515183, + "tokens_seen": 557383680 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006715827482447342, + "loss": 1.5274, + "theoretical_loss": 3.8981533357983746, + "tokens_seen": 557449216 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006715667001003009, + "loss": 1.4997, + "theoretical_loss": 3.8981052532813294, + "tokens_seen": 557514752 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006715506519558676, + "loss": 1.4937, + "theoretical_loss": 3.898057177998443, + "tokens_seen": 557580288 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006715346038114343, + "loss": 1.4534, + "theoretical_loss": 3.898009109947777, + "tokens_seen": 557645824 + }, + { + "epoch": 0.17, + "learning_rate": 0.000671518555667001, + "loss": 1.4208, + "theoretical_loss": 3.897961049127395, + "tokens_seen": 557711360 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006715025075225677, + "loss": 1.4858, + "theoretical_loss": 3.8979129955353584, + "tokens_seen": 557776896 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006714864593781344, + "loss": 1.4906, + "theoretical_loss": 3.8978649491697315, + "tokens_seen": 557842432 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006714704112337011, + "loss": 1.4761, + "theoretical_loss": 3.8978169100285784, + "tokens_seen": 557907968 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006714543630892679, + "loss": 1.4484, + "theoretical_loss": 3.8977688781099653, + "tokens_seen": 557973504 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006714383149448345, + "loss": 1.4814, + "theoretical_loss": 3.8977208534119576, + "tokens_seen": 558039040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006714222668004013, + "loss": 1.5969, + "theoretical_loss": 3.8976728359326223, + "tokens_seen": 558104576 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006714062186559679, + "loss": 1.4922, + "theoretical_loss": 3.897624825670026, + "tokens_seen": 558170112 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006713901705115347, + "loss": 1.5842, + "theoretical_loss": 3.8975768226222383, + "tokens_seen": 558235648 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006713741223671013, + "loss": 1.5195, + "theoretical_loss": 3.897528826787327, + "tokens_seen": 558301184 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006713580742226681, + "loss": 1.5225, + "theoretical_loss": 3.8974808381633625, + "tokens_seen": 558366720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006713420260782347, + "loss": 1.6559, + "theoretical_loss": 3.8974328567484147, + "tokens_seen": 558432256 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006713259779338015, + "loss": 1.5824, + "theoretical_loss": 3.8973848825405555, + "tokens_seen": 558497792 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006713099297893681, + "loss": 1.5199, + "theoretical_loss": 3.8973369155378563, + "tokens_seen": 558563328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006712938816449349, + "loss": 1.5308, + "theoretical_loss": 3.89728895573839, + "tokens_seen": 558628864 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006712778335005015, + "loss": 1.5398, + "theoretical_loss": 3.8972410031402296, + "tokens_seen": 558694400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006712617853560683, + "loss": 1.5637, + "theoretical_loss": 3.8971930577414495, + "tokens_seen": 558759936 + }, + { + "epoch": 0.17, + "learning_rate": 0.000671245737211635, + "loss": 1.5734, + "theoretical_loss": 3.8971451195401254, + "tokens_seen": 558825472 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006712296890672017, + "loss": 1.5455, + "theoretical_loss": 3.897097188534332, + "tokens_seen": 558891008 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006712136409227683, + "loss": 1.5712, + "theoretical_loss": 3.8970492647221464, + "tokens_seen": 558956544 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006711975927783351, + "loss": 1.5926, + "theoretical_loss": 3.897001348101645, + "tokens_seen": 559022080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006711815446339017, + "loss": 1.4839, + "theoretical_loss": 3.8969534386709053, + "tokens_seen": 559087616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006711654964894685, + "loss": 1.6006, + "theoretical_loss": 3.896905536428007, + "tokens_seen": 559153152 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006711494483450352, + "loss": 1.5918, + "theoretical_loss": 3.8968576413710294, + "tokens_seen": 559218688 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006711334002006019, + "loss": 1.5314, + "theoretical_loss": 3.8968097534980517, + "tokens_seen": 559284224 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006711173520561686, + "loss": 1.4818, + "theoretical_loss": 3.8967618728071556, + "tokens_seen": 559349760 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006711013039117353, + "loss": 1.5564, + "theoretical_loss": 3.896713999296422, + "tokens_seen": 559415296 + }, + { + "epoch": 0.17, + "learning_rate": 0.000671085255767302, + "loss": 1.5428, + "theoretical_loss": 3.8966661329639334, + "tokens_seen": 559480832 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006710692076228687, + "loss": 1.5739, + "theoretical_loss": 3.896618273807773, + "tokens_seen": 559546368 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006710531594784354, + "loss": 1.5241, + "theoretical_loss": 3.8965704218260244, + "tokens_seen": 559611904 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006710371113340021, + "loss": 1.5953, + "theoretical_loss": 3.896522577016772, + "tokens_seen": 559677440 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006710210631895688, + "loss": 1.5532, + "theoretical_loss": 3.896474739378102, + "tokens_seen": 559742976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006710050150451355, + "loss": 1.5751, + "theoretical_loss": 3.896426908908099, + "tokens_seen": 559808512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006709889669007022, + "loss": 1.5193, + "theoretical_loss": 3.89637908560485, + "tokens_seen": 559874048 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006709729187562689, + "loss": 1.5087, + "theoretical_loss": 3.896331269466443, + "tokens_seen": 559939584 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006709568706118356, + "loss": 1.4583, + "theoretical_loss": 3.8962834604909657, + "tokens_seen": 560005120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006709408224674023, + "loss": 1.4657, + "theoretical_loss": 3.8962356586765075, + "tokens_seen": 560070656 + }, + { + "epoch": 0.17, + "learning_rate": 0.000670924774322969, + "loss": 1.4466, + "theoretical_loss": 3.896187864021157, + "tokens_seen": 560136192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006709087261785357, + "loss": 1.5732, + "theoretical_loss": 3.8961400765230065, + "tokens_seen": 560201728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006708926780341024, + "loss": 1.497, + "theoretical_loss": 3.8960922961801447, + "tokens_seen": 560267264 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.9928003549575806, + "objective/train/avg_token_score": 0.00719958683475852, + "objective/train/docs_used": 209977, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4785184860229492, + "objective/train/original_loss": 1.4902803897857666, + "objective/train/theoretical_loss": 3.8960445229906657, + "objective/train/tokens_used": 580792800, + "objective/train/unlikelihood_loss": 0.017253099009394646, + "theoretical_loss": 3.8960445229906657, + "tokens_seen": 560332800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006708766298896691, + "loss": 1.4553, + "theoretical_loss": 3.8960445229906657, + "tokens_seen": 560332800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006708605817452358, + "loss": 1.4894, + "theoretical_loss": 3.895996756952661, + "tokens_seen": 560398336 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006708445336008024, + "loss": 1.4878, + "theoretical_loss": 3.895948998064223, + "tokens_seen": 560463872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006708284854563692, + "loss": 1.5113, + "theoretical_loss": 3.8959012463234473, + "tokens_seen": 560529408 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006708124373119358, + "loss": 1.3533, + "theoretical_loss": 3.8958535017284284, + "tokens_seen": 560594944 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006707963891675026, + "loss": 1.4667, + "theoretical_loss": 3.895805764277261, + "tokens_seen": 560660480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006707803410230692, + "loss": 1.5513, + "theoretical_loss": 3.895758033968042, + "tokens_seen": 560726016 + }, + { + "epoch": 0.17, + "learning_rate": 0.000670764292878636, + "loss": 1.4683, + "theoretical_loss": 3.895710310798868, + "tokens_seen": 560791552 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006707482447342026, + "loss": 1.5326, + "theoretical_loss": 3.895662594767837, + "tokens_seen": 560857088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006707321965897693, + "loss": 1.5913, + "theoretical_loss": 3.895614885873047, + "tokens_seen": 560922624 + }, + { + "epoch": 0.17, + "learning_rate": 0.000670716148445336, + "loss": 1.5312, + "theoretical_loss": 3.895567184112598, + "tokens_seen": 560988160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006707001003009027, + "loss": 1.6297, + "theoretical_loss": 3.8955194894845886, + "tokens_seen": 561053696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006706840521564694, + "loss": 1.4936, + "theoretical_loss": 3.89547180198712, + "tokens_seen": 561119232 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006706680040120361, + "loss": 1.4351, + "theoretical_loss": 3.8954241216182943, + "tokens_seen": 561184768 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006706519558676028, + "loss": 1.5395, + "theoretical_loss": 3.8953764483762123, + "tokens_seen": 561250304 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006706359077231695, + "loss": 1.4756, + "theoretical_loss": 3.8953287822589777, + "tokens_seen": 561315840 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006706198595787362, + "loss": 1.408, + "theoretical_loss": 3.8952811232646933, + "tokens_seen": 561381376 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006706038114343029, + "loss": 1.5696, + "theoretical_loss": 3.895233471391464, + "tokens_seen": 561446912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006705877632898696, + "loss": 1.4363, + "theoretical_loss": 3.895185826637394, + "tokens_seen": 561512448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006705717151454363, + "loss": 1.6089, + "theoretical_loss": 3.89513818900059, + "tokens_seen": 561577984 + }, + { + "epoch": 0.17, + "learning_rate": 0.000670555667001003, + "loss": 1.4978, + "theoretical_loss": 3.8950905584791577, + "tokens_seen": 561643520 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006705396188565697, + "loss": 1.5366, + "theoretical_loss": 3.8950429350712037, + "tokens_seen": 561709056 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006705235707121364, + "loss": 1.4898, + "theoretical_loss": 3.8949953187748374, + "tokens_seen": 561774592 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006705075225677031, + "loss": 1.5193, + "theoretical_loss": 3.894947709588166, + "tokens_seen": 561840128 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006704914744232698, + "loss": 1.4059, + "theoretical_loss": 3.8949001075093, + "tokens_seen": 561905664 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006704754262788365, + "loss": 1.5626, + "theoretical_loss": 3.8948525125363487, + "tokens_seen": 561971200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006704593781344032, + "loss": 1.5543, + "theoretical_loss": 3.8948049246674223, + "tokens_seen": 562036736 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006704433299899699, + "loss": 1.5013, + "theoretical_loss": 3.894757343900633, + "tokens_seen": 562102272 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006704272818455366, + "loss": 1.5973, + "theoretical_loss": 3.894709770234093, + "tokens_seen": 562167808 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006704112337011033, + "loss": 1.5309, + "theoretical_loss": 3.8946622036659155, + "tokens_seen": 562233344 + }, + { + "epoch": 0.17, + "learning_rate": 0.00067039518555667, + "loss": 1.6564, + "theoretical_loss": 3.8946146441942138, + "tokens_seen": 562298880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006703791374122367, + "loss": 1.5245, + "theoretical_loss": 3.894567091817102, + "tokens_seen": 562364416 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006703630892678034, + "loss": 1.4511, + "theoretical_loss": 3.8945195465326954, + "tokens_seen": 562429952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006703470411233701, + "loss": 1.5028, + "theoretical_loss": 3.8944720083391093, + "tokens_seen": 562495488 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006703309929789368, + "loss": 1.5487, + "theoretical_loss": 3.8944244772344616, + "tokens_seen": 562561024 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006703149448345035, + "loss": 1.4887, + "theoretical_loss": 3.894376953216868, + "tokens_seen": 562626560 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006702988966900702, + "loss": 1.5559, + "theoretical_loss": 3.8943294362844476, + "tokens_seen": 562692096 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006702828485456369, + "loss": 1.5493, + "theoretical_loss": 3.8942819264353186, + "tokens_seen": 562757632 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006702668004012036, + "loss": 1.5431, + "theoretical_loss": 3.8942344236676005, + "tokens_seen": 562823168 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006702507522567704, + "loss": 1.5448, + "theoretical_loss": 3.894186927979413, + "tokens_seen": 562888704 + }, + { + "epoch": 0.17, + "learning_rate": 0.000670234704112337, + "loss": 1.6521, + "theoretical_loss": 3.8941394393688777, + "tokens_seen": 562954240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006702186559679038, + "loss": 1.5143, + "theoretical_loss": 3.8940919578341155, + "tokens_seen": 563019776 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006702026078234704, + "loss": 1.513, + "theoretical_loss": 3.894044483373249, + "tokens_seen": 563085312 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006701865596790372, + "loss": 1.6627, + "theoretical_loss": 3.893997015984401, + "tokens_seen": 563150848 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006701705115346038, + "loss": 1.5401, + "theoretical_loss": 3.893949555665696, + "tokens_seen": 563216384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006701544633901706, + "loss": 1.4164, + "theoretical_loss": 3.893902102415257, + "tokens_seen": 563281920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006701384152457372, + "loss": 1.6505, + "theoretical_loss": 3.89385465623121, + "tokens_seen": 563347456 + }, + { + "epoch": 0.17, + "learning_rate": 0.000670122367101304, + "loss": 1.5396, + "theoretical_loss": 3.8938072171116804, + "tokens_seen": 563412992 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006701063189568706, + "loss": 1.4668, + "theoretical_loss": 3.893759785054796, + "tokens_seen": 563478528 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006700902708124374, + "loss": 1.4783, + "theoretical_loss": 3.8937123600586823, + "tokens_seen": 563544064 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.9916663765907288, + "objective/train/avg_token_score": 0.00833361130207777, + "objective/train/docs_used": 211161, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6325665712356567, + "objective/train/original_loss": 1.6481006145477295, + "objective/train/theoretical_loss": 3.8936649421214686, + "objective/train/tokens_used": 584069600, + "objective/train/unlikelihood_loss": 0.016445279121398926, + "theoretical_loss": 3.8936649421214686, + "tokens_seen": 563609600 + }, + { + "epoch": 0.17, + "learning_rate": 0.000670074222668004, + "loss": 1.5193, + "theoretical_loss": 3.8936649421214686, + "tokens_seen": 563609600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006700581745235708, + "loss": 1.5438, + "theoretical_loss": 3.8936175312412833, + "tokens_seen": 563675136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006700421263791374, + "loss": 1.5158, + "theoretical_loss": 3.8935701274162553, + "tokens_seen": 563740672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006700260782347042, + "loss": 1.4616, + "theoretical_loss": 3.8935227306445155, + "tokens_seen": 563806208 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006700100300902708, + "loss": 1.5875, + "theoretical_loss": 3.8934753409241942, + "tokens_seen": 563871744 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006699939819458376, + "loss": 1.6013, + "theoretical_loss": 3.8934279582534232, + "tokens_seen": 563937280 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006699779338014042, + "loss": 1.5611, + "theoretical_loss": 3.8933805826303347, + "tokens_seen": 564002816 + }, + { + "epoch": 0.17, + "learning_rate": 0.000669961885656971, + "loss": 1.4827, + "theoretical_loss": 3.893333214053062, + "tokens_seen": 564068352 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006699458375125376, + "loss": 1.5825, + "theoretical_loss": 3.8932858525197385, + "tokens_seen": 564133888 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006699297893681044, + "loss": 1.5624, + "theoretical_loss": 3.8932384980284986, + "tokens_seen": 564199424 + }, + { + "epoch": 0.17, + "learning_rate": 0.000669913741223671, + "loss": 1.4478, + "theoretical_loss": 3.8931911505774766, + "tokens_seen": 564264960 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006698976930792378, + "loss": 1.538, + "theoretical_loss": 3.89314381016481, + "tokens_seen": 564330496 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006698816449348044, + "loss": 1.528, + "theoretical_loss": 3.8930964767886347, + "tokens_seen": 564396032 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006698655967903712, + "loss": 1.4502, + "theoretical_loss": 3.893049150447087, + "tokens_seen": 564461568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006698495486459378, + "loss": 1.5078, + "theoretical_loss": 3.8930018311383066, + "tokens_seen": 564527104 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006698335005015046, + "loss": 1.4234, + "theoretical_loss": 3.8929545188604306, + "tokens_seen": 564592640 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006698174523570712, + "loss": 1.5143, + "theoretical_loss": 3.8929072136115987, + "tokens_seen": 564658176 + }, + { + "epoch": 0.17, + "learning_rate": 0.000669801404212638, + "loss": 1.5329, + "theoretical_loss": 3.8928599153899515, + "tokens_seen": 564723712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006697853560682047, + "loss": 1.4679, + "theoretical_loss": 3.8928126241936294, + "tokens_seen": 564789248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006697693079237714, + "loss": 1.4659, + "theoretical_loss": 3.8927653400207745, + "tokens_seen": 564854784 + }, + { + "epoch": 0.17, + "learning_rate": 0.000669753259779338, + "loss": 1.4646, + "theoretical_loss": 3.892718062869528, + "tokens_seen": 564920320 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006697372116349048, + "loss": 1.5588, + "theoretical_loss": 3.8926707927380337, + "tokens_seen": 564985856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006697211634904715, + "loss": 1.5643, + "theoretical_loss": 3.892623529624435, + "tokens_seen": 565051392 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006697051153460382, + "loss": 1.4625, + "theoretical_loss": 3.892576273526876, + "tokens_seen": 565116928 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006696890672016049, + "loss": 1.5192, + "theoretical_loss": 3.892529024443502, + "tokens_seen": 565182464 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006696730190571716, + "loss": 1.6557, + "theoretical_loss": 3.892481782372458, + "tokens_seen": 565248000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006696569709127383, + "loss": 1.4611, + "theoretical_loss": 3.8924345473118915, + "tokens_seen": 565313536 + }, + { + "epoch": 0.17, + "learning_rate": 0.000669640922768305, + "loss": 1.5733, + "theoretical_loss": 3.892387319259949, + "tokens_seen": 565379072 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006696248746238717, + "loss": 1.5428, + "theoretical_loss": 3.8923400982147784, + "tokens_seen": 565444608 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006696088264794384, + "loss": 1.5329, + "theoretical_loss": 3.8922928841745286, + "tokens_seen": 565510144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006695927783350051, + "loss": 1.6043, + "theoretical_loss": 3.892245677137349, + "tokens_seen": 565575680 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006695767301905718, + "loss": 1.4836, + "theoretical_loss": 3.8921984771013887, + "tokens_seen": 565641216 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006695606820461385, + "loss": 1.5065, + "theoretical_loss": 3.8921512840647985, + "tokens_seen": 565706752 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006695446339017052, + "loss": 1.4843, + "theoretical_loss": 3.892104098025731, + "tokens_seen": 565772288 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006695285857572719, + "loss": 1.5104, + "theoretical_loss": 3.892056918982337, + "tokens_seen": 565837824 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006695125376128386, + "loss": 1.6239, + "theoretical_loss": 3.8920097469327697, + "tokens_seen": 565903360 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006694964894684053, + "loss": 1.4758, + "theoretical_loss": 3.8919625818751826, + "tokens_seen": 565968896 + }, + { + "epoch": 0.17, + "learning_rate": 0.000669480441323972, + "loss": 1.5095, + "theoretical_loss": 3.8919154238077294, + "tokens_seen": 566034432 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006694643931795387, + "loss": 1.4777, + "theoretical_loss": 3.8918682727285665, + "tokens_seen": 566099968 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006694483450351053, + "loss": 1.282, + "theoretical_loss": 3.8918211286358475, + "tokens_seen": 566165504 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006694322968906721, + "loss": 1.5238, + "theoretical_loss": 3.89177399152773, + "tokens_seen": 566231040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006694162487462387, + "loss": 1.5386, + "theoretical_loss": 3.89172686140237, + "tokens_seen": 566296576 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006694002006018055, + "loss": 1.5211, + "theoretical_loss": 3.8916797382579262, + "tokens_seen": 566362112 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006693841524573721, + "loss": 1.5077, + "theoretical_loss": 3.8916326220925566, + "tokens_seen": 566427648 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006693681043129389, + "loss": 1.4119, + "theoretical_loss": 3.8915855129044195, + "tokens_seen": 566493184 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006693520561685055, + "loss": 1.5041, + "theoretical_loss": 3.891538410691676, + "tokens_seen": 566558720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006693360080240723, + "loss": 1.4729, + "theoretical_loss": 3.8914913154524857, + "tokens_seen": 566624256 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006693199598796389, + "loss": 1.458, + "theoretical_loss": 3.89144422718501, + "tokens_seen": 566689792 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006693039117352056, + "loss": 1.5142, + "theoretical_loss": 3.891397145887411, + "tokens_seen": 566755328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006692878635907723, + "loss": 1.5729, + "theoretical_loss": 3.8913500715578504, + "tokens_seen": 566820864 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.9925703406333923, + "objective/train/avg_token_score": 0.007429648190736771, + "objective/train/docs_used": 212374, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.520574927330017, + "objective/train/original_loss": 1.532004475593567, + "objective/train/theoretical_loss": 3.8913030041944925, + "objective/train/tokens_used": 587346400, + "objective/train/unlikelihood_loss": 0.01818997412919998, + "theoretical_loss": 3.8913030041944925, + "tokens_seen": 566886400 + }, + { + "epoch": 0.17, + "learning_rate": 0.000669271815446339, + "loss": 1.5789, + "theoretical_loss": 3.8913030041944925, + "tokens_seen": 566886400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006692557673019057, + "loss": 1.5432, + "theoretical_loss": 3.891255943795501, + "tokens_seen": 566951936 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006692397191574724, + "loss": 1.5459, + "theoretical_loss": 3.8912088903590405, + "tokens_seen": 567017472 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006692236710130391, + "loss": 1.4907, + "theoretical_loss": 3.8911618438832756, + "tokens_seen": 567083008 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006692076228686058, + "loss": 1.517, + "theoretical_loss": 3.8911148043663735, + "tokens_seen": 567148544 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006691915747241725, + "loss": 1.4289, + "theoretical_loss": 3.8910677718065, + "tokens_seen": 567214080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006691755265797392, + "loss": 1.6755, + "theoretical_loss": 3.8910207462018236, + "tokens_seen": 567279616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006691594784353059, + "loss": 1.5557, + "theoretical_loss": 3.890973727550511, + "tokens_seen": 567345152 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006691434302908726, + "loss": 1.5028, + "theoretical_loss": 3.890926715850733, + "tokens_seen": 567410688 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006691273821464393, + "loss": 1.6833, + "theoretical_loss": 3.890879711100657, + "tokens_seen": 567476224 + }, + { + "epoch": 0.17, + "learning_rate": 0.000669111334002006, + "loss": 1.5239, + "theoretical_loss": 3.8908327132984546, + "tokens_seen": 567541760 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006690952858575727, + "loss": 1.4625, + "theoretical_loss": 3.890785722442296, + "tokens_seen": 567607296 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006690792377131394, + "loss": 1.483, + "theoretical_loss": 3.890738738530353, + "tokens_seen": 567672832 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006690631895687061, + "loss": 1.54, + "theoretical_loss": 3.890691761560798, + "tokens_seen": 567738368 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006690471414242729, + "loss": 1.5534, + "theoretical_loss": 3.8906447915318036, + "tokens_seen": 567803904 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006690310932798395, + "loss": 1.3733, + "theoretical_loss": 3.8905978284415443, + "tokens_seen": 567869440 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006690150451354063, + "loss": 1.5991, + "theoretical_loss": 3.890550872288194, + "tokens_seen": 567934976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006689989969909729, + "loss": 1.5167, + "theoretical_loss": 3.890503923069927, + "tokens_seen": 568000512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006689829488465397, + "loss": 1.6034, + "theoretical_loss": 3.8904569807849203, + "tokens_seen": 568066048 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006689669007021063, + "loss": 1.5363, + "theoretical_loss": 3.8904100454313495, + "tokens_seen": 568131584 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006689508525576731, + "loss": 1.5979, + "theoretical_loss": 3.890363117007392, + "tokens_seen": 568197120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006689348044132397, + "loss": 1.5063, + "theoretical_loss": 3.890316195511226, + "tokens_seen": 568262656 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006689187562688065, + "loss": 1.5246, + "theoretical_loss": 3.8902692809410295, + "tokens_seen": 568328192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006689027081243731, + "loss": 1.5305, + "theoretical_loss": 3.8902223732949817, + "tokens_seen": 568393728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006688866599799399, + "loss": 1.6034, + "theoretical_loss": 3.8901754725712627, + "tokens_seen": 568459264 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006688706118355065, + "loss": 1.5323, + "theoretical_loss": 3.890128578768053, + "tokens_seen": 568524800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006688545636910733, + "loss": 1.5263, + "theoretical_loss": 3.8900816918835335, + "tokens_seen": 568590336 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006688385155466399, + "loss": 1.5111, + "theoretical_loss": 3.8900348119158865, + "tokens_seen": 568655872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006688224674022067, + "loss": 1.6168, + "theoretical_loss": 3.8899879388632947, + "tokens_seen": 568721408 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006688064192577733, + "loss": 1.5787, + "theoretical_loss": 3.889941072723942, + "tokens_seen": 568786944 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006687903711133401, + "loss": 1.535, + "theoretical_loss": 3.8898942134960106, + "tokens_seen": 568852480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006687743229689067, + "loss": 1.5645, + "theoretical_loss": 3.8898473611776865, + "tokens_seen": 568918016 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006687582748244735, + "loss": 1.461, + "theoretical_loss": 3.8898005157671554, + "tokens_seen": 568983552 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006687422266800401, + "loss": 1.4303, + "theoretical_loss": 3.889753677262603, + "tokens_seen": 569049088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006687261785356069, + "loss": 1.6166, + "theoretical_loss": 3.8897068456622153, + "tokens_seen": 569114624 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006687101303911735, + "loss": 1.4902, + "theoretical_loss": 3.8896600209641807, + "tokens_seen": 569180160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006686940822467403, + "loss": 1.5782, + "theoretical_loss": 3.889613203166687, + "tokens_seen": 569245696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006686780341023069, + "loss": 1.6321, + "theoretical_loss": 3.8895663922679224, + "tokens_seen": 569311232 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006686619859578737, + "loss": 1.495, + "theoretical_loss": 3.8895195882660776, + "tokens_seen": 569376768 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006686459378134403, + "loss": 1.5263, + "theoretical_loss": 3.8894727911593416, + "tokens_seen": 569442304 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006686298896690071, + "loss": 1.6645, + "theoretical_loss": 3.889426000945906, + "tokens_seen": 569507840 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006686138415245737, + "loss": 1.5007, + "theoretical_loss": 3.889379217623962, + "tokens_seen": 569573376 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006685977933801405, + "loss": 1.427, + "theoretical_loss": 3.8893324411917014, + "tokens_seen": 569638912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006685817452357071, + "loss": 1.5753, + "theoretical_loss": 3.8892856716473183, + "tokens_seen": 569704448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006685656970912739, + "loss": 1.6496, + "theoretical_loss": 3.8892389089890056, + "tokens_seen": 569769984 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006685496489468405, + "loss": 1.5246, + "theoretical_loss": 3.8891921532149576, + "tokens_seen": 569835520 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006685336008024073, + "loss": 1.5173, + "theoretical_loss": 3.8891454043233686, + "tokens_seen": 569901056 + }, + { + "epoch": 0.17, + "learning_rate": 0.000668517552657974, + "loss": 1.4937, + "theoretical_loss": 3.8890986623124353, + "tokens_seen": 569966592 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006685015045135407, + "loss": 1.6465, + "theoretical_loss": 3.889051927180353, + "tokens_seen": 570032128 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006684854563691073, + "loss": 1.451, + "theoretical_loss": 3.88900519892532, + "tokens_seen": 570097664 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.9926186800003052, + "objective/train/avg_token_score": 0.007381328381597996, + "objective/train/docs_used": 213119, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.451456069946289, + "objective/train/original_loss": 1.460935354232788, + "objective/train/theoretical_loss": 3.8889584775455326, + "objective/train/tokens_used": 590623200, + "objective/train/unlikelihood_loss": 0.02144254371523857, + "theoretical_loss": 3.8889584775455326, + "tokens_seen": 570163200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006684694082246741, + "loss": 1.5468, + "theoretical_loss": 3.8889584775455326, + "tokens_seen": 570163200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006684533600802407, + "loss": 1.4847, + "theoretical_loss": 3.8889117630391894, + "tokens_seen": 570228736 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006684373119358075, + "loss": 1.4613, + "theoretical_loss": 3.888865055404491, + "tokens_seen": 570294272 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006684212637913742, + "loss": 1.4374, + "theoretical_loss": 3.8888183546396347, + "tokens_seen": 570359808 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006684052156469409, + "loss": 1.5231, + "theoretical_loss": 3.8887716607428224, + "tokens_seen": 570425344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006683891675025076, + "loss": 1.4426, + "theoretical_loss": 3.8887249737122547, + "tokens_seen": 570490880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006683731193580743, + "loss": 1.4465, + "theoretical_loss": 3.8886782935461333, + "tokens_seen": 570556416 + }, + { + "epoch": 0.17, + "learning_rate": 0.000668357071213641, + "loss": 1.5959, + "theoretical_loss": 3.888631620242661, + "tokens_seen": 570621952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006683410230692077, + "loss": 1.5596, + "theoretical_loss": 3.8885849538000397, + "tokens_seen": 570687488 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006683249749247744, + "loss": 1.5335, + "theoretical_loss": 3.8885382942164752, + "tokens_seen": 570753024 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006683089267803411, + "loss": 1.447, + "theoretical_loss": 3.8884916414901705, + "tokens_seen": 570818560 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006682928786359078, + "loss": 1.5533, + "theoretical_loss": 3.8884449956193308, + "tokens_seen": 570884096 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006682768304914745, + "loss": 1.6773, + "theoretical_loss": 3.8883983566021625, + "tokens_seen": 570949632 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006682607823470412, + "loss": 1.4164, + "theoretical_loss": 3.8883517244368715, + "tokens_seen": 571015168 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006682447342026079, + "loss": 1.5639, + "theoretical_loss": 3.8883050991216654, + "tokens_seen": 571080704 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006682286860581746, + "loss": 1.5112, + "theoretical_loss": 3.888258480654751, + "tokens_seen": 571146240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006682126379137413, + "loss": 1.4381, + "theoretical_loss": 3.888211869034339, + "tokens_seen": 571211776 + }, + { + "epoch": 0.17, + "learning_rate": 0.000668196589769308, + "loss": 1.5807, + "theoretical_loss": 3.888165264258636, + "tokens_seen": 571277312 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006681805416248747, + "loss": 1.6071, + "theoretical_loss": 3.888118666325853, + "tokens_seen": 571342848 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006681644934804414, + "loss": 1.5421, + "theoretical_loss": 3.888072075234201, + "tokens_seen": 571408384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006681484453360081, + "loss": 1.4502, + "theoretical_loss": 3.8880254909818905, + "tokens_seen": 571473920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006681323971915748, + "loss": 1.683, + "theoretical_loss": 3.8879789135671334, + "tokens_seen": 571539456 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006681163490471415, + "loss": 1.4951, + "theoretical_loss": 3.8879323429881425, + "tokens_seen": 571604992 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006681003009027082, + "loss": 1.5195, + "theoretical_loss": 3.887885779243131, + "tokens_seen": 571670528 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006680842527582749, + "loss": 1.5734, + "theoretical_loss": 3.887839222330313, + "tokens_seen": 571736064 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006680682046138416, + "loss": 1.6403, + "theoretical_loss": 3.887792672247902, + "tokens_seen": 571801600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006680521564694082, + "loss": 1.4874, + "theoretical_loss": 3.887746128994114, + "tokens_seen": 571867136 + }, + { + "epoch": 0.17, + "learning_rate": 0.000668036108324975, + "loss": 1.6664, + "theoretical_loss": 3.8876995925671656, + "tokens_seen": 571932672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006680200601805416, + "loss": 1.435, + "theoretical_loss": 3.8876530629652724, + "tokens_seen": 571998208 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006680040120361084, + "loss": 1.5875, + "theoretical_loss": 3.8876065401866513, + "tokens_seen": 572063744 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667987963891675, + "loss": 1.5525, + "theoretical_loss": 3.887560024229521, + "tokens_seen": 572129280 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006679719157472418, + "loss": 1.52, + "theoretical_loss": 3.8875135150921, + "tokens_seen": 572194816 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006679558676028084, + "loss": 1.5972, + "theoretical_loss": 3.887467012772607, + "tokens_seen": 572260352 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006679398194583752, + "loss": 1.6497, + "theoretical_loss": 3.8874205172692626, + "tokens_seen": 572325888 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006679237713139418, + "loss": 1.506, + "theoretical_loss": 3.887374028580287, + "tokens_seen": 572391424 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006679077231695085, + "loss": 1.6645, + "theoretical_loss": 3.8873275467039017, + "tokens_seen": 572456960 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006678916750250752, + "loss": 1.4957, + "theoretical_loss": 3.887281071638328, + "tokens_seen": 572522496 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006678756268806419, + "loss": 1.5003, + "theoretical_loss": 3.8872346033817893, + "tokens_seen": 572588032 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006678595787362086, + "loss": 1.6354, + "theoretical_loss": 3.887188141932508, + "tokens_seen": 572653568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006678435305917754, + "loss": 1.5767, + "theoretical_loss": 3.8871416872887083, + "tokens_seen": 572719104 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667827482447342, + "loss": 1.5471, + "theoretical_loss": 3.8870952394486156, + "tokens_seen": 572784640 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006678114343029088, + "loss": 1.4275, + "theoretical_loss": 3.8870487984104543, + "tokens_seen": 572850176 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006677953861584754, + "loss": 1.3713, + "theoretical_loss": 3.8870023641724503, + "tokens_seen": 572915712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006677793380140422, + "loss": 1.6037, + "theoretical_loss": 3.886955936732831, + "tokens_seen": 572981248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006677632898696088, + "loss": 1.4835, + "theoretical_loss": 3.886909516089822, + "tokens_seen": 573046784 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006677472417251756, + "loss": 1.6649, + "theoretical_loss": 3.8868631022416533, + "tokens_seen": 573112320 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006677311935807422, + "loss": 1.609, + "theoretical_loss": 3.886816695186552, + "tokens_seen": 573177856 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667715145436309, + "loss": 1.5226, + "theoretical_loss": 3.886770294922748, + "tokens_seen": 573243392 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006676990972918756, + "loss": 1.4492, + "theoretical_loss": 3.8867239014484705, + "tokens_seen": 573308928 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006676830491474424, + "loss": 1.6439, + "theoretical_loss": 3.886677514761951, + "tokens_seen": 573374464 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.9906141757965088, + "objective/train/avg_token_score": 0.0093858502805233, + "objective/train/docs_used": 214228, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5532665252685547, + "objective/train/original_loss": 1.568472146987915, + "objective/train/theoretical_loss": 3.8866311348614198, + "objective/train/tokens_used": 593900000, + "objective/train/unlikelihood_loss": 0.021340688690543175, + "theoretical_loss": 3.8866311348614198, + "tokens_seen": 573440000 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667667001003009, + "loss": 1.4917, + "theoretical_loss": 3.8866311348614198, + "tokens_seen": 573440000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006676509528585758, + "loss": 1.4355, + "theoretical_loss": 3.8865847617451097, + "tokens_seen": 573505536 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006676349047141424, + "loss": 1.5765, + "theoretical_loss": 3.8865383954112525, + "tokens_seen": 573571072 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006676188565697092, + "loss": 1.6258, + "theoretical_loss": 3.8864920358580823, + "tokens_seen": 573636608 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006676028084252758, + "loss": 1.5032, + "theoretical_loss": 3.8864456830838314, + "tokens_seen": 573702144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006675867602808426, + "loss": 1.567, + "theoretical_loss": 3.8863993370867362, + "tokens_seen": 573767680 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006675707121364092, + "loss": 1.4544, + "theoretical_loss": 3.886352997865031, + "tokens_seen": 573833216 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667554663991976, + "loss": 1.4452, + "theoretical_loss": 3.8863066654169516, + "tokens_seen": 573898752 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006675386158475426, + "loss": 1.5854, + "theoretical_loss": 3.886260339740735, + "tokens_seen": 573964288 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006675225677031094, + "loss": 1.6137, + "theoretical_loss": 3.8862140208346174, + "tokens_seen": 574029824 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667506519558676, + "loss": 1.5309, + "theoretical_loss": 3.8861677086968376, + "tokens_seen": 574095360 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006674904714142428, + "loss": 1.4617, + "theoretical_loss": 3.8861214033256344, + "tokens_seen": 574160896 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006674744232698094, + "loss": 1.5364, + "theoretical_loss": 3.8860751047192457, + "tokens_seen": 574226432 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006674583751253762, + "loss": 1.613, + "theoretical_loss": 3.886028812875912, + "tokens_seen": 574291968 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006674423269809428, + "loss": 1.4854, + "theoretical_loss": 3.885982527793874, + "tokens_seen": 574357504 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006674262788365096, + "loss": 1.5933, + "theoretical_loss": 3.8859362494713725, + "tokens_seen": 574423040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006674102306920762, + "loss": 1.5955, + "theoretical_loss": 3.8858899779066496, + "tokens_seen": 574488576 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667394182547643, + "loss": 1.5188, + "theoretical_loss": 3.8858437130979473, + "tokens_seen": 574554112 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006673781344032096, + "loss": 1.6526, + "theoretical_loss": 3.8857974550435097, + "tokens_seen": 574619648 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006673620862587764, + "loss": 1.5673, + "theoretical_loss": 3.885751203741579, + "tokens_seen": 574685184 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667346038114343, + "loss": 1.5144, + "theoretical_loss": 3.8857049591904014, + "tokens_seen": 574750720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006673299899699098, + "loss": 1.3797, + "theoretical_loss": 3.8856587213882205, + "tokens_seen": 574816256 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006673139418254764, + "loss": 1.4558, + "theoretical_loss": 3.8856124903332825, + "tokens_seen": 574881792 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006672978936810432, + "loss": 1.6314, + "theoretical_loss": 3.885566266023834, + "tokens_seen": 574947328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006672818455366098, + "loss": 1.5797, + "theoretical_loss": 3.8855200484581225, + "tokens_seen": 575012864 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006672657973921766, + "loss": 1.5132, + "theoretical_loss": 3.885473837634395, + "tokens_seen": 575078400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006672497492477432, + "loss": 1.4555, + "theoretical_loss": 3.8854276335508997, + "tokens_seen": 575143936 + }, + { + "epoch": 0.17, + "learning_rate": 0.00066723370110331, + "loss": 1.5382, + "theoretical_loss": 3.885381436205886, + "tokens_seen": 575209472 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006672176529588766, + "loss": 1.5639, + "theoretical_loss": 3.885335245597604, + "tokens_seen": 575275008 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006672016048144434, + "loss": 1.5532, + "theoretical_loss": 3.885289061724303, + "tokens_seen": 575340544 + }, + { + "epoch": 0.17, + "learning_rate": 0.00066718555667001, + "loss": 1.5765, + "theoretical_loss": 3.885242884584235, + "tokens_seen": 575406080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006671695085255768, + "loss": 1.4623, + "theoretical_loss": 3.8851967141756507, + "tokens_seen": 575471616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006671534603811434, + "loss": 1.3994, + "theoretical_loss": 3.885150550496803, + "tokens_seen": 575537152 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006671374122367102, + "loss": 1.5404, + "theoretical_loss": 3.8851043935459453, + "tokens_seen": 575602688 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006671213640922768, + "loss": 1.6166, + "theoretical_loss": 3.88505824332133, + "tokens_seen": 575668224 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006671053159478436, + "loss": 1.5273, + "theoretical_loss": 3.8850120998212123, + "tokens_seen": 575733760 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006670892678034102, + "loss": 1.6934, + "theoretical_loss": 3.884965963043847, + "tokens_seen": 575799296 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667073219658977, + "loss": 1.4788, + "theoretical_loss": 3.884919832987489, + "tokens_seen": 575864832 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006670571715145436, + "loss": 1.526, + "theoretical_loss": 3.884873709650395, + "tokens_seen": 575930368 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006670411233701104, + "loss": 1.4833, + "theoretical_loss": 3.8848275930308223, + "tokens_seen": 575995904 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667025075225677, + "loss": 1.3512, + "theoretical_loss": 3.884781483127027, + "tokens_seen": 576061440 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006670090270812438, + "loss": 1.4389, + "theoretical_loss": 3.8847353799372684, + "tokens_seen": 576126976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006669929789368105, + "loss": 1.5784, + "theoretical_loss": 3.8846892834598052, + "tokens_seen": 576192512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006669769307923772, + "loss": 1.5662, + "theoretical_loss": 3.884643193692896, + "tokens_seen": 576258048 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006669608826479439, + "loss": 1.5592, + "theoretical_loss": 3.884597110634802, + "tokens_seen": 576323584 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006669448345035106, + "loss": 1.4489, + "theoretical_loss": 3.884551034283784, + "tokens_seen": 576389120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006669287863590773, + "loss": 1.4333, + "theoretical_loss": 3.8845049646381025, + "tokens_seen": 576454656 + }, + { + "epoch": 0.17, + "learning_rate": 0.000666912738214644, + "loss": 1.5144, + "theoretical_loss": 3.8844589016960196, + "tokens_seen": 576520192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006668966900702107, + "loss": 1.4097, + "theoretical_loss": 3.884412845455799, + "tokens_seen": 576585728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006668806419257774, + "loss": 1.4908, + "theoretical_loss": 3.8843667959157027, + "tokens_seen": 576651264 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.9917870163917542, + "objective/train/avg_token_score": 0.008212994784116745, + "objective/train/docs_used": 215374, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.571872353553772, + "objective/train/original_loss": 1.5862627029418945, + "objective/train/theoretical_loss": 3.884320753073996, + "objective/train/tokens_used": 597176800, + "objective/train/unlikelihood_loss": 0.017573470249772072, + "theoretical_loss": 3.884320753073996, + "tokens_seen": 576716800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006668645937813441, + "loss": 1.5215, + "theoretical_loss": 3.884320753073996, + "tokens_seen": 576716800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006668485456369108, + "loss": 1.5623, + "theoretical_loss": 3.884274716928942, + "tokens_seen": 576782336 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006668324974924775, + "loss": 1.4848, + "theoretical_loss": 3.8842286874788075, + "tokens_seen": 576847872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006668164493480442, + "loss": 1.5834, + "theoretical_loss": 3.8841826647218576, + "tokens_seen": 576913408 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006668004012036109, + "loss": 1.5494, + "theoretical_loss": 3.8841366486563595, + "tokens_seen": 576978944 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006668004012036109, + "loss": 1.5511, + "theoretical_loss": 3.8840906392805796, + "tokens_seen": 577044480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006667843530591776, + "loss": 1.597, + "theoretical_loss": 3.8840446365927863, + "tokens_seen": 577110016 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006667683049147443, + "loss": 1.5844, + "theoretical_loss": 3.8839986405912477, + "tokens_seen": 577175552 + }, + { + "epoch": 0.17, + "learning_rate": 0.000666752256770311, + "loss": 1.698, + "theoretical_loss": 3.8839526512742335, + "tokens_seen": 577241088 + }, + { + "epoch": 0.17, + "learning_rate": 0.000666752256770311, + "loss": 1.7023, + "theoretical_loss": 3.883906668640013, + "tokens_seen": 577306624 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006667362086258777, + "loss": 1.9737, + "theoretical_loss": 3.8838606926868575, + "tokens_seen": 577372160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006667201604814444, + "loss": 1.5303, + "theoretical_loss": 3.8838147234130367, + "tokens_seen": 577437696 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006667041123370111, + "loss": 1.5309, + "theoretical_loss": 3.8837687608168237, + "tokens_seen": 577503232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006666880641925778, + "loss": 1.5966, + "theoretical_loss": 3.8837228048964896, + "tokens_seen": 577568768 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006666720160481445, + "loss": 1.8343, + "theoretical_loss": 3.8836768556503083, + "tokens_seen": 577634304 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006666559679037111, + "loss": 1.6539, + "theoretical_loss": 3.883630913076553, + "tokens_seen": 577699840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006666399197592779, + "loss": 1.6964, + "theoretical_loss": 3.883584977173499, + "tokens_seen": 577765376 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006666238716148445, + "loss": 1.6737, + "theoretical_loss": 3.8835390479394203, + "tokens_seen": 577830912 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006666078234704113, + "loss": 1.5155, + "theoretical_loss": 3.883493125372592, + "tokens_seen": 577896448 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006665917753259779, + "loss": 1.6441, + "theoretical_loss": 3.8834472094712913, + "tokens_seen": 577961984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006665757271815447, + "loss": 1.7023, + "theoretical_loss": 3.8834013002337953, + "tokens_seen": 578027520 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006665596790371113, + "loss": 1.6047, + "theoretical_loss": 3.883355397658381, + "tokens_seen": 578093056 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006665436308926781, + "loss": 1.6443, + "theoretical_loss": 3.8833095017433257, + "tokens_seen": 578158592 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006665275827482447, + "loss": 1.5856, + "theoretical_loss": 3.8832636124869095, + "tokens_seen": 578224128 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006665115346038115, + "loss": 1.6112, + "theoretical_loss": 3.883217729887411, + "tokens_seen": 578289664 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006664954864593781, + "loss": 1.6259, + "theoretical_loss": 3.883171853943111, + "tokens_seen": 578355200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006664794383149449, + "loss": 1.7352, + "theoretical_loss": 3.8831259846522896, + "tokens_seen": 578420736 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006664633901705115, + "loss": 1.5014, + "theoretical_loss": 3.8830801220132285, + "tokens_seen": 578486272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006664473420260783, + "loss": 1.6052, + "theoretical_loss": 3.883034266024209, + "tokens_seen": 578551808 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006664312938816449, + "loss": 1.5009, + "theoretical_loss": 3.8829884166835154, + "tokens_seen": 578617344 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006664152457372117, + "loss": 1.6144, + "theoretical_loss": 3.882942573989429, + "tokens_seen": 578682880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006663991975927783, + "loss": 1.6263, + "theoretical_loss": 3.882896737940235, + "tokens_seen": 578748416 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006663831494483451, + "loss": 1.5221, + "theoretical_loss": 3.882850908534217, + "tokens_seen": 578813952 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006663671013039117, + "loss": 1.6571, + "theoretical_loss": 3.882805085769661, + "tokens_seen": 578879488 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006663510531594785, + "loss": 1.6813, + "theoretical_loss": 3.8827592696448523, + "tokens_seen": 578945024 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006663350050150451, + "loss": 1.6023, + "theoretical_loss": 3.8827134601580777, + "tokens_seen": 579010560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006663189568706119, + "loss": 1.5495, + "theoretical_loss": 3.8826676573076235, + "tokens_seen": 579076096 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006663029087261785, + "loss": 1.5976, + "theoretical_loss": 3.8826218610917786, + "tokens_seen": 579141632 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006662868605817453, + "loss": 1.6406, + "theoretical_loss": 3.8825760715088307, + "tokens_seen": 579207168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006662708124373119, + "loss": 1.5135, + "theoretical_loss": 3.882530288557069, + "tokens_seen": 579272704 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006662547642928787, + "loss": 1.6107, + "theoretical_loss": 3.8824845122347824, + "tokens_seen": 579338240 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006662387161484453, + "loss": 1.5605, + "theoretical_loss": 3.8824387425402618, + "tokens_seen": 579403776 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006662226680040121, + "loss": 1.539, + "theoretical_loss": 3.8823929794717986, + "tokens_seen": 579469312 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006662066198595787, + "loss": 1.692, + "theoretical_loss": 3.882347223027683, + "tokens_seen": 579534848 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006661905717151455, + "loss": 1.5668, + "theoretical_loss": 3.882301473206209, + "tokens_seen": 579600384 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006661745235707121, + "loss": 1.6092, + "theoretical_loss": 3.8822557300056673, + "tokens_seen": 579665920 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006661584754262789, + "loss": 1.5508, + "theoretical_loss": 3.882209993424353, + "tokens_seen": 579731456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006661424272818455, + "loss": 1.5561, + "theoretical_loss": 3.8821642634605587, + "tokens_seen": 579796992 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006661263791374123, + "loss": 1.6266, + "theoretical_loss": 3.8821185401125806, + "tokens_seen": 579862528 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006661103309929789, + "loss": 1.5354, + "theoretical_loss": 3.8820728233787136, + "tokens_seen": 579928064 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.9898017644882202, + "objective/train/avg_token_score": 0.010198263451457024, + "objective/train/docs_used": 216668, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6812628507614136, + "objective/train/original_loss": 1.700021743774414, + "objective/train/theoretical_loss": 3.8820271132572524, + "objective/train/tokens_used": 600453600, + "objective/train/unlikelihood_loss": 0.0202188640832901, + "theoretical_loss": 3.8820271132572524, + "tokens_seen": 579993600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006660942828485457, + "loss": 1.6258, + "theoretical_loss": 3.8820271132572524, + "tokens_seen": 579993600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006660782347041123, + "loss": 1.6538, + "theoretical_loss": 3.881981409746495, + "tokens_seen": 580059136 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006660621865596791, + "loss": 1.5573, + "theoretical_loss": 3.8819357128447383, + "tokens_seen": 580124672 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006660461384152457, + "loss": 1.5217, + "theoretical_loss": 3.88189002255028, + "tokens_seen": 580190208 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006660300902708125, + "loss": 1.5769, + "theoretical_loss": 3.8818443388614186, + "tokens_seen": 580255744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006660140421263791, + "loss": 1.5658, + "theoretical_loss": 3.881798661776453, + "tokens_seen": 580321280 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006659979939819459, + "loss": 1.6139, + "theoretical_loss": 3.8817529912936832, + "tokens_seen": 580386816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006659819458375125, + "loss": 1.5664, + "theoretical_loss": 3.88170732741141, + "tokens_seen": 580452352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006659658976930793, + "loss": 1.6289, + "theoretical_loss": 3.8816616701279334, + "tokens_seen": 580517888 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006659498495486459, + "loss": 1.5786, + "theoretical_loss": 3.881616019441556, + "tokens_seen": 580583424 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006659338014042127, + "loss": 1.4814, + "theoretical_loss": 3.881570375350579, + "tokens_seen": 580648960 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006659177532597793, + "loss": 1.6148, + "theoretical_loss": 3.881524737853306, + "tokens_seen": 580714496 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006659017051153461, + "loss": 1.6198, + "theoretical_loss": 3.881479106948041, + "tokens_seen": 580780032 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006658856569709127, + "loss": 1.4799, + "theoretical_loss": 3.8814334826330876, + "tokens_seen": 580845568 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006658696088264795, + "loss": 1.6084, + "theoretical_loss": 3.88138786490675, + "tokens_seen": 580911104 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006658535606820461, + "loss": 1.5491, + "theoretical_loss": 3.8813422537673343, + "tokens_seen": 580976640 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006658375125376129, + "loss": 1.5713, + "theoretical_loss": 3.881296649213146, + "tokens_seen": 581042176 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006658214643931795, + "loss": 1.5313, + "theoretical_loss": 3.8812510512424927, + "tokens_seen": 581107712 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006658054162487463, + "loss": 1.7103, + "theoretical_loss": 3.881205459853681, + "tokens_seen": 581173248 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006657893681043129, + "loss": 1.548, + "theoretical_loss": 3.8811598750450185, + "tokens_seen": 581238784 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006657733199598797, + "loss": 1.5942, + "theoretical_loss": 3.8811142968148142, + "tokens_seen": 581304320 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006657572718154463, + "loss": 1.5031, + "theoretical_loss": 3.881068725161377, + "tokens_seen": 581369856 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006657412236710131, + "loss": 1.5145, + "theoretical_loss": 3.881023160083018, + "tokens_seen": 581435392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006657251755265797, + "loss": 1.5941, + "theoretical_loss": 3.8809776015780453, + "tokens_seen": 581500928 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006657091273821465, + "loss": 1.5112, + "theoretical_loss": 3.880932049644771, + "tokens_seen": 581566464 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006656930792377131, + "loss": 1.5437, + "theoretical_loss": 3.8808865042815075, + "tokens_seen": 581632000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006656770310932799, + "loss": 1.5106, + "theoretical_loss": 3.880840965486566, + "tokens_seen": 581697536 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006656609829488466, + "loss": 1.4965, + "theoretical_loss": 3.8807954332582595, + "tokens_seen": 581763072 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006656449348044133, + "loss": 1.6261, + "theoretical_loss": 3.880749907594902, + "tokens_seen": 581828608 + }, + { + "epoch": 0.18, + "learning_rate": 0.00066562888665998, + "loss": 1.5846, + "theoretical_loss": 3.8807043884948085, + "tokens_seen": 581894144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006656128385155467, + "loss": 1.5823, + "theoretical_loss": 3.880658875956292, + "tokens_seen": 581959680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006655967903711134, + "loss": 1.525, + "theoretical_loss": 3.8806133699776684, + "tokens_seen": 582025216 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006655807422266801, + "loss": 1.5444, + "theoretical_loss": 3.880567870557254, + "tokens_seen": 582090752 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006655646940822468, + "loss": 1.6781, + "theoretical_loss": 3.8805223776933655, + "tokens_seen": 582156288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006655486459378135, + "loss": 1.7087, + "theoretical_loss": 3.88047689138432, + "tokens_seen": 582221824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006655325977933802, + "loss": 1.5042, + "theoretical_loss": 3.880431411628436, + "tokens_seen": 582287360 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006655165496489469, + "loss": 1.5984, + "theoretical_loss": 3.8803859384240305, + "tokens_seen": 582352896 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006655005015045136, + "loss": 1.5009, + "theoretical_loss": 3.880340471769424, + "tokens_seen": 582418432 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006654844533600803, + "loss": 1.6003, + "theoretical_loss": 3.880295011662936, + "tokens_seen": 582483968 + }, + { + "epoch": 0.18, + "learning_rate": 0.000665468405215647, + "loss": 1.5248, + "theoretical_loss": 3.8802495581028857, + "tokens_seen": 582549504 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006654523570712137, + "loss": 1.7301, + "theoretical_loss": 3.880204111087596, + "tokens_seen": 582615040 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006654363089267805, + "loss": 1.545, + "theoretical_loss": 3.880158670615387, + "tokens_seen": 582680576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006654202607823471, + "loss": 1.5239, + "theoretical_loss": 3.880113236684582, + "tokens_seen": 582746112 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006654042126379139, + "loss": 1.6238, + "theoretical_loss": 3.880067809293503, + "tokens_seen": 582811648 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006653881644934805, + "loss": 1.6616, + "theoretical_loss": 3.880022388440474, + "tokens_seen": 582877184 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006653721163490472, + "loss": 1.5646, + "theoretical_loss": 3.879976974123818, + "tokens_seen": 582942720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006653560682046139, + "loss": 1.5881, + "theoretical_loss": 3.8799315663418614, + "tokens_seen": 583008256 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006653400200601806, + "loss": 1.4273, + "theoretical_loss": 3.8798861650929277, + "tokens_seen": 583073792 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006653239719157473, + "loss": 1.5884, + "theoretical_loss": 3.8798407703753446, + "tokens_seen": 583139328 + }, + { + "epoch": 0.18, + "learning_rate": 0.000665307923771314, + "loss": 1.5888, + "theoretical_loss": 3.879795382187438, + "tokens_seen": 583204864 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.9888534545898438, + "objective/train/avg_token_score": 0.011146516539156437, + "objective/train/docs_used": 217803, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6140764951705933, + "objective/train/original_loss": 1.6316395998001099, + "objective/train/theoretical_loss": 3.879750000527534, + "objective/train/tokens_used": 603730400, + "objective/train/unlikelihood_loss": 0.025212906301021576, + "theoretical_loss": 3.879750000527534, + "tokens_seen": 583270400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006652918756268807, + "loss": 1.5881, + "theoretical_loss": 3.879750000527534, + "tokens_seen": 583270400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006652758274824474, + "loss": 1.5524, + "theoretical_loss": 3.8797046253939618, + "tokens_seen": 583335936 + }, + { + "epoch": 0.18, + "learning_rate": 0.000665259779338014, + "loss": 1.5973, + "theoretical_loss": 3.8796592567850485, + "tokens_seen": 583401472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006652437311935808, + "loss": 1.5576, + "theoretical_loss": 3.8796138946991245, + "tokens_seen": 583467008 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006652276830491474, + "loss": 1.6658, + "theoretical_loss": 3.8795685391345183, + "tokens_seen": 583532544 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006652116349047142, + "loss": 1.5541, + "theoretical_loss": 3.879523190089561, + "tokens_seen": 583598080 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006651955867602808, + "loss": 1.5464, + "theoretical_loss": 3.8794778475625833, + "tokens_seen": 583663616 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006651795386158476, + "loss": 1.6326, + "theoretical_loss": 3.879432511551916, + "tokens_seen": 583729152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006651634904714142, + "loss": 1.4855, + "theoretical_loss": 3.8793871820558916, + "tokens_seen": 583794688 + }, + { + "epoch": 0.18, + "learning_rate": 0.000665147442326981, + "loss": 1.5997, + "theoretical_loss": 3.8793418590728423, + "tokens_seen": 583860224 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006651313941825476, + "loss": 1.534, + "theoretical_loss": 3.879296542601103, + "tokens_seen": 583925760 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006651153460381144, + "loss": 1.4899, + "theoretical_loss": 3.879251232639006, + "tokens_seen": 583991296 + }, + { + "epoch": 0.18, + "learning_rate": 0.000665099297893681, + "loss": 1.5832, + "theoretical_loss": 3.879205929184886, + "tokens_seen": 584056832 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006650832497492478, + "loss": 1.486, + "theoretical_loss": 3.8791606322370793, + "tokens_seen": 584122368 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006650672016048144, + "loss": 1.5049, + "theoretical_loss": 3.8791153417939204, + "tokens_seen": 584187904 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006650511534603812, + "loss": 1.4931, + "theoretical_loss": 3.8790700578537463, + "tokens_seen": 584253440 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006650351053159478, + "loss": 1.6648, + "theoretical_loss": 3.879024780414894, + "tokens_seen": 584318976 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006650190571715146, + "loss": 1.5607, + "theoretical_loss": 3.878979509475701, + "tokens_seen": 584384512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006650030090270812, + "loss": 1.574, + "theoretical_loss": 3.8789342450345052, + "tokens_seen": 584450048 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664986960882648, + "loss": 1.5349, + "theoretical_loss": 3.8788889870896455, + "tokens_seen": 584515584 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006649709127382146, + "loss": 1.5121, + "theoretical_loss": 3.878843735639462, + "tokens_seen": 584581120 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006649548645937814, + "loss": 1.4855, + "theoretical_loss": 3.8787984906822945, + "tokens_seen": 584646656 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664938816449348, + "loss": 1.5382, + "theoretical_loss": 3.878753252216483, + "tokens_seen": 584712192 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006649227683049148, + "loss": 1.6243, + "theoretical_loss": 3.8787080202403694, + "tokens_seen": 584777728 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006649067201604814, + "loss": 1.5226, + "theoretical_loss": 3.8786627947522954, + "tokens_seen": 584843264 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006648906720160482, + "loss": 1.5865, + "theoretical_loss": 3.878617575750603, + "tokens_seen": 584908800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006648746238716148, + "loss": 1.5725, + "theoretical_loss": 3.8785723632336366, + "tokens_seen": 584974336 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006648585757271816, + "loss": 1.4587, + "theoretical_loss": 3.8785271571997386, + "tokens_seen": 585039872 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006648425275827482, + "loss": 1.6366, + "theoretical_loss": 3.878481957647254, + "tokens_seen": 585105408 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664826479438315, + "loss": 1.4982, + "theoretical_loss": 3.878436764574527, + "tokens_seen": 585170944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006648104312938816, + "loss": 1.5715, + "theoretical_loss": 3.8783915779799036, + "tokens_seen": 585236480 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006647943831494484, + "loss": 1.6186, + "theoretical_loss": 3.8783463978617303, + "tokens_seen": 585302016 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664778335005015, + "loss": 1.5369, + "theoretical_loss": 3.878301224218353, + "tokens_seen": 585367552 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006647622868605818, + "loss": 1.4992, + "theoretical_loss": 3.8782560570481204, + "tokens_seen": 585433088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006647462387161484, + "loss": 1.5183, + "theoretical_loss": 3.8782108963493793, + "tokens_seen": 585498624 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006647301905717152, + "loss": 1.6142, + "theoretical_loss": 3.878165742120478, + "tokens_seen": 585564160 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006647141424272818, + "loss": 1.5154, + "theoretical_loss": 3.8781205943597667, + "tokens_seen": 585629696 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006646980942828486, + "loss": 1.5796, + "theoretical_loss": 3.8780754530655948, + "tokens_seen": 585695232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006646820461384152, + "loss": 1.4731, + "theoretical_loss": 3.8780303182363123, + "tokens_seen": 585760768 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664665997993982, + "loss": 1.5518, + "theoretical_loss": 3.877985189870271, + "tokens_seen": 585826304 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006646499498495486, + "loss": 1.5165, + "theoretical_loss": 3.877940067965821, + "tokens_seen": 585891840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006646339017051154, + "loss": 1.4347, + "theoretical_loss": 3.8778949525213164, + "tokens_seen": 585957376 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664617853560682, + "loss": 1.5093, + "theoretical_loss": 3.8778498435351088, + "tokens_seen": 586022912 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006646018054162488, + "loss": 1.6257, + "theoretical_loss": 3.8778047410055514, + "tokens_seen": 586088448 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006645857572718154, + "loss": 1.5128, + "theoretical_loss": 3.8777596449309994, + "tokens_seen": 586153984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006645697091273822, + "loss": 1.6338, + "theoretical_loss": 3.8777145553098062, + "tokens_seen": 586219520 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006645536609829488, + "loss": 1.5607, + "theoretical_loss": 3.8776694721403278, + "tokens_seen": 586285056 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006645376128385156, + "loss": 1.4991, + "theoretical_loss": 3.877624395420919, + "tokens_seen": 586350592 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006645215646940822, + "loss": 1.4994, + "theoretical_loss": 3.8775793251499375, + "tokens_seen": 586416128 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664505516549649, + "loss": 1.556, + "theoretical_loss": 3.8775342613257395, + "tokens_seen": 586481664 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.9921160936355591, + "objective/train/avg_token_score": 0.007883934304118156, + "objective/train/docs_used": 218927, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6958613395690918, + "objective/train/original_loss": 1.711035966873169, + "objective/train/theoretical_loss": 3.8774892039466833, + "objective/train/tokens_used": 607007200, + "objective/train/unlikelihood_loss": 0.015616772696375847, + "theoretical_loss": 3.8774892039466833, + "tokens_seen": 586547200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006644894684052156, + "loss": 1.6625, + "theoretical_loss": 3.8774892039466833, + "tokens_seen": 586547200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006644734202607824, + "loss": 1.5016, + "theoretical_loss": 3.877444153011126, + "tokens_seen": 586612736 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664457372116349, + "loss": 1.5004, + "theoretical_loss": 3.8773991085174284, + "tokens_seen": 586678272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006644413239719158, + "loss": 1.4851, + "theoretical_loss": 3.8773540704639475, + "tokens_seen": 586743808 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006644252758274824, + "loss": 1.5717, + "theoretical_loss": 3.8773090388490448, + "tokens_seen": 586809344 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006644092276830492, + "loss": 1.4767, + "theoretical_loss": 3.8772640136710805, + "tokens_seen": 586874880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006643931795386158, + "loss": 1.5303, + "theoretical_loss": 3.877218994928416, + "tokens_seen": 586940416 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006643771313941826, + "loss": 1.66, + "theoretical_loss": 3.8771739826194134, + "tokens_seen": 587005952 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006643610832497492, + "loss": 1.5505, + "theoretical_loss": 3.8771289767424344, + "tokens_seen": 587071488 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664345035105316, + "loss": 1.5975, + "theoretical_loss": 3.8770839772958428, + "tokens_seen": 587137024 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006643289869608826, + "loss": 1.5292, + "theoretical_loss": 3.877038984278002, + "tokens_seen": 587202560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006643129388164494, + "loss": 1.5435, + "theoretical_loss": 3.8769939976872756, + "tokens_seen": 587268096 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664296890672016, + "loss": 1.4941, + "theoretical_loss": 3.8769490175220294, + "tokens_seen": 587333632 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006642808425275828, + "loss": 1.4642, + "theoretical_loss": 3.876904043780628, + "tokens_seen": 587399168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006642647943831495, + "loss": 1.5535, + "theoretical_loss": 3.8768590764614377, + "tokens_seen": 587464704 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006642487462387162, + "loss": 1.524, + "theoretical_loss": 3.8768141155628255, + "tokens_seen": 587530240 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664232698094283, + "loss": 1.5339, + "theoretical_loss": 3.8767691610831587, + "tokens_seen": 587595776 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006642166499498496, + "loss": 1.5399, + "theoretical_loss": 3.876724213020804, + "tokens_seen": 587661312 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006642006018054164, + "loss": 1.4985, + "theoretical_loss": 3.876679271374131, + "tokens_seen": 587726848 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664184553660983, + "loss": 1.472, + "theoretical_loss": 3.8766343361415085, + "tokens_seen": 587792384 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006641685055165498, + "loss": 1.5763, + "theoretical_loss": 3.876589407321305, + "tokens_seen": 587857920 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006641524573721164, + "loss": 1.5154, + "theoretical_loss": 3.8765444849118924, + "tokens_seen": 587923456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006641364092276832, + "loss": 1.4731, + "theoretical_loss": 3.8764995689116404, + "tokens_seen": 587988992 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006641203610832498, + "loss": 1.5038, + "theoretical_loss": 3.8764546593189206, + "tokens_seen": 588054528 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006641043129388166, + "loss": 1.5661, + "theoretical_loss": 3.876409756132105, + "tokens_seen": 588120064 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006640882647943832, + "loss": 1.5428, + "theoretical_loss": 3.8763648593495663, + "tokens_seen": 588185600 + }, + { + "epoch": 0.18, + "learning_rate": 0.00066407221664995, + "loss": 1.5963, + "theoretical_loss": 3.8763199689696783, + "tokens_seen": 588251136 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006640561685055166, + "loss": 1.4821, + "theoretical_loss": 3.876275084990813, + "tokens_seen": 588316672 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006640401203610834, + "loss": 1.3815, + "theoretical_loss": 3.876230207411347, + "tokens_seen": 588382208 + }, + { + "epoch": 0.18, + "learning_rate": 0.00066402407221665, + "loss": 1.4254, + "theoretical_loss": 3.8761853362296534, + "tokens_seen": 588447744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006640080240722168, + "loss": 1.47, + "theoretical_loss": 3.8761404714441086, + "tokens_seen": 588513280 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006639919759277834, + "loss": 1.4735, + "theoretical_loss": 3.8760956130530886, + "tokens_seen": 588578816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006639759277833501, + "loss": 1.6019, + "theoretical_loss": 3.8760507610549704, + "tokens_seen": 588644352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006639598796389168, + "loss": 1.586, + "theoretical_loss": 3.876005915448131, + "tokens_seen": 588709888 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006639438314944835, + "loss": 1.4918, + "theoretical_loss": 3.8759610762309484, + "tokens_seen": 588775424 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006639277833500502, + "loss": 1.6067, + "theoretical_loss": 3.875916243401801, + "tokens_seen": 588840960 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006639117352056169, + "loss": 1.5633, + "theoretical_loss": 3.8758714169590682, + "tokens_seen": 588906496 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006638956870611836, + "loss": 1.6256, + "theoretical_loss": 3.87582659690113, + "tokens_seen": 588972032 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006638796389167503, + "loss": 1.5965, + "theoretical_loss": 3.875781783226366, + "tokens_seen": 589037568 + }, + { + "epoch": 0.18, + "learning_rate": 0.000663863590772317, + "loss": 1.5412, + "theoretical_loss": 3.8757369759331572, + "tokens_seen": 589103104 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006638475426278837, + "loss": 1.4504, + "theoretical_loss": 3.8756921750198856, + "tokens_seen": 589168640 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006638314944834503, + "loss": 1.4567, + "theoretical_loss": 3.8756473804849323, + "tokens_seen": 589234176 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006638154463390171, + "loss": 1.5402, + "theoretical_loss": 3.8756025923266813, + "tokens_seen": 589299712 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006637993981945837, + "loss": 1.5945, + "theoretical_loss": 3.8755578105435147, + "tokens_seen": 589365248 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006637833500501505, + "loss": 1.5253, + "theoretical_loss": 3.8755130351338165, + "tokens_seen": 589430784 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006637673019057171, + "loss": 1.6834, + "theoretical_loss": 3.8754682660959716, + "tokens_seen": 589496320 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006637512537612839, + "loss": 1.4944, + "theoretical_loss": 3.8754235034283653, + "tokens_seen": 589561856 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006637352056168505, + "loss": 1.4886, + "theoretical_loss": 3.875378747129382, + "tokens_seen": 589627392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006637191574724173, + "loss": 1.5502, + "theoretical_loss": 3.8753339971974086, + "tokens_seen": 589692928 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006637031093279839, + "loss": 1.5212, + "theoretical_loss": 3.8752892536308323, + "tokens_seen": 589758464 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.9909812808036804, + "objective/train/avg_token_score": 0.009018725715577602, + "objective/train/docs_used": 220165, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.592782735824585, + "objective/train/original_loss": 1.6081464290618896, + "objective/train/theoretical_loss": 3.87524451642804, + "objective/train/tokens_used": 610284000, + "objective/train/unlikelihood_loss": 0.022084740921854973, + "theoretical_loss": 3.87524451642804, + "tokens_seen": 589824000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006636870611835507, + "loss": 1.6122, + "theoretical_loss": 3.87524451642804, + "tokens_seen": 589824000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006636710130391173, + "loss": 1.4983, + "theoretical_loss": 3.8751997855874194, + "tokens_seen": 589889536 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006636549648946841, + "loss": 1.5562, + "theoretical_loss": 3.8751550611073595, + "tokens_seen": 589955072 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006636389167502507, + "loss": 1.487, + "theoretical_loss": 3.875110342986249, + "tokens_seen": 590020608 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006636228686058175, + "loss": 1.5421, + "theoretical_loss": 3.875065631222478, + "tokens_seen": 590086144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006636068204613841, + "loss": 1.581, + "theoretical_loss": 3.875020925814437, + "tokens_seen": 590151680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006635907723169509, + "loss": 1.7093, + "theoretical_loss": 3.8749762267605163, + "tokens_seen": 590217216 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006635747241725175, + "loss": 1.5507, + "theoretical_loss": 3.8749315340591073, + "tokens_seen": 590282752 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006635586760280843, + "loss": 1.5943, + "theoretical_loss": 3.874886847708603, + "tokens_seen": 590348288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006635426278836509, + "loss": 1.5212, + "theoretical_loss": 3.8748421677073956, + "tokens_seen": 590413824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006635265797392177, + "loss": 1.5227, + "theoretical_loss": 3.874797494053878, + "tokens_seen": 590479360 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006635105315947843, + "loss": 1.5126, + "theoretical_loss": 3.8747528267464437, + "tokens_seen": 590544896 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006634944834503511, + "loss": 1.332, + "theoretical_loss": 3.8747081657834874, + "tokens_seen": 590610432 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006634784353059177, + "loss": 1.559, + "theoretical_loss": 3.874663511163405, + "tokens_seen": 590675968 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006634623871614845, + "loss": 1.3771, + "theoretical_loss": 3.8746188628845912, + "tokens_seen": 590741504 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006634463390170511, + "loss": 1.526, + "theoretical_loss": 3.8745742209454423, + "tokens_seen": 590807040 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006634302908726179, + "loss": 1.488, + "theoretical_loss": 3.874529585344355, + "tokens_seen": 590872576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006634142427281845, + "loss": 1.5175, + "theoretical_loss": 3.874484956079727, + "tokens_seen": 590938112 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006633981945837513, + "loss": 1.5156, + "theoretical_loss": 3.8744403331499555, + "tokens_seen": 591003648 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006633821464393179, + "loss": 1.5551, + "theoretical_loss": 3.8743957165534395, + "tokens_seen": 591069184 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006633660982948847, + "loss": 1.6461, + "theoretical_loss": 3.8743511062885783, + "tokens_seen": 591134720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006633500501504513, + "loss": 1.4501, + "theoretical_loss": 3.874306502353771, + "tokens_seen": 591200256 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006633340020060181, + "loss": 1.5649, + "theoretical_loss": 3.8742619047474176, + "tokens_seen": 591265792 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006633179538615847, + "loss": 1.5111, + "theoretical_loss": 3.87421731346792, + "tokens_seen": 591331328 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006633019057171515, + "loss": 1.5391, + "theoretical_loss": 3.874172728513678, + "tokens_seen": 591396864 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006632858575727181, + "loss": 1.5075, + "theoretical_loss": 3.874128149883095, + "tokens_seen": 591462400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006632698094282849, + "loss": 1.5708, + "theoretical_loss": 3.8740835775745732, + "tokens_seen": 591527936 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006632537612838515, + "loss": 1.4816, + "theoretical_loss": 3.8740390115865155, + "tokens_seen": 591593472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006632377131394183, + "loss": 1.647, + "theoretical_loss": 3.873994451917326, + "tokens_seen": 591659008 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006632216649949849, + "loss": 1.3744, + "theoretical_loss": 3.8739498985654084, + "tokens_seen": 591724544 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006632056168505517, + "loss": 1.5155, + "theoretical_loss": 3.873905351529168, + "tokens_seen": 591790080 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006631895687061183, + "loss": 1.544, + "theoretical_loss": 3.8738608108070096, + "tokens_seen": 591855616 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006631735205616851, + "loss": 1.4386, + "theoretical_loss": 3.8738162763973407, + "tokens_seen": 591921152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006631574724172517, + "loss": 1.5281, + "theoretical_loss": 3.8737717482985667, + "tokens_seen": 591986688 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006631414242728185, + "loss": 1.4689, + "theoretical_loss": 3.8737272265090947, + "tokens_seen": 592052224 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006631253761283851, + "loss": 1.4655, + "theoretical_loss": 3.8736827110273335, + "tokens_seen": 592117760 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006631093279839519, + "loss": 1.5406, + "theoretical_loss": 3.8736382018516906, + "tokens_seen": 592183296 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006630932798395185, + "loss": 1.4947, + "theoretical_loss": 3.8735936989805744, + "tokens_seen": 592248832 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006630772316950853, + "loss": 1.5904, + "theoretical_loss": 3.873549202412396, + "tokens_seen": 592314368 + }, + { + "epoch": 0.18, + "learning_rate": 0.000663061183550652, + "loss": 1.5744, + "theoretical_loss": 3.8735047121455644, + "tokens_seen": 592379904 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006630451354062187, + "loss": 1.6039, + "theoretical_loss": 3.873460228178491, + "tokens_seen": 592445440 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006630290872617855, + "loss": 1.4261, + "theoretical_loss": 3.8734157505095856, + "tokens_seen": 592510976 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006630130391173521, + "loss": 1.5184, + "theoretical_loss": 3.8733712791372614, + "tokens_seen": 592576512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006629969909729189, + "loss": 1.5138, + "theoretical_loss": 3.8733268140599306, + "tokens_seen": 592642048 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006629809428284855, + "loss": 1.4651, + "theoretical_loss": 3.8732823552760056, + "tokens_seen": 592707584 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006629648946840523, + "loss": 1.517, + "theoretical_loss": 3.8732379027839, + "tokens_seen": 592773120 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006629488465396189, + "loss": 1.5209, + "theoretical_loss": 3.8731934565820287, + "tokens_seen": 592838656 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006629327983951857, + "loss": 1.483, + "theoretical_loss": 3.8731490166688056, + "tokens_seen": 592904192 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006629167502507523, + "loss": 1.4689, + "theoretical_loss": 3.873104583042646, + "tokens_seen": 592969728 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006629007021063191, + "loss": 1.5707, + "theoretical_loss": 3.8730601557019657, + "tokens_seen": 593035264 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.9914178252220154, + "objective/train/avg_token_score": 0.008582190610468388, + "objective/train/docs_used": 221319, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5671429634094238, + "objective/train/original_loss": 1.5812089443206787, + "objective/train/theoretical_loss": 3.873015734645182, + "objective/train/tokens_used": 613560800, + "objective/train/unlikelihood_loss": 0.02061164565384388, + "theoretical_loss": 3.873015734645182, + "tokens_seen": 593100800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006628846539618857, + "loss": 1.5321, + "theoretical_loss": 3.873015734645182, + "tokens_seen": 593100800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006628686058174525, + "loss": 1.5209, + "theoretical_loss": 3.872971319870711, + "tokens_seen": 593166336 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006628525576730191, + "loss": 1.3468, + "theoretical_loss": 3.872926911376971, + "tokens_seen": 593231872 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006628365095285859, + "loss": 1.5362, + "theoretical_loss": 3.8728825091623786, + "tokens_seen": 593297408 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006628204613841525, + "loss": 1.5164, + "theoretical_loss": 3.872838113225354, + "tokens_seen": 593362944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006628044132397193, + "loss": 1.5299, + "theoretical_loss": 3.8727937235643157, + "tokens_seen": 593428480 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006627883650952859, + "loss": 1.4526, + "theoretical_loss": 3.872749340177685, + "tokens_seen": 593494016 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006627723169508527, + "loss": 1.5701, + "theoretical_loss": 3.87270496306388, + "tokens_seen": 593559552 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006627562688064193, + "loss": 1.6004, + "theoretical_loss": 3.8726605922213233, + "tokens_seen": 593625088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006627402206619861, + "loss": 1.5124, + "theoretical_loss": 3.872616227648436, + "tokens_seen": 593690624 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006627241725175527, + "loss": 1.4599, + "theoretical_loss": 3.8725718693436404, + "tokens_seen": 593756160 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006627081243731195, + "loss": 1.4835, + "theoretical_loss": 3.8725275173053593, + "tokens_seen": 593821696 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006626920762286861, + "loss": 1.4553, + "theoretical_loss": 3.872483171532015, + "tokens_seen": 593887232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006626760280842529, + "loss": 1.5161, + "theoretical_loss": 3.8724388320220333, + "tokens_seen": 593952768 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006626599799398195, + "loss": 1.5113, + "theoretical_loss": 3.872394498773837, + "tokens_seen": 594018304 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006626439317953863, + "loss": 1.5881, + "theoretical_loss": 3.8723501717858513, + "tokens_seen": 594083840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006626278836509529, + "loss": 1.411, + "theoretical_loss": 3.872305851056502, + "tokens_seen": 594149376 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006626118355065197, + "loss": 1.5362, + "theoretical_loss": 3.8722615365842157, + "tokens_seen": 594214912 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006625957873620863, + "loss": 1.5998, + "theoretical_loss": 3.8722172283674183, + "tokens_seen": 594280448 + }, + { + "epoch": 0.18, + "learning_rate": 0.000662579739217653, + "loss": 1.4962, + "theoretical_loss": 3.8721729264045375, + "tokens_seen": 594345984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006625636910732197, + "loss": 1.4674, + "theoretical_loss": 3.872128630694001, + "tokens_seen": 594411520 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006625476429287864, + "loss": 1.6768, + "theoretical_loss": 3.8720843412342374, + "tokens_seen": 594477056 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006625315947843531, + "loss": 1.5539, + "theoretical_loss": 3.872040058023676, + "tokens_seen": 594542592 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006625155466399198, + "loss": 1.5384, + "theoretical_loss": 3.871995781060745, + "tokens_seen": 594608128 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006624994984954865, + "loss": 1.4387, + "theoretical_loss": 3.8719515103438757, + "tokens_seen": 594673664 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006624834503510532, + "loss": 1.5106, + "theoretical_loss": 3.871907245871499, + "tokens_seen": 594739200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006624674022066199, + "loss": 1.6378, + "theoretical_loss": 3.871862987642045, + "tokens_seen": 594804736 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006624513540621866, + "loss": 1.4011, + "theoretical_loss": 3.871818735653947, + "tokens_seen": 594870272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006624353059177532, + "loss": 1.4121, + "theoretical_loss": 3.8717744899056354, + "tokens_seen": 594935808 + }, + { + "epoch": 0.18, + "learning_rate": 0.00066241925777332, + "loss": 1.5365, + "theoretical_loss": 3.8717302503955446, + "tokens_seen": 595001344 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006624032096288866, + "loss": 1.5174, + "theoretical_loss": 3.8716860171221086, + "tokens_seen": 595066880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006623871614844534, + "loss": 1.512, + "theoretical_loss": 3.8716417900837596, + "tokens_seen": 595132416 + }, + { + "epoch": 0.18, + "learning_rate": 0.00066237111334002, + "loss": 1.5443, + "theoretical_loss": 3.8715975692789337, + "tokens_seen": 595197952 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006623550651955868, + "loss": 1.5006, + "theoretical_loss": 3.871553354706066, + "tokens_seen": 595263488 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006623390170511534, + "loss": 1.6514, + "theoretical_loss": 3.871509146363592, + "tokens_seen": 595329024 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006623229689067202, + "loss": 1.5213, + "theoretical_loss": 3.8714649442499476, + "tokens_seen": 595394560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006623069207622868, + "loss": 1.4722, + "theoretical_loss": 3.8714207483635703, + "tokens_seen": 595460096 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006622908726178536, + "loss": 1.4876, + "theoretical_loss": 3.8713765587028974, + "tokens_seen": 595525632 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006622748244734202, + "loss": 1.5071, + "theoretical_loss": 3.871332375266367, + "tokens_seen": 595591168 + }, + { + "epoch": 0.18, + "learning_rate": 0.000662258776328987, + "loss": 1.5269, + "theoretical_loss": 3.871288198052418, + "tokens_seen": 595656704 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006622427281845536, + "loss": 1.5505, + "theoretical_loss": 3.871244027059489, + "tokens_seen": 595722240 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006622266800401204, + "loss": 1.3877, + "theoretical_loss": 3.8711998622860193, + "tokens_seen": 595787776 + }, + { + "epoch": 0.18, + "learning_rate": 0.000662210631895687, + "loss": 1.6627, + "theoretical_loss": 3.8711557037304507, + "tokens_seen": 595853312 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006621945837512538, + "loss": 1.5007, + "theoretical_loss": 3.8711115513912224, + "tokens_seen": 595918848 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006621785356068204, + "loss": 1.4224, + "theoretical_loss": 3.8710674052667766, + "tokens_seen": 595984384 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006621624874623872, + "loss": 1.5724, + "theoretical_loss": 3.8710232653555554, + "tokens_seen": 596049920 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006621464393179538, + "loss": 1.4747, + "theoretical_loss": 3.870979131656001, + "tokens_seen": 596115456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006621303911735206, + "loss": 1.6178, + "theoretical_loss": 3.870935004166557, + "tokens_seen": 596180992 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006621143430290872, + "loss": 1.4234, + "theoretical_loss": 3.870890882885667, + "tokens_seen": 596246528 + }, + { + "epoch": 0.18, + "learning_rate": 0.000662098294884654, + "loss": 1.4921, + "theoretical_loss": 3.8708467678117744, + "tokens_seen": 596312064 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.9931483268737793, + "objective/train/avg_token_score": 0.006851656828075647, + "objective/train/docs_used": 222574, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.464540719985962, + "objective/train/original_loss": 1.47487211227417, + "objective/train/theoretical_loss": 3.870802658943324, + "objective/train/tokens_used": 616837600, + "objective/train/unlikelihood_loss": 0.020491760224103928, + "theoretical_loss": 3.870802658943324, + "tokens_seen": 596377600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006620822467402206, + "loss": 1.4373, + "theoretical_loss": 3.870802658943324, + "tokens_seen": 596377600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006620661985957874, + "loss": 1.4573, + "theoretical_loss": 3.8707585562787625, + "tokens_seen": 596443136 + }, + { + "epoch": 0.18, + "learning_rate": 0.000662050150451354, + "loss": 1.6672, + "theoretical_loss": 3.8707144598165346, + "tokens_seen": 596508672 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006620341023069208, + "loss": 1.6316, + "theoretical_loss": 3.8706703695550875, + "tokens_seen": 596574208 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006620180541624874, + "loss": 1.3836, + "theoretical_loss": 3.8706262854928672, + "tokens_seen": 596639744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006620020060180542, + "loss": 1.5576, + "theoretical_loss": 3.8705822076283223, + "tokens_seen": 596705280 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006619859578736208, + "loss": 1.4473, + "theoretical_loss": 3.870538135959901, + "tokens_seen": 596770816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006619699097291876, + "loss": 1.4236, + "theoretical_loss": 3.8704940704860507, + "tokens_seen": 596836352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006619538615847542, + "loss": 1.5921, + "theoretical_loss": 3.8704500112052216, + "tokens_seen": 596901888 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661937813440321, + "loss": 1.5333, + "theoretical_loss": 3.870405958115864, + "tokens_seen": 596967424 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006619217652958876, + "loss": 1.6462, + "theoretical_loss": 3.870361911216427, + "tokens_seen": 597032960 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006619057171514544, + "loss": 1.5966, + "theoretical_loss": 3.8703178705053625, + "tokens_seen": 597098496 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661889669007021, + "loss": 1.5374, + "theoretical_loss": 3.8702738359811217, + "tokens_seen": 597164032 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006618736208625878, + "loss": 1.5358, + "theoretical_loss": 3.8702298076421564, + "tokens_seen": 597229568 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006618575727181545, + "loss": 1.5249, + "theoretical_loss": 3.87018578548692, + "tokens_seen": 597295104 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006618415245737212, + "loss": 1.3971, + "theoretical_loss": 3.8701417695138645, + "tokens_seen": 597360640 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661825476429288, + "loss": 1.5231, + "theoretical_loss": 3.8700977597214443, + "tokens_seen": 597426176 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006618094282848546, + "loss": 1.4985, + "theoretical_loss": 3.870053756108114, + "tokens_seen": 597491712 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006617933801404213, + "loss": 1.4083, + "theoretical_loss": 3.8700097586723277, + "tokens_seen": 597557248 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661777331995988, + "loss": 1.5819, + "theoretical_loss": 3.869965767412541, + "tokens_seen": 597622784 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006617612838515548, + "loss": 1.5279, + "theoretical_loss": 3.8699217823272103, + "tokens_seen": 597688320 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006617452357071214, + "loss": 1.652, + "theoretical_loss": 3.8698778034147914, + "tokens_seen": 597753856 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006617291875626882, + "loss": 1.534, + "theoretical_loss": 3.869833830673742, + "tokens_seen": 597819392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006617131394182548, + "loss": 1.5503, + "theoretical_loss": 3.8697898641025192, + "tokens_seen": 597884928 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006616970912738216, + "loss": 1.6599, + "theoretical_loss": 3.8697459036995814, + "tokens_seen": 597950464 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006616810431293882, + "loss": 1.4422, + "theoretical_loss": 3.869701949463387, + "tokens_seen": 598016000 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661664994984955, + "loss": 1.5468, + "theoretical_loss": 3.869658001392396, + "tokens_seen": 598081536 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006616489468405216, + "loss": 1.5979, + "theoretical_loss": 3.869614059485068, + "tokens_seen": 598147072 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006616328986960884, + "loss": 1.5976, + "theoretical_loss": 3.8695701237398623, + "tokens_seen": 598212608 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661616850551655, + "loss": 1.6217, + "theoretical_loss": 3.869526194155241, + "tokens_seen": 598278144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006616008024072218, + "loss": 1.4577, + "theoretical_loss": 3.869482270729665, + "tokens_seen": 598343680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006615847542627884, + "loss": 1.5972, + "theoretical_loss": 3.8694383534615966, + "tokens_seen": 598409216 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006615687061183552, + "loss": 1.5393, + "theoretical_loss": 3.8693944423494986, + "tokens_seen": 598474752 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006615526579739218, + "loss": 1.5203, + "theoretical_loss": 3.869350537391834, + "tokens_seen": 598540288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006615366098294886, + "loss": 1.4385, + "theoretical_loss": 3.869306638587066, + "tokens_seen": 598605824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006615205616850552, + "loss": 1.56, + "theoretical_loss": 3.8692627459336597, + "tokens_seen": 598671360 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661504513540622, + "loss": 1.5414, + "theoretical_loss": 3.8692188594300787, + "tokens_seen": 598736896 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006614884653961886, + "loss": 1.4797, + "theoretical_loss": 3.8691749790747894, + "tokens_seen": 598802432 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006614724172517554, + "loss": 1.4484, + "theoretical_loss": 3.8691311048662573, + "tokens_seen": 598867968 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661456369107322, + "loss": 1.5141, + "theoretical_loss": 3.869087236802949, + "tokens_seen": 598933504 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006614403209628888, + "loss": 1.5369, + "theoretical_loss": 3.869043374883331, + "tokens_seen": 598999040 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006614242728184554, + "loss": 1.5227, + "theoretical_loss": 3.8689995191058717, + "tokens_seen": 599064576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006614082246740222, + "loss": 1.5676, + "theoretical_loss": 3.868955669469039, + "tokens_seen": 599130112 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006613921765295888, + "loss": 1.5715, + "theoretical_loss": 3.868911825971301, + "tokens_seen": 599195648 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006613761283851556, + "loss": 1.6039, + "theoretical_loss": 3.868867988611127, + "tokens_seen": 599261184 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006613600802407222, + "loss": 1.6573, + "theoretical_loss": 3.868824157386987, + "tokens_seen": 599326720 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661344032096289, + "loss": 1.459, + "theoretical_loss": 3.8687803322973506, + "tokens_seen": 599392256 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006613279839518556, + "loss": 1.4713, + "theoretical_loss": 3.86873651334069, + "tokens_seen": 599457792 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006613119358074224, + "loss": 1.5157, + "theoretical_loss": 3.8686927005154756, + "tokens_seen": 599523328 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661295887662989, + "loss": 1.4645, + "theoretical_loss": 3.8686488938201795, + "tokens_seen": 599588864 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.9909364581108093, + "objective/train/avg_token_score": 0.009063506498932838, + "objective/train/docs_used": 223680, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6003732681274414, + "objective/train/original_loss": 1.6141016483306885, + "objective/train/theoretical_loss": 3.868605093253274, + "objective/train/tokens_used": 620114400, + "objective/train/unlikelihood_loss": 0.02402293123304844, + "theoretical_loss": 3.868605093253274, + "tokens_seen": 599654400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006612798395185558, + "loss": 1.5019, + "theoretical_loss": 3.868605093253274, + "tokens_seen": 599654400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006612637913741224, + "loss": 1.5636, + "theoretical_loss": 3.8685612988132334, + "tokens_seen": 599719936 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006612477432296892, + "loss": 1.5986, + "theoretical_loss": 3.8685175104985294, + "tokens_seen": 599785472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006612316950852558, + "loss": 1.4864, + "theoretical_loss": 3.868473728307637, + "tokens_seen": 599851008 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006612156469408226, + "loss": 1.5957, + "theoretical_loss": 3.868429952239031, + "tokens_seen": 599916544 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006611995987963892, + "loss": 1.57, + "theoretical_loss": 3.8683861822911867, + "tokens_seen": 599982080 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006611835506519559, + "loss": 1.4888, + "theoretical_loss": 3.8683424184625794, + "tokens_seen": 600047616 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006611675025075226, + "loss": 1.5012, + "theoretical_loss": 3.8682986607516856, + "tokens_seen": 600113152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006611514543630893, + "loss": 1.5791, + "theoretical_loss": 3.8682549091569824, + "tokens_seen": 600178688 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661135406218656, + "loss": 1.502, + "theoretical_loss": 3.868211163676947, + "tokens_seen": 600244224 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006611193580742227, + "loss": 1.6747, + "theoretical_loss": 3.868167424310057, + "tokens_seen": 600309760 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006611033099297894, + "loss": 1.5874, + "theoretical_loss": 3.868123691054792, + "tokens_seen": 600375296 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006610872617853561, + "loss": 1.5012, + "theoretical_loss": 3.8680799639096297, + "tokens_seen": 600440832 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006610712136409228, + "loss": 1.5397, + "theoretical_loss": 3.8680362428730506, + "tokens_seen": 600506368 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006610551654964895, + "loss": 1.5895, + "theoretical_loss": 3.8679925279435343, + "tokens_seen": 600571904 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006610391173520561, + "loss": 1.5178, + "theoretical_loss": 3.8679488191195617, + "tokens_seen": 600637440 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006610230692076229, + "loss": 1.5801, + "theoretical_loss": 3.8679051163996143, + "tokens_seen": 600702976 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006610070210631895, + "loss": 1.4755, + "theoretical_loss": 3.867861419782173, + "tokens_seen": 600768512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006609909729187563, + "loss": 1.6363, + "theoretical_loss": 3.867817729265721, + "tokens_seen": 600834048 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006609749247743229, + "loss": 1.5785, + "theoretical_loss": 3.8677740448487414, + "tokens_seen": 600899584 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006609588766298897, + "loss": 1.6068, + "theoretical_loss": 3.867730366529716, + "tokens_seen": 600965120 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006609428284854563, + "loss": 1.4835, + "theoretical_loss": 3.86768669430713, + "tokens_seen": 601030656 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006609267803410231, + "loss": 1.5605, + "theoretical_loss": 3.867643028179468, + "tokens_seen": 601096192 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006609107321965897, + "loss": 1.5705, + "theoretical_loss": 3.867599368145214, + "tokens_seen": 601161728 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006608946840521565, + "loss": 1.6103, + "theoretical_loss": 3.8675557142028545, + "tokens_seen": 601227264 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006608786359077231, + "loss": 1.5221, + "theoretical_loss": 3.8675120663508755, + "tokens_seen": 601292800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006608625877632899, + "loss": 1.6024, + "theoretical_loss": 3.867468424587763, + "tokens_seen": 601358336 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006608465396188565, + "loss": 1.6477, + "theoretical_loss": 3.867424788912005, + "tokens_seen": 601423872 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006608304914744233, + "loss": 1.5623, + "theoretical_loss": 3.8673811593220884, + "tokens_seen": 601489408 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006608144433299899, + "loss": 1.4367, + "theoretical_loss": 3.8673375358165014, + "tokens_seen": 601554944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006607983951855567, + "loss": 1.5076, + "theoretical_loss": 3.867293918393734, + "tokens_seen": 601620480 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006607823470411233, + "loss": 1.4436, + "theoretical_loss": 3.8672503070522746, + "tokens_seen": 601686016 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006607662988966901, + "loss": 1.5483, + "theoretical_loss": 3.8672067017906135, + "tokens_seen": 601751552 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006607502507522567, + "loss": 1.4316, + "theoretical_loss": 3.8671631026072406, + "tokens_seen": 601817088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006607342026078235, + "loss": 1.6043, + "theoretical_loss": 3.8671195095006468, + "tokens_seen": 601882624 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006607181544633901, + "loss": 1.6188, + "theoretical_loss": 3.8670759224693247, + "tokens_seen": 601948160 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006607021063189569, + "loss": 1.4858, + "theoretical_loss": 3.867032341511765, + "tokens_seen": 602013696 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006606860581745235, + "loss": 1.5187, + "theoretical_loss": 3.866988766626461, + "tokens_seen": 602079232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006606700100300903, + "loss": 1.5894, + "theoretical_loss": 3.866945197811906, + "tokens_seen": 602144768 + }, + { + "epoch": 0.18, + "learning_rate": 0.000660653961885657, + "loss": 1.3851, + "theoretical_loss": 3.866901635066593, + "tokens_seen": 602210304 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006606379137412237, + "loss": 1.5674, + "theoretical_loss": 3.8668580783890167, + "tokens_seen": 602275840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006606218655967904, + "loss": 1.5042, + "theoretical_loss": 3.866814527777671, + "tokens_seen": 602341376 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006606058174523571, + "loss": 1.5681, + "theoretical_loss": 3.8667709832310524, + "tokens_seen": 602406912 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006605897693079238, + "loss": 1.6183, + "theoretical_loss": 3.866727444747656, + "tokens_seen": 602472448 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006605737211634905, + "loss": 1.4639, + "theoretical_loss": 3.8666839123259784, + "tokens_seen": 602537984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006605576730190572, + "loss": 1.4855, + "theoretical_loss": 3.8666403859645158, + "tokens_seen": 602603520 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006605416248746239, + "loss": 1.5982, + "theoretical_loss": 3.8665968656617666, + "tokens_seen": 602669056 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006605255767301906, + "loss": 1.4812, + "theoretical_loss": 3.8665533514162282, + "tokens_seen": 602734592 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006605095285857573, + "loss": 1.4927, + "theoretical_loss": 3.8665098432263987, + "tokens_seen": 602800128 + }, + { + "epoch": 0.18, + "learning_rate": 0.000660493480441324, + "loss": 1.5165, + "theoretical_loss": 3.866466341090778, + "tokens_seen": 602865664 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.9939973950386047, + "objective/train/avg_token_score": 0.006002615671604872, + "objective/train/docs_used": 224736, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.489720344543457, + "objective/train/original_loss": 1.4989256858825684, + "objective/train/theoretical_loss": 3.866422845007865, + "objective/train/tokens_used": 623391200, + "objective/train/unlikelihood_loss": 0.01382203958928585, + "theoretical_loss": 3.866422845007865, + "tokens_seen": 602931200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006604774322968907, + "loss": 1.4792, + "theoretical_loss": 3.866422845007865, + "tokens_seen": 602931200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006604613841524574, + "loss": 1.5172, + "theoretical_loss": 3.86637935497616, + "tokens_seen": 602996736 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006604453360080241, + "loss": 1.4326, + "theoretical_loss": 3.8663358709941633, + "tokens_seen": 603062272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006604292878635908, + "loss": 1.4686, + "theoretical_loss": 3.8662923930603768, + "tokens_seen": 603127808 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006604132397191575, + "loss": 1.343, + "theoretical_loss": 3.8662489211733018, + "tokens_seen": 603193344 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006603971915747243, + "loss": 1.6125, + "theoretical_loss": 3.8662054553314404, + "tokens_seen": 603258880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006603811434302909, + "loss": 1.5092, + "theoretical_loss": 3.866161995533295, + "tokens_seen": 603324416 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006603650952858577, + "loss": 1.5319, + "theoretical_loss": 3.86611854177737, + "tokens_seen": 603389952 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006603490471414243, + "loss": 1.5142, + "theoretical_loss": 3.866075094062168, + "tokens_seen": 603455488 + }, + { + "epoch": 0.18, + "learning_rate": 0.000660332998996991, + "loss": 1.4702, + "theoretical_loss": 3.8660316523861944, + "tokens_seen": 603521024 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006603169508525577, + "loss": 1.4434, + "theoretical_loss": 3.8659882167479527, + "tokens_seen": 603586560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006603009027081245, + "loss": 1.5686, + "theoretical_loss": 3.86594478714595, + "tokens_seen": 603652096 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006602848545636911, + "loss": 1.4328, + "theoretical_loss": 3.8659013635786907, + "tokens_seen": 603717632 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006602688064192579, + "loss": 1.5647, + "theoretical_loss": 3.8658579460446827, + "tokens_seen": 603783168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006602527582748245, + "loss": 1.5893, + "theoretical_loss": 3.865814534542432, + "tokens_seen": 603848704 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006602367101303913, + "loss": 1.5378, + "theoretical_loss": 3.8657711290704473, + "tokens_seen": 603914240 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006602206619859579, + "loss": 1.6071, + "theoretical_loss": 3.8657277296272348, + "tokens_seen": 603979776 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006602046138415247, + "loss": 1.4755, + "theoretical_loss": 3.865684336211305, + "tokens_seen": 604045312 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006601885656970913, + "loss": 1.5088, + "theoretical_loss": 3.8656409488211656, + "tokens_seen": 604110848 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006601725175526581, + "loss": 1.5512, + "theoretical_loss": 3.8655975674553273, + "tokens_seen": 604176384 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006601564694082247, + "loss": 1.4869, + "theoretical_loss": 3.8655541921123, + "tokens_seen": 604241920 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006601404212637915, + "loss": 1.4415, + "theoretical_loss": 3.8655108227905943, + "tokens_seen": 604307456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006601243731193581, + "loss": 1.6175, + "theoretical_loss": 3.8654674594887215, + "tokens_seen": 604372992 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006601083249749249, + "loss": 1.61, + "theoretical_loss": 3.8654241022051927, + "tokens_seen": 604438528 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006600922768304915, + "loss": 1.5525, + "theoretical_loss": 3.865380750938522, + "tokens_seen": 604504064 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006600762286860583, + "loss": 1.5837, + "theoretical_loss": 3.86533740568722, + "tokens_seen": 604569600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006600601805416249, + "loss": 1.5536, + "theoretical_loss": 3.8652940664498026, + "tokens_seen": 604635136 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006600441323971917, + "loss": 1.4899, + "theoretical_loss": 3.865250733224781, + "tokens_seen": 604700672 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006600280842527583, + "loss": 1.494, + "theoretical_loss": 3.865207406010672, + "tokens_seen": 604766208 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006600120361083251, + "loss": 1.5317, + "theoretical_loss": 3.865164084805989, + "tokens_seen": 604831744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006599959879638917, + "loss": 1.534, + "theoretical_loss": 3.865120769609248, + "tokens_seen": 604897280 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006599799398194585, + "loss": 1.5021, + "theoretical_loss": 3.8650774604189655, + "tokens_seen": 604962816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006599638916750251, + "loss": 1.5978, + "theoretical_loss": 3.8650341572336573, + "tokens_seen": 605028352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006599478435305919, + "loss": 1.3931, + "theoretical_loss": 3.8649908600518406, + "tokens_seen": 605093888 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006599317953861585, + "loss": 1.5171, + "theoretical_loss": 3.8649475688720334, + "tokens_seen": 605159424 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006599157472417253, + "loss": 1.6608, + "theoretical_loss": 3.864904283692754, + "tokens_seen": 605224960 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006598996990972919, + "loss": 1.5243, + "theoretical_loss": 3.8648610045125196, + "tokens_seen": 605290496 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006598836509528587, + "loss": 1.5829, + "theoretical_loss": 3.8648177313298513, + "tokens_seen": 605356032 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006598676028084253, + "loss": 1.5754, + "theoretical_loss": 3.8647744641432675, + "tokens_seen": 605421568 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006598515546639921, + "loss": 1.4984, + "theoretical_loss": 3.8647312029512886, + "tokens_seen": 605487104 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006598355065195587, + "loss": 1.4873, + "theoretical_loss": 3.864687947752436, + "tokens_seen": 605552640 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006598194583751255, + "loss": 1.6437, + "theoretical_loss": 3.8646446985452307, + "tokens_seen": 605618176 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006598034102306921, + "loss": 1.5327, + "theoretical_loss": 3.864601455328194, + "tokens_seen": 605683712 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006597873620862588, + "loss": 1.4741, + "theoretical_loss": 3.864558218099849, + "tokens_seen": 605749248 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006597713139418255, + "loss": 1.5162, + "theoretical_loss": 3.864514986858718, + "tokens_seen": 605814784 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006597552657973922, + "loss": 1.5547, + "theoretical_loss": 3.8644717616033244, + "tokens_seen": 605880320 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006597392176529589, + "loss": 1.5122, + "theoretical_loss": 3.8644285423321927, + "tokens_seen": 605945856 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006597231695085256, + "loss": 1.4686, + "theoretical_loss": 3.864385329043847, + "tokens_seen": 606011392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006597071213640923, + "loss": 1.5657, + "theoretical_loss": 3.8643421217368115, + "tokens_seen": 606076928 + }, + { + "epoch": 0.18, + "learning_rate": 0.000659691073219659, + "loss": 1.4404, + "theoretical_loss": 3.8642989204096123, + "tokens_seen": 606142464 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.9944770932197571, + "objective/train/avg_token_score": 0.005522888619452715, + "objective/train/docs_used": 225863, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5205339193344116, + "objective/train/original_loss": 1.5294148921966553, + "objective/train/theoretical_loss": 3.8642557250607754, + "objective/train/tokens_used": 626668000, + "objective/train/unlikelihood_loss": 0.015800734981894493, + "theoretical_loss": 3.8642557250607754, + "tokens_seen": 606208000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006596750250752257, + "loss": 1.4855, + "theoretical_loss": 3.8642557250607754, + "tokens_seen": 606208000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006596589769307924, + "loss": 1.561, + "theoretical_loss": 3.8642125356888277, + "tokens_seen": 606273536 + }, + { + "epoch": 0.18, + "learning_rate": 0.000659642928786359, + "loss": 1.5191, + "theoretical_loss": 3.8641693522922953, + "tokens_seen": 606339072 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006596268806419258, + "loss": 1.5982, + "theoretical_loss": 3.8641261748697064, + "tokens_seen": 606404608 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006596108324974924, + "loss": 1.5418, + "theoretical_loss": 3.8640830034195894, + "tokens_seen": 606470144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006595947843530592, + "loss": 1.6153, + "theoretical_loss": 3.864039837940472, + "tokens_seen": 606535680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006595787362086258, + "loss": 1.5168, + "theoretical_loss": 3.863996678430884, + "tokens_seen": 606601216 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006595626880641926, + "loss": 1.595, + "theoretical_loss": 3.8639535248893555, + "tokens_seen": 606666752 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006595466399197592, + "loss": 1.4755, + "theoretical_loss": 3.863910377314415, + "tokens_seen": 606732288 + }, + { + "epoch": 0.18, + "learning_rate": 0.000659530591775326, + "loss": 1.5451, + "theoretical_loss": 3.863867235704595, + "tokens_seen": 606797824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006595145436308926, + "loss": 1.5119, + "theoretical_loss": 3.8638241000584257, + "tokens_seen": 606863360 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006594984954864594, + "loss": 1.4747, + "theoretical_loss": 3.863780970374439, + "tokens_seen": 606928896 + }, + { + "epoch": 0.18, + "learning_rate": 0.000659482447342026, + "loss": 1.387, + "theoretical_loss": 3.863737846651167, + "tokens_seen": 606994432 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006594663991975928, + "loss": 1.5291, + "theoretical_loss": 3.8636947288871424, + "tokens_seen": 607059968 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006594503510531595, + "loss": 1.5333, + "theoretical_loss": 3.863651617080899, + "tokens_seen": 607125504 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006594343029087262, + "loss": 1.4328, + "theoretical_loss": 3.8636085112309715, + "tokens_seen": 607191040 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006594182547642929, + "loss": 1.5039, + "theoretical_loss": 3.863565411335892, + "tokens_seen": 607256576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006594022066198596, + "loss": 1.5661, + "theoretical_loss": 3.8635223173941964, + "tokens_seen": 607322112 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006593861584754263, + "loss": 1.48, + "theoretical_loss": 3.8634792294044202, + "tokens_seen": 607387648 + }, + { + "epoch": 0.18, + "learning_rate": 0.000659370110330993, + "loss": 1.4558, + "theoretical_loss": 3.8634361473650993, + "tokens_seen": 607453184 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006593540621865597, + "loss": 1.5967, + "theoretical_loss": 3.8633930712747704, + "tokens_seen": 607518720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006593380140421264, + "loss": 1.5159, + "theoretical_loss": 3.8633500011319692, + "tokens_seen": 607584256 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006593219658976931, + "loss": 1.6199, + "theoretical_loss": 3.8633069369352344, + "tokens_seen": 607649792 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006593059177532598, + "loss": 1.5414, + "theoretical_loss": 3.8632638786831035, + "tokens_seen": 607715328 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006592898696088265, + "loss": 1.6388, + "theoretical_loss": 3.863220826374115, + "tokens_seen": 607780864 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006592738214643932, + "loss": 1.5762, + "theoretical_loss": 3.863177780006808, + "tokens_seen": 607846400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006592577733199599, + "loss": 1.567, + "theoretical_loss": 3.8631347395797215, + "tokens_seen": 607911936 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006592417251755266, + "loss": 1.4871, + "theoretical_loss": 3.8630917050913958, + "tokens_seen": 607977472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006592256770310933, + "loss": 1.5382, + "theoretical_loss": 3.863048676540372, + "tokens_seen": 608043008 + }, + { + "epoch": 0.18, + "learning_rate": 0.00065920962888666, + "loss": 1.4029, + "theoretical_loss": 3.8630056539251902, + "tokens_seen": 608108544 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006591935807422267, + "loss": 1.5149, + "theoretical_loss": 3.862962637244393, + "tokens_seen": 608174080 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006591775325977934, + "loss": 1.5096, + "theoretical_loss": 3.862919626496521, + "tokens_seen": 608239616 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006591614844533601, + "loss": 1.6936, + "theoretical_loss": 3.862876621680119, + "tokens_seen": 608305152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006591454363089268, + "loss": 1.5374, + "theoretical_loss": 3.862833622793728, + "tokens_seen": 608370688 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006591293881644935, + "loss": 1.5218, + "theoretical_loss": 3.8627906298358923, + "tokens_seen": 608436224 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006591133400200602, + "loss": 1.4163, + "theoretical_loss": 3.862747642805157, + "tokens_seen": 608501760 + }, + { + "epoch": 0.18, + "learning_rate": 0.000659097291875627, + "loss": 1.616, + "theoretical_loss": 3.8627046617000653, + "tokens_seen": 608567296 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006590812437311936, + "loss": 1.4404, + "theoretical_loss": 3.862661686519163, + "tokens_seen": 608632832 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006590651955867603, + "loss": 1.5155, + "theoretical_loss": 3.862618717260996, + "tokens_seen": 608698368 + }, + { + "epoch": 0.18, + "learning_rate": 0.000659049147442327, + "loss": 1.4971, + "theoretical_loss": 3.8625757539241103, + "tokens_seen": 608763904 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006590330992978938, + "loss": 1.5896, + "theoretical_loss": 3.862532796507053, + "tokens_seen": 608829440 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006590170511534604, + "loss": 1.4805, + "theoretical_loss": 3.8624898450083704, + "tokens_seen": 608894976 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006590010030090272, + "loss": 1.4741, + "theoretical_loss": 3.862446899426611, + "tokens_seen": 608960512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006589849548645938, + "loss": 1.5541, + "theoretical_loss": 3.8624039597603232, + "tokens_seen": 609026048 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006589689067201606, + "loss": 1.5528, + "theoretical_loss": 3.862361026008055, + "tokens_seen": 609091584 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006589528585757272, + "loss": 1.4389, + "theoretical_loss": 3.8623180981683563, + "tokens_seen": 609157120 + }, + { + "epoch": 0.18, + "learning_rate": 0.000658936810431294, + "loss": 1.3706, + "theoretical_loss": 3.8622751762397765, + "tokens_seen": 609222656 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006589207622868606, + "loss": 1.5496, + "theoretical_loss": 3.862232260220866, + "tokens_seen": 609288192 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006589047141424274, + "loss": 1.4987, + "theoretical_loss": 3.8621893501101763, + "tokens_seen": 609353728 + }, + { + "epoch": 0.18, + "learning_rate": 0.000658888665997994, + "loss": 1.5404, + "theoretical_loss": 3.8621464459062573, + "tokens_seen": 609419264 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.9962924718856812, + "objective/train/avg_token_score": 0.0037075344007462263, + "objective/train/docs_used": 227108, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.467069149017334, + "objective/train/original_loss": 1.4732375144958496, + "objective/train/theoretical_loss": 3.8621035476076617, + "objective/train/tokens_used": 629944800, + "objective/train/unlikelihood_loss": 0.010265999473631382, + "theoretical_loss": 3.8621035476076617, + "tokens_seen": 609484800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006588726178535608, + "loss": 1.47, + "theoretical_loss": 3.8621035476076617, + "tokens_seen": 609484800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006588565697091274, + "loss": 1.5415, + "theoretical_loss": 3.862060655212942, + "tokens_seen": 609550336 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006588405215646942, + "loss": 1.4712, + "theoretical_loss": 3.8620177687206514, + "tokens_seen": 609615872 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006588244734202608, + "loss": 1.471, + "theoretical_loss": 3.861974888129342, + "tokens_seen": 609681408 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006588084252758276, + "loss": 1.5095, + "theoretical_loss": 3.8619320134375688, + "tokens_seen": 609746944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006587923771313942, + "loss": 1.5458, + "theoretical_loss": 3.861889144643885, + "tokens_seen": 609812480 + }, + { + "epoch": 0.18, + "learning_rate": 0.000658776328986961, + "loss": 1.3418, + "theoretical_loss": 3.861846281746847, + "tokens_seen": 609878016 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006587602808425276, + "loss": 1.532, + "theoretical_loss": 3.861803424745009, + "tokens_seen": 609943552 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006587442326980944, + "loss": 1.5296, + "theoretical_loss": 3.8617605736369276, + "tokens_seen": 610009088 + }, + { + "epoch": 0.18, + "learning_rate": 0.000658728184553661, + "loss": 1.6806, + "theoretical_loss": 3.8617177284211595, + "tokens_seen": 610074624 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006587121364092278, + "loss": 1.4644, + "theoretical_loss": 3.8616748890962604, + "tokens_seen": 610140160 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006586960882647944, + "loss": 1.508, + "theoretical_loss": 3.8616320556607886, + "tokens_seen": 610205696 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006586800401203612, + "loss": 1.6166, + "theoretical_loss": 3.861589228113302, + "tokens_seen": 610271232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006586639919759278, + "loss": 1.4703, + "theoretical_loss": 3.8615464064523586, + "tokens_seen": 610336768 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006586479438314946, + "loss": 1.5503, + "theoretical_loss": 3.861503590676518, + "tokens_seen": 610402304 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006586318956870612, + "loss": 1.508, + "theoretical_loss": 3.861460780784339, + "tokens_seen": 610467840 + }, + { + "epoch": 0.19, + "learning_rate": 0.000658615847542628, + "loss": 1.5942, + "theoretical_loss": 3.8614179767743826, + "tokens_seen": 610533376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006585997993981946, + "loss": 1.4038, + "theoretical_loss": 3.861375178645208, + "tokens_seen": 610598912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006585837512537614, + "loss": 1.6397, + "theoretical_loss": 3.861332386395377, + "tokens_seen": 610664448 + }, + { + "epoch": 0.19, + "learning_rate": 0.000658567703109328, + "loss": 1.5165, + "theoretical_loss": 3.8612896000234516, + "tokens_seen": 610729984 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006585516549648948, + "loss": 1.5115, + "theoretical_loss": 3.861246819527992, + "tokens_seen": 610795520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006585356068204614, + "loss": 1.5559, + "theoretical_loss": 3.861204044907563, + "tokens_seen": 610861056 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006585195586760282, + "loss": 1.5964, + "theoretical_loss": 3.8611612761607255, + "tokens_seen": 610926592 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006585035105315948, + "loss": 1.4207, + "theoretical_loss": 3.861118513286044, + "tokens_seen": 610992128 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006584874623871616, + "loss": 1.4782, + "theoretical_loss": 3.8610757562820828, + "tokens_seen": 611057664 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006584714142427282, + "loss": 1.4653, + "theoretical_loss": 3.861033005147406, + "tokens_seen": 611123200 + }, + { + "epoch": 0.19, + "learning_rate": 0.000658455366098295, + "loss": 1.444, + "theoretical_loss": 3.8609902598805785, + "tokens_seen": 611188736 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006584393179538616, + "loss": 1.42, + "theoretical_loss": 3.860947520480167, + "tokens_seen": 611254272 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006584232698094284, + "loss": 1.4834, + "theoretical_loss": 3.8609047869447357, + "tokens_seen": 611319808 + }, + { + "epoch": 0.19, + "learning_rate": 0.000658407221664995, + "loss": 1.5163, + "theoretical_loss": 3.8608620592728524, + "tokens_seen": 611385344 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006583911735205618, + "loss": 1.5042, + "theoretical_loss": 3.860819337463084, + "tokens_seen": 611450880 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006583751253761284, + "loss": 1.6017, + "theoretical_loss": 3.860776621513998, + "tokens_seen": 611516416 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006583590772316951, + "loss": 1.5825, + "theoretical_loss": 3.8607339114241617, + "tokens_seen": 611581952 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006583430290872618, + "loss": 1.5461, + "theoretical_loss": 3.860691207192145, + "tokens_seen": 611647488 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006583269809428285, + "loss": 1.4629, + "theoretical_loss": 3.860648508816516, + "tokens_seen": 611713024 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006583109327983952, + "loss": 1.3862, + "theoretical_loss": 3.8606058162958443, + "tokens_seen": 611778560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006582948846539619, + "loss": 1.4256, + "theoretical_loss": 3.8605631296287006, + "tokens_seen": 611844096 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006582788365095286, + "loss": 1.4718, + "theoretical_loss": 3.860520448813655, + "tokens_seen": 611909632 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006582627883650953, + "loss": 1.6006, + "theoretical_loss": 3.8604777738492784, + "tokens_seen": 611975168 + }, + { + "epoch": 0.19, + "learning_rate": 0.000658246740220662, + "loss": 1.4982, + "theoretical_loss": 3.860435104734143, + "tokens_seen": 612040704 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006582306920762287, + "loss": 1.6076, + "theoretical_loss": 3.86039244146682, + "tokens_seen": 612106240 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006582146439317954, + "loss": 1.39, + "theoretical_loss": 3.860349784045883, + "tokens_seen": 612171776 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006581985957873621, + "loss": 1.4516, + "theoretical_loss": 3.8603071324699045, + "tokens_seen": 612237312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006581825476429288, + "loss": 1.4805, + "theoretical_loss": 3.8602644867374583, + "tokens_seen": 612302848 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006581664994984955, + "loss": 1.5379, + "theoretical_loss": 3.860221846847118, + "tokens_seen": 612368384 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006581504513540622, + "loss": 1.5956, + "theoretical_loss": 3.8601792127974583, + "tokens_seen": 612433920 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006581344032096289, + "loss": 1.3121, + "theoretical_loss": 3.860136584587055, + "tokens_seen": 612499456 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006581183550651956, + "loss": 1.5001, + "theoretical_loss": 3.860093962214483, + "tokens_seen": 612564992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006581023069207623, + "loss": 1.4263, + "theoretical_loss": 3.860051345678319, + "tokens_seen": 612630528 + }, + { + "epoch": 0.19, + "learning_rate": 0.000658086258776329, + "loss": 1.5535, + "theoretical_loss": 3.860008734977139, + "tokens_seen": 612696064 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.9955649971961975, + "objective/train/avg_token_score": 0.004435012117028236, + "objective/train/docs_used": 228263, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.680903673171997, + "objective/train/original_loss": 1.6895028352737427, + "objective/train/theoretical_loss": 3.8599661301095196, + "objective/train/tokens_used": 633221600, + "objective/train/unlikelihood_loss": 0.008195028640329838, + "theoretical_loss": 3.8599661301095196, + "tokens_seen": 612761600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006580702106318957, + "loss": 1.5941, + "theoretical_loss": 3.8599661301095196, + "tokens_seen": 612761600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006580541624874624, + "loss": 1.4134, + "theoretical_loss": 3.85992353107404, + "tokens_seen": 612827136 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006580381143430291, + "loss": 1.4733, + "theoretical_loss": 3.8598809378692764, + "tokens_seen": 612892672 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006580220661985958, + "loss": 1.4162, + "theoretical_loss": 3.859838350493809, + "tokens_seen": 612958208 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006580060180541625, + "loss": 1.5126, + "theoretical_loss": 3.8597957689462157, + "tokens_seen": 613023744 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006579899699097292, + "loss": 1.5503, + "theoretical_loss": 3.859753193225077, + "tokens_seen": 613089280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006579739217652959, + "loss": 1.3343, + "theoretical_loss": 3.859710623328972, + "tokens_seen": 613154816 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006579578736208626, + "loss": 1.514, + "theoretical_loss": 3.8596680592564816, + "tokens_seen": 613220352 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006579418254764293, + "loss": 1.5763, + "theoretical_loss": 3.859625501006188, + "tokens_seen": 613285888 + }, + { + "epoch": 0.19, + "learning_rate": 0.000657925777331996, + "loss": 1.5305, + "theoretical_loss": 3.859582948576671, + "tokens_seen": 613351424 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006579097291875627, + "loss": 1.547, + "theoretical_loss": 3.8595404019665143, + "tokens_seen": 613416960 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006578936810431294, + "loss": 1.4717, + "theoretical_loss": 3.859497861174299, + "tokens_seen": 613482496 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006578776328986961, + "loss": 1.5307, + "theoretical_loss": 3.859455326198609, + "tokens_seen": 613548032 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006578615847542628, + "loss": 1.5402, + "theoretical_loss": 3.8594127970380274, + "tokens_seen": 613613568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006578455366098295, + "loss": 1.4372, + "theoretical_loss": 3.859370273691139, + "tokens_seen": 613679104 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006578294884653962, + "loss": 1.3982, + "theoretical_loss": 3.8593277561565276, + "tokens_seen": 613744640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006578134403209629, + "loss": 1.5126, + "theoretical_loss": 3.8592852444327788, + "tokens_seen": 613810176 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006577973921765296, + "loss": 1.4233, + "theoretical_loss": 3.8592427385184775, + "tokens_seen": 613875712 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006577813440320963, + "loss": 1.5516, + "theoretical_loss": 3.85920023841221, + "tokens_seen": 613941248 + }, + { + "epoch": 0.19, + "learning_rate": 0.000657765295887663, + "loss": 1.4608, + "theoretical_loss": 3.859157744112563, + "tokens_seen": 614006784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006577492477432297, + "loss": 1.5465, + "theoretical_loss": 3.859115255618123, + "tokens_seen": 614072320 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006577331995987964, + "loss": 1.5295, + "theoretical_loss": 3.8590727729274787, + "tokens_seen": 614137856 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006577171514543631, + "loss": 1.3972, + "theoretical_loss": 3.859030296039217, + "tokens_seen": 614203392 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006577011033099298, + "loss": 1.6219, + "theoretical_loss": 3.8589878249519263, + "tokens_seen": 614268928 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006576850551654965, + "loss": 1.5279, + "theoretical_loss": 3.8589453596641965, + "tokens_seen": 614334464 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006576690070210633, + "loss": 1.4757, + "theoretical_loss": 3.858902900174617, + "tokens_seen": 614400000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006576529588766299, + "loss": 1.3876, + "theoretical_loss": 3.858860446481777, + "tokens_seen": 614465536 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006576369107321967, + "loss": 1.5297, + "theoretical_loss": 3.8588179985842674, + "tokens_seen": 614531072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006576208625877633, + "loss": 1.3833, + "theoretical_loss": 3.8587755564806794, + "tokens_seen": 614596608 + }, + { + "epoch": 0.19, + "learning_rate": 0.00065760481444333, + "loss": 1.5395, + "theoretical_loss": 3.8587331201696045, + "tokens_seen": 614662144 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006575887662988967, + "loss": 1.4786, + "theoretical_loss": 3.8586906896496336, + "tokens_seen": 614727680 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006575727181544635, + "loss": 1.4285, + "theoretical_loss": 3.8586482649193607, + "tokens_seen": 614793216 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006575566700100301, + "loss": 1.402, + "theoretical_loss": 3.8586058459773778, + "tokens_seen": 614858752 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006575406218655969, + "loss": 1.4077, + "theoretical_loss": 3.8585634328222778, + "tokens_seen": 614924288 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006575245737211635, + "loss": 1.4799, + "theoretical_loss": 3.8585210254526565, + "tokens_seen": 614989824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006575085255767303, + "loss": 1.4011, + "theoretical_loss": 3.8584786238671063, + "tokens_seen": 615055360 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006574924774322969, + "loss": 1.5145, + "theoretical_loss": 3.8584362280642233, + "tokens_seen": 615120896 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006574764292878637, + "loss": 1.4865, + "theoretical_loss": 3.858393838042603, + "tokens_seen": 615186432 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006574603811434303, + "loss": 1.543, + "theoretical_loss": 3.8583514538008403, + "tokens_seen": 615251968 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006574443329989971, + "loss": 1.4408, + "theoretical_loss": 3.858309075337532, + "tokens_seen": 615317504 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006574282848545637, + "loss": 1.3707, + "theoretical_loss": 3.8582667026512754, + "tokens_seen": 615383040 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006574122367101305, + "loss": 1.4476, + "theoretical_loss": 3.858224335740668, + "tokens_seen": 615448576 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006573961885656971, + "loss": 1.51, + "theoretical_loss": 3.8581819746043067, + "tokens_seen": 615514112 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006573801404212639, + "loss": 1.5639, + "theoretical_loss": 3.8581396192407906, + "tokens_seen": 615579648 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006573640922768305, + "loss": 1.4566, + "theoretical_loss": 3.8580972696487184, + "tokens_seen": 615645184 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006573480441323973, + "loss": 1.4388, + "theoretical_loss": 3.858054925826689, + "tokens_seen": 615710720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006573319959879639, + "loss": 1.559, + "theoretical_loss": 3.8580125877733025, + "tokens_seen": 615776256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006573159478435307, + "loss": 1.4303, + "theoretical_loss": 3.8579702554871593, + "tokens_seen": 615841792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006572998996990973, + "loss": 1.3962, + "theoretical_loss": 3.8579279289668604, + "tokens_seen": 615907328 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006572838515546641, + "loss": 1.4835, + "theoretical_loss": 3.8578856082110065, + "tokens_seen": 615972864 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.9928308129310608, + "objective/train/avg_token_score": 0.007169243413954973, + "objective/train/docs_used": 229528, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.454120397567749, + "objective/train/original_loss": 1.465273380279541, + "objective/train/theoretical_loss": 3.8578432932182, + "objective/train/tokens_used": 636498400, + "objective/train/unlikelihood_loss": 0.01767963543534279, + "theoretical_loss": 3.8578432932182, + "tokens_seen": 616038400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006572678034102307, + "loss": 1.5113, + "theoretical_loss": 3.8578432932182, + "tokens_seen": 616038400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006572517552657975, + "loss": 1.4828, + "theoretical_loss": 3.8578009839870426, + "tokens_seen": 616103936 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006572357071213641, + "loss": 1.4973, + "theoretical_loss": 3.8577586805161372, + "tokens_seen": 616169472 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006572196589769309, + "loss": 1.4659, + "theoretical_loss": 3.857716382804087, + "tokens_seen": 616235008 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006572036108324975, + "loss": 1.5144, + "theoretical_loss": 3.8576740908494958, + "tokens_seen": 616300544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006571875626880643, + "loss": 1.5645, + "theoretical_loss": 3.8576318046509677, + "tokens_seen": 616366080 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006571715145436309, + "loss": 1.4411, + "theoretical_loss": 3.857589524207108, + "tokens_seen": 616431616 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006571554663991977, + "loss": 1.4622, + "theoretical_loss": 3.8575472495165206, + "tokens_seen": 616497152 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006571394182547643, + "loss": 1.5152, + "theoretical_loss": 3.8575049805778123, + "tokens_seen": 616562688 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006571233701103311, + "loss": 1.465, + "theoretical_loss": 3.8574627173895886, + "tokens_seen": 616628224 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006571073219658977, + "loss": 1.6054, + "theoretical_loss": 3.8574204599504567, + "tokens_seen": 616693760 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006570912738214645, + "loss": 1.3285, + "theoretical_loss": 3.857378208259023, + "tokens_seen": 616759296 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006570752256770311, + "loss": 1.4336, + "theoretical_loss": 3.8573359623138956, + "tokens_seen": 616824832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006570591775325979, + "loss": 1.5966, + "theoretical_loss": 3.857293722113683, + "tokens_seen": 616890368 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006570431293881645, + "loss": 1.6593, + "theoretical_loss": 3.8572514876569923, + "tokens_seen": 616955904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006570270812437313, + "loss": 1.4997, + "theoretical_loss": 3.857209258942434, + "tokens_seen": 617021440 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006570110330992979, + "loss": 1.5145, + "theoretical_loss": 3.857167035968617, + "tokens_seen": 617086976 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006569949849548647, + "loss": 1.5905, + "theoretical_loss": 3.8571248187341522, + "tokens_seen": 617152512 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006569789368104313, + "loss": 1.4589, + "theoretical_loss": 3.8570826072376487, + "tokens_seen": 617218048 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656962888665998, + "loss": 1.5411, + "theoretical_loss": 3.8570404014777178, + "tokens_seen": 617283584 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006569468405215647, + "loss": 1.5132, + "theoretical_loss": 3.8569982014529716, + "tokens_seen": 617349120 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006569307923771314, + "loss": 1.5089, + "theoretical_loss": 3.8569560071620224, + "tokens_seen": 617414656 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006569147442326981, + "loss": 1.4568, + "theoretical_loss": 3.8569138186034815, + "tokens_seen": 617480192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006568986960882648, + "loss": 1.5084, + "theoretical_loss": 3.8568716357759625, + "tokens_seen": 617545728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006568826479438315, + "loss": 1.6105, + "theoretical_loss": 3.856829458678079, + "tokens_seen": 617611264 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006568665997993982, + "loss": 1.4847, + "theoretical_loss": 3.856787287308444, + "tokens_seen": 617676800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006568505516549649, + "loss": 1.5418, + "theoretical_loss": 3.8567451216656723, + "tokens_seen": 617742336 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006568345035105316, + "loss": 1.5509, + "theoretical_loss": 3.8567029617483795, + "tokens_seen": 617807872 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006568184553660983, + "loss": 1.4056, + "theoretical_loss": 3.8566608075551803, + "tokens_seen": 617873408 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656802407221665, + "loss": 1.5413, + "theoretical_loss": 3.8566186590846905, + "tokens_seen": 617938944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006567863590772317, + "loss": 1.5401, + "theoretical_loss": 3.856576516335526, + "tokens_seen": 618004480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006567703109327984, + "loss": 1.5161, + "theoretical_loss": 3.8565343793063045, + "tokens_seen": 618070016 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006567542627883651, + "loss": 1.5703, + "theoretical_loss": 3.856492247995643, + "tokens_seen": 618135552 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006567382146439318, + "loss": 1.4732, + "theoretical_loss": 3.856450122402159, + "tokens_seen": 618201088 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006567221664994985, + "loss": 1.6347, + "theoretical_loss": 3.8564080025244705, + "tokens_seen": 618266624 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006567061183550652, + "loss": 1.5217, + "theoretical_loss": 3.8563658883611964, + "tokens_seen": 618332160 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006566900702106319, + "loss": 1.4645, + "theoretical_loss": 3.856323779910956, + "tokens_seen": 618397696 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006566740220661986, + "loss": 1.5874, + "theoretical_loss": 3.856281677172369, + "tokens_seen": 618463232 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006566579739217653, + "loss": 1.4492, + "theoretical_loss": 3.856239580144056, + "tokens_seen": 618528768 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656641925777332, + "loss": 1.4414, + "theoretical_loss": 3.8561974888246366, + "tokens_seen": 618594304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006566258776328987, + "loss": 1.5317, + "theoretical_loss": 3.8561554032127323, + "tokens_seen": 618659840 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006566098294884654, + "loss": 1.4749, + "theoretical_loss": 3.856113323306965, + "tokens_seen": 618725376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006565937813440321, + "loss": 1.5379, + "theoretical_loss": 3.8560712491059563, + "tokens_seen": 618790912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006565777331995988, + "loss": 1.3596, + "theoretical_loss": 3.8560291806083296, + "tokens_seen": 618856448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006565616850551655, + "loss": 1.4528, + "theoretical_loss": 3.855987117812707, + "tokens_seen": 618921984 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006565456369107322, + "loss": 1.6002, + "theoretical_loss": 3.8559450607177124, + "tokens_seen": 618987520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006565295887662989, + "loss": 1.5599, + "theoretical_loss": 3.8559030093219695, + "tokens_seen": 619053056 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006565135406218656, + "loss": 1.5749, + "theoretical_loss": 3.855860963624103, + "tokens_seen": 619118592 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006564974924774323, + "loss": 1.5059, + "theoretical_loss": 3.855818923622738, + "tokens_seen": 619184128 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656481444332999, + "loss": 1.449, + "theoretical_loss": 3.855776889316499, + "tokens_seen": 619249664 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.9949982166290283, + "objective/train/avg_token_score": 0.005001781042665243, + "objective/train/docs_used": 230719, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.664671778678894, + "objective/train/original_loss": 1.6744531393051147, + "objective/train/theoretical_loss": 3.855734860704014, + "objective/train/tokens_used": 639775200, + "objective/train/unlikelihood_loss": 0.008834917098283768, + "theoretical_loss": 3.855734860704014, + "tokens_seen": 619315200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006564653961885657, + "loss": 1.6181, + "theoretical_loss": 3.855734860704014, + "tokens_seen": 619315200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006564493480441324, + "loss": 1.5016, + "theoretical_loss": 3.8556928377839066, + "tokens_seen": 619380736 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006564332998996991, + "loss": 1.614, + "theoretical_loss": 3.8556508205548057, + "tokens_seen": 619446272 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006564172517552658, + "loss": 1.6157, + "theoretical_loss": 3.8556088090153375, + "tokens_seen": 619511808 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006564012036108325, + "loss": 1.5132, + "theoretical_loss": 3.8555668031641304, + "tokens_seen": 619577344 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006563851554663992, + "loss": 1.5834, + "theoretical_loss": 3.8555248029998124, + "tokens_seen": 619642880 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656369107321966, + "loss": 1.4118, + "theoretical_loss": 3.855482808521012, + "tokens_seen": 619708416 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006563530591775326, + "loss": 1.4789, + "theoretical_loss": 3.855440819726359, + "tokens_seen": 619773952 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006563370110330993, + "loss": 1.539, + "theoretical_loss": 3.8553988366144827, + "tokens_seen": 619839488 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656320962888666, + "loss": 1.5626, + "theoretical_loss": 3.8553568591840133, + "tokens_seen": 619905024 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006563049147442327, + "loss": 1.547, + "theoretical_loss": 3.8553148874335816, + "tokens_seen": 619970560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006562888665997994, + "loss": 1.5206, + "theoretical_loss": 3.855272921361819, + "tokens_seen": 620036096 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006562728184553662, + "loss": 1.5702, + "theoretical_loss": 3.855230960967356, + "tokens_seen": 620101632 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006562567703109328, + "loss": 1.5273, + "theoretical_loss": 3.8551890062488265, + "tokens_seen": 620167168 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006562407221664996, + "loss": 1.5178, + "theoretical_loss": 3.855147057204861, + "tokens_seen": 620232704 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006562246740220662, + "loss": 1.5396, + "theoretical_loss": 3.8551051138340937, + "tokens_seen": 620298240 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656208625877633, + "loss": 1.5655, + "theoretical_loss": 3.8550631761351575, + "tokens_seen": 620363776 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006561925777331996, + "loss": 1.5519, + "theoretical_loss": 3.8550212441066876, + "tokens_seen": 620429312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006561765295887664, + "loss": 1.4698, + "theoretical_loss": 3.8549793177473166, + "tokens_seen": 620494848 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656160481444333, + "loss": 1.5174, + "theoretical_loss": 3.8549373970556813, + "tokens_seen": 620560384 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006561444332998998, + "loss": 1.4461, + "theoretical_loss": 3.8548954820304155, + "tokens_seen": 620625920 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006561283851554664, + "loss": 1.5207, + "theoretical_loss": 3.854853572670156, + "tokens_seen": 620691456 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006561123370110332, + "loss": 1.4529, + "theoretical_loss": 3.8548116689735386, + "tokens_seen": 620756992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006560962888665998, + "loss": 1.4558, + "theoretical_loss": 3.8547697709392006, + "tokens_seen": 620822528 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006560802407221666, + "loss": 1.5503, + "theoretical_loss": 3.854727878565779, + "tokens_seen": 620888064 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006560641925777332, + "loss": 1.6126, + "theoretical_loss": 3.8546859918519107, + "tokens_seen": 620953600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006560481444333, + "loss": 1.4876, + "theoretical_loss": 3.8546441107962357, + "tokens_seen": 621019136 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006560320962888666, + "loss": 1.6171, + "theoretical_loss": 3.854602235397391, + "tokens_seen": 621084672 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006560160481444334, + "loss": 1.4891, + "theoretical_loss": 3.8545603656540175, + "tokens_seen": 621150208 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656, + "loss": 1.4648, + "theoretical_loss": 3.854518501564753, + "tokens_seen": 621215744 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006559839518555668, + "loss": 1.4742, + "theoretical_loss": 3.8544766431282387, + "tokens_seen": 621281280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006559679037111334, + "loss": 1.5403, + "theoretical_loss": 3.854434790343115, + "tokens_seen": 621346816 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006559518555667002, + "loss": 1.339, + "theoretical_loss": 3.8543929432080226, + "tokens_seen": 621412352 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006559358074222668, + "loss": 1.5478, + "theoretical_loss": 3.8543511017216034, + "tokens_seen": 621477888 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006559197592778336, + "loss": 1.4493, + "theoretical_loss": 3.8543092658825, + "tokens_seen": 621543424 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006559037111334002, + "loss": 1.4908, + "theoretical_loss": 3.8542674356893527, + "tokens_seen": 621608960 + }, + { + "epoch": 0.19, + "learning_rate": 0.000655887662988967, + "loss": 1.467, + "theoretical_loss": 3.8542256111408064, + "tokens_seen": 621674496 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006558716148445336, + "loss": 1.5103, + "theoretical_loss": 3.854183792235504, + "tokens_seen": 621740032 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006558555667001004, + "loss": 1.4225, + "theoretical_loss": 3.8541419789720894, + "tokens_seen": 621805568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006558395185556671, + "loss": 1.5027, + "theoretical_loss": 3.854100171349206, + "tokens_seen": 621871104 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006558234704112338, + "loss": 1.6033, + "theoretical_loss": 3.8540583693655, + "tokens_seen": 621936640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006558074222668005, + "loss": 1.5056, + "theoretical_loss": 3.854016573019616, + "tokens_seen": 622002176 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006557913741223672, + "loss": 1.5441, + "theoretical_loss": 3.8539747823101997, + "tokens_seen": 622067712 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006557753259779339, + "loss": 1.4864, + "theoretical_loss": 3.853932997235897, + "tokens_seen": 622133248 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006557592778335006, + "loss": 1.549, + "theoretical_loss": 3.8538912177953555, + "tokens_seen": 622198784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006557432296890673, + "loss": 1.4291, + "theoretical_loss": 3.8538494439872215, + "tokens_seen": 622264320 + }, + { + "epoch": 0.19, + "learning_rate": 0.000655727181544634, + "loss": 1.3711, + "theoretical_loss": 3.8538076758101427, + "tokens_seen": 622329856 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006557111334002006, + "loss": 1.6322, + "theoretical_loss": 3.8537659132627677, + "tokens_seen": 622395392 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006556950852557674, + "loss": 1.607, + "theoretical_loss": 3.853724156343744, + "tokens_seen": 622460928 + }, + { + "epoch": 0.19, + "learning_rate": 0.000655679037111334, + "loss": 1.5168, + "theoretical_loss": 3.853682405051722, + "tokens_seen": 622526464 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.9887832999229431, + "objective/train/avg_token_score": 0.011216707527637482, + "objective/train/docs_used": 231894, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5806920528411865, + "objective/train/original_loss": 1.5983679294586182, + "objective/train/theoretical_loss": 3.8536406593853503, + "objective/train/tokens_used": 643052000, + "objective/train/unlikelihood_loss": 0.02259984239935875, + "theoretical_loss": 3.8536406593853503, + "tokens_seen": 622592000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006556629889669008, + "loss": 1.528, + "theoretical_loss": 3.8536406593853503, + "tokens_seen": 622592000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006556469408224674, + "loss": 1.5265, + "theoretical_loss": 3.8535989193432787, + "tokens_seen": 622657536 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006556308926780342, + "loss": 1.5663, + "theoretical_loss": 3.8535571849241577, + "tokens_seen": 622723072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006556148445336008, + "loss": 1.4024, + "theoretical_loss": 3.8535154561266385, + "tokens_seen": 622788608 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006555987963891676, + "loss": 1.4892, + "theoretical_loss": 3.8534737329493725, + "tokens_seen": 622854144 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006555827482447342, + "loss": 1.5157, + "theoretical_loss": 3.853432015391011, + "tokens_seen": 622919680 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006555667001003009, + "loss": 1.4112, + "theoretical_loss": 3.8533903034502064, + "tokens_seen": 622985216 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006555506519558676, + "loss": 1.5077, + "theoretical_loss": 3.8533485971256125, + "tokens_seen": 623050752 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006555346038114343, + "loss": 1.4965, + "theoretical_loss": 3.8533068964158805, + "tokens_seen": 623116288 + }, + { + "epoch": 0.19, + "learning_rate": 0.000655518555667001, + "loss": 1.4678, + "theoretical_loss": 3.8532652013196653, + "tokens_seen": 623181824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006555025075225677, + "loss": 1.7007, + "theoretical_loss": 3.853223511835621, + "tokens_seen": 623247360 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006554864593781344, + "loss": 1.4837, + "theoretical_loss": 3.853181827962402, + "tokens_seen": 623312896 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006554704112337011, + "loss": 1.484, + "theoretical_loss": 3.853140149698664, + "tokens_seen": 623378432 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006554543630892678, + "loss": 1.4778, + "theoretical_loss": 3.8530984770430607, + "tokens_seen": 623443968 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006554383149448345, + "loss": 1.4514, + "theoretical_loss": 3.85305680999425, + "tokens_seen": 623509504 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006554222668004012, + "loss": 1.4961, + "theoretical_loss": 3.8530151485508872, + "tokens_seen": 623575040 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006554062186559679, + "loss": 1.5622, + "theoretical_loss": 3.85297349271163, + "tokens_seen": 623640576 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006553901705115346, + "loss": 1.501, + "theoretical_loss": 3.852931842475135, + "tokens_seen": 623706112 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006553741223671013, + "loss": 1.4444, + "theoretical_loss": 3.8528901978400603, + "tokens_seen": 623771648 + }, + { + "epoch": 0.19, + "learning_rate": 0.000655358074222668, + "loss": 1.4924, + "theoretical_loss": 3.852848558805065, + "tokens_seen": 623837184 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006553420260782347, + "loss": 1.4653, + "theoretical_loss": 3.8528069253688066, + "tokens_seen": 623902720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006553259779338014, + "loss": 1.4318, + "theoretical_loss": 3.852765297529945, + "tokens_seen": 623968256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006553099297893681, + "loss": 1.5847, + "theoretical_loss": 3.8527236752871397, + "tokens_seen": 624033792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006552938816449348, + "loss": 1.591, + "theoretical_loss": 3.8526820586390507, + "tokens_seen": 624099328 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006552778335005015, + "loss": 1.5047, + "theoretical_loss": 3.8526404475843385, + "tokens_seen": 624164864 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006552617853560682, + "loss": 1.5859, + "theoretical_loss": 3.8525988421216653, + "tokens_seen": 624230400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006552457372116349, + "loss": 1.4491, + "theoretical_loss": 3.852557242249691, + "tokens_seen": 624295936 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006552296890672016, + "loss": 1.619, + "theoretical_loss": 3.8525156479670786, + "tokens_seen": 624361472 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006552136409227683, + "loss": 1.5328, + "theoretical_loss": 3.8524740592724904, + "tokens_seen": 624427008 + }, + { + "epoch": 0.19, + "learning_rate": 0.000655197592778335, + "loss": 1.6493, + "theoretical_loss": 3.8524324761645894, + "tokens_seen": 624492544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006551815446339017, + "loss": 1.4805, + "theoretical_loss": 3.8523908986420383, + "tokens_seen": 624558080 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006551654964894684, + "loss": 1.51, + "theoretical_loss": 3.8523493267035014, + "tokens_seen": 624623616 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006551494483450351, + "loss": 1.4569, + "theoretical_loss": 3.852307760347643, + "tokens_seen": 624689152 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006551334002006018, + "loss": 1.5094, + "theoretical_loss": 3.852266199573128, + "tokens_seen": 624754688 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006551173520561685, + "loss": 1.5513, + "theoretical_loss": 3.852224644378621, + "tokens_seen": 624820224 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006551013039117352, + "loss": 1.531, + "theoretical_loss": 3.852183094762789, + "tokens_seen": 624885760 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006550852557673019, + "loss": 1.5476, + "theoretical_loss": 3.852141550724297, + "tokens_seen": 624951296 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006550692076228686, + "loss": 1.3544, + "theoretical_loss": 3.852100012261811, + "tokens_seen": 625016832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006550531594784353, + "loss": 1.4228, + "theoretical_loss": 3.8520584793739996, + "tokens_seen": 625082368 + }, + { + "epoch": 0.19, + "learning_rate": 0.000655037111334002, + "loss": 1.6692, + "theoretical_loss": 3.852016952059529, + "tokens_seen": 625147904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006550210631895687, + "loss": 1.4752, + "theoretical_loss": 3.8519754303170677, + "tokens_seen": 625213440 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006550050150451354, + "loss": 1.5447, + "theoretical_loss": 3.8519339141452846, + "tokens_seen": 625278976 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006549889669007021, + "loss": 1.4625, + "theoretical_loss": 3.8518924035428475, + "tokens_seen": 625344512 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006549729187562688, + "loss": 1.4502, + "theoretical_loss": 3.8518508985084265, + "tokens_seen": 625410048 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006549568706118355, + "loss": 1.5103, + "theoretical_loss": 3.8518093990406914, + "tokens_seen": 625475584 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006549408224674022, + "loss": 1.4846, + "theoretical_loss": 3.8517679051383116, + "tokens_seen": 625541120 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006549247743229689, + "loss": 1.5321, + "theoretical_loss": 3.851726416799959, + "tokens_seen": 625606656 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006549087261785357, + "loss": 1.4677, + "theoretical_loss": 3.8516849340243042, + "tokens_seen": 625672192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006548926780341023, + "loss": 1.4804, + "theoretical_loss": 3.8516434568100184, + "tokens_seen": 625737728 + }, + { + "epoch": 0.19, + "learning_rate": 0.000654876629889669, + "loss": 1.5789, + "theoretical_loss": 3.8516019851557743, + "tokens_seen": 625803264 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.9916975498199463, + "objective/train/avg_token_score": 0.008302461355924606, + "objective/train/docs_used": 233149, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.3803077936172485, + "objective/train/original_loss": 1.3918105363845825, + "objective/train/theoretical_loss": 3.8515605190602447, + "objective/train/tokens_used": 646328800, + "objective/train/unlikelihood_loss": 0.02204250358045101, + "theoretical_loss": 3.8515605190602447, + "tokens_seen": 625868800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006548605817452357, + "loss": 1.4793, + "theoretical_loss": 3.8515605190602447, + "tokens_seen": 625868800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006548445336008025, + "loss": 1.4245, + "theoretical_loss": 3.8515190585221015, + "tokens_seen": 625934336 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006548284854563691, + "loss": 1.5105, + "theoretical_loss": 3.851477603540019, + "tokens_seen": 625999872 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006548124373119359, + "loss": 1.5237, + "theoretical_loss": 3.8514361541126703, + "tokens_seen": 626065408 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006547963891675025, + "loss": 1.5314, + "theoretical_loss": 3.85139471023873, + "tokens_seen": 626130944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006547803410230693, + "loss": 1.5992, + "theoretical_loss": 3.8513532719168744, + "tokens_seen": 626196480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006547642928786359, + "loss": 1.5357, + "theoretical_loss": 3.851311839145777, + "tokens_seen": 626262016 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006547482447342027, + "loss": 1.5008, + "theoretical_loss": 3.851270411924114, + "tokens_seen": 626327552 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006547321965897693, + "loss": 1.6089, + "theoretical_loss": 3.8512289902505605, + "tokens_seen": 626393088 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006547161484453361, + "loss": 1.4183, + "theoretical_loss": 3.8511875741237955, + "tokens_seen": 626458624 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006547001003009027, + "loss": 1.6776, + "theoretical_loss": 3.851146163542495, + "tokens_seen": 626524160 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006546840521564695, + "loss": 1.4476, + "theoretical_loss": 3.8511047585053353, + "tokens_seen": 626589696 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006546680040120361, + "loss": 1.6197, + "theoretical_loss": 3.851063359010996, + "tokens_seen": 626655232 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006546519558676029, + "loss": 1.4348, + "theoretical_loss": 3.851021965058155, + "tokens_seen": 626720768 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006546359077231696, + "loss": 1.4409, + "theoretical_loss": 3.850980576645491, + "tokens_seen": 626786304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006546198595787363, + "loss": 1.435, + "theoretical_loss": 3.8509391937716835, + "tokens_seen": 626851840 + }, + { + "epoch": 0.19, + "learning_rate": 0.000654603811434303, + "loss": 1.5298, + "theoretical_loss": 3.8508978164354124, + "tokens_seen": 626917376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006545877632898697, + "loss": 1.5033, + "theoretical_loss": 3.8508564446353573, + "tokens_seen": 626982912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006545717151454364, + "loss": 1.6488, + "theoretical_loss": 3.8508150783702, + "tokens_seen": 627048448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006545556670010031, + "loss": 1.4554, + "theoretical_loss": 3.850773717638621, + "tokens_seen": 627113984 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006545396188565698, + "loss": 1.5769, + "theoretical_loss": 3.850732362439302, + "tokens_seen": 627179520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006545235707121365, + "loss": 1.6022, + "theoretical_loss": 3.850691012770925, + "tokens_seen": 627245056 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006545075225677032, + "loss": 1.4898, + "theoretical_loss": 3.8506496686321725, + "tokens_seen": 627310592 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006544914744232699, + "loss": 1.5573, + "theoretical_loss": 3.8506083300217275, + "tokens_seen": 627376128 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006544754262788366, + "loss": 1.517, + "theoretical_loss": 3.850566996938274, + "tokens_seen": 627441664 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006544593781344033, + "loss": 1.6332, + "theoretical_loss": 3.8505256693804943, + "tokens_seen": 627507200 + }, + { + "epoch": 0.19, + "learning_rate": 0.00065444332998997, + "loss": 1.5102, + "theoretical_loss": 3.8504843473470745, + "tokens_seen": 627572736 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006544272818455367, + "loss": 1.6071, + "theoretical_loss": 3.850443030836698, + "tokens_seen": 627638272 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006544112337011034, + "loss": 1.4109, + "theoretical_loss": 3.850401719848051, + "tokens_seen": 627703808 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006543951855566701, + "loss": 1.4579, + "theoretical_loss": 3.8503604143798187, + "tokens_seen": 627769344 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006543791374122368, + "loss": 1.5613, + "theoretical_loss": 3.850319114430687, + "tokens_seen": 627834880 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006543630892678035, + "loss": 1.439, + "theoretical_loss": 3.8502778199993433, + "tokens_seen": 627900416 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006543470411233702, + "loss": 1.5388, + "theoretical_loss": 3.850236531084473, + "tokens_seen": 627965952 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006543309929789369, + "loss": 1.417, + "theoretical_loss": 3.8501952476847654, + "tokens_seen": 628031488 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006543149448345035, + "loss": 1.5327, + "theoretical_loss": 3.8501539697989076, + "tokens_seen": 628097024 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006542988966900703, + "loss": 1.6308, + "theoretical_loss": 3.8501126974255877, + "tokens_seen": 628162560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006542828485456369, + "loss": 1.5548, + "theoretical_loss": 3.850071430563495, + "tokens_seen": 628228096 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006542668004012037, + "loss": 1.5047, + "theoretical_loss": 3.850030169211319, + "tokens_seen": 628293632 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006542507522567703, + "loss": 1.5442, + "theoretical_loss": 3.8499889133677483, + "tokens_seen": 628359168 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006542347041123371, + "loss": 1.4247, + "theoretical_loss": 3.8499476630314735, + "tokens_seen": 628424704 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006542186559679037, + "loss": 1.5409, + "theoretical_loss": 3.849906418201186, + "tokens_seen": 628490240 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006542026078234705, + "loss": 1.5181, + "theoretical_loss": 3.849865178875576, + "tokens_seen": 628555776 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006541865596790371, + "loss": 1.5731, + "theoretical_loss": 3.8498239450533354, + "tokens_seen": 628621312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006541705115346038, + "loss": 1.4387, + "theoretical_loss": 3.8497827167331558, + "tokens_seen": 628686848 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006541544633901705, + "loss": 1.4037, + "theoretical_loss": 3.84974149391373, + "tokens_seen": 628752384 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006541384152457372, + "loss": 1.4231, + "theoretical_loss": 3.849700276593751, + "tokens_seen": 628817920 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006541223671013039, + "loss": 1.4615, + "theoretical_loss": 3.8496590647719113, + "tokens_seen": 628883456 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006541063189568706, + "loss": 1.5872, + "theoretical_loss": 3.849617858446905, + "tokens_seen": 628948992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006540902708124373, + "loss": 1.5561, + "theoretical_loss": 3.8495766576174266, + "tokens_seen": 629014528 + }, + { + "epoch": 0.19, + "learning_rate": 0.000654074222668004, + "loss": 1.5334, + "theoretical_loss": 3.8495354622821703, + "tokens_seen": 629080064 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.9909608364105225, + "objective/train/avg_token_score": 0.00903921015560627, + "objective/train/docs_used": 234355, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4052876234054565, + "objective/train/original_loss": 1.4209628105163574, + "objective/train/theoretical_loss": 3.8494942724398316, + "objective/train/tokens_used": 649605600, + "objective/train/unlikelihood_loss": 0.021053869277238846, + "theoretical_loss": 3.8494942724398316, + "tokens_seen": 629145600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006540581745235707, + "loss": 1.438, + "theoretical_loss": 3.8494942724398316, + "tokens_seen": 629145600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006540421263791374, + "loss": 1.499, + "theoretical_loss": 3.8494530880891054, + "tokens_seen": 629211136 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006540260782347041, + "loss": 1.5368, + "theoretical_loss": 3.8494119092286887, + "tokens_seen": 629276672 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006540100300902708, + "loss": 1.5681, + "theoretical_loss": 3.849370735857277, + "tokens_seen": 629342208 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006539939819458375, + "loss": 1.4694, + "theoretical_loss": 3.849329567973567, + "tokens_seen": 629407744 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006539779338014042, + "loss": 1.5845, + "theoretical_loss": 3.8492884055762566, + "tokens_seen": 629473280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006539618856569709, + "loss": 1.4375, + "theoretical_loss": 3.849247248664043, + "tokens_seen": 629538816 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006539458375125376, + "loss": 1.4016, + "theoretical_loss": 3.8492060972356255, + "tokens_seen": 629604352 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006539297893681043, + "loss": 1.409, + "theoretical_loss": 3.8491649512897017, + "tokens_seen": 629669888 + }, + { + "epoch": 0.19, + "learning_rate": 0.000653913741223671, + "loss": 1.5469, + "theoretical_loss": 3.849123810824971, + "tokens_seen": 629735424 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006538976930792377, + "loss": 1.5725, + "theoretical_loss": 3.8490826758401324, + "tokens_seen": 629800960 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006538816449348044, + "loss": 1.5907, + "theoretical_loss": 3.849041546333887, + "tokens_seen": 629866496 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006538655967903711, + "loss": 1.4548, + "theoretical_loss": 3.8490004223049343, + "tokens_seen": 629932032 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006538495486459378, + "loss": 1.4952, + "theoretical_loss": 3.8489593037519754, + "tokens_seen": 629997568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006538335005015045, + "loss": 1.4702, + "theoretical_loss": 3.8489181906737118, + "tokens_seen": 630063104 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006538174523570712, + "loss": 1.5317, + "theoretical_loss": 3.848877083068845, + "tokens_seen": 630128640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006538014042126379, + "loss": 1.5795, + "theoretical_loss": 3.8488359809360766, + "tokens_seen": 630194176 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006537853560682046, + "loss": 1.5506, + "theoretical_loss": 3.8487948842741107, + "tokens_seen": 630259712 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006537693079237713, + "loss": 1.5084, + "theoretical_loss": 3.8487537930816487, + "tokens_seen": 630325248 + }, + { + "epoch": 0.19, + "learning_rate": 0.000653753259779338, + "loss": 1.5817, + "theoretical_loss": 3.8487127073573957, + "tokens_seen": 630390784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006537372116349047, + "loss": 1.5869, + "theoretical_loss": 3.8486716271000545, + "tokens_seen": 630456320 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006537211634904714, + "loss": 1.4859, + "theoretical_loss": 3.8486305523083297, + "tokens_seen": 630521856 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006537051153460381, + "loss": 1.5866, + "theoretical_loss": 3.848589482980927, + "tokens_seen": 630587392 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006536890672016048, + "loss": 1.5059, + "theoretical_loss": 3.8485484191165504, + "tokens_seen": 630652928 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006536730190571715, + "loss": 1.5299, + "theoretical_loss": 3.8485073607139064, + "tokens_seen": 630718464 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006536569709127382, + "loss": 1.487, + "theoretical_loss": 3.8484663077717007, + "tokens_seen": 630784000 + }, + { + "epoch": 0.19, + "learning_rate": 0.000653640922768305, + "loss": 1.4084, + "theoretical_loss": 3.84842526028864, + "tokens_seen": 630849536 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006536248746238716, + "loss": 1.6476, + "theoretical_loss": 3.8483842182634325, + "tokens_seen": 630915072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006536088264794383, + "loss": 1.4979, + "theoretical_loss": 3.848343181694784, + "tokens_seen": 630980608 + }, + { + "epoch": 0.19, + "learning_rate": 0.000653592778335005, + "loss": 1.4032, + "theoretical_loss": 3.8483021505814032, + "tokens_seen": 631046144 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006535767301905717, + "loss": 1.5129, + "theoretical_loss": 3.8482611249219985, + "tokens_seen": 631111680 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006535606820461384, + "loss": 1.554, + "theoretical_loss": 3.8482201047152786, + "tokens_seen": 631177216 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006535446339017052, + "loss": 1.5008, + "theoretical_loss": 3.8481790899599524, + "tokens_seen": 631242752 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006535285857572718, + "loss": 1.597, + "theoretical_loss": 3.84813808065473, + "tokens_seen": 631308288 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006535125376128386, + "loss": 1.5175, + "theoretical_loss": 3.8480970767983216, + "tokens_seen": 631373824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006534964894684052, + "loss": 1.5901, + "theoretical_loss": 3.8480560783894373, + "tokens_seen": 631439360 + }, + { + "epoch": 0.19, + "learning_rate": 0.000653480441323972, + "loss": 1.5713, + "theoretical_loss": 3.8480150854267885, + "tokens_seen": 631504896 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006534643931795386, + "loss": 1.4713, + "theoretical_loss": 3.847974097909087, + "tokens_seen": 631570432 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006534483450351054, + "loss": 1.4026, + "theoretical_loss": 3.8479331158350436, + "tokens_seen": 631635968 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006534322968906721, + "loss": 1.4434, + "theoretical_loss": 3.8478921392033714, + "tokens_seen": 631701504 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006534162487462388, + "loss": 1.4404, + "theoretical_loss": 3.8478511680127827, + "tokens_seen": 631767040 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006534002006018055, + "loss": 1.5431, + "theoretical_loss": 3.847810202261991, + "tokens_seen": 631832576 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006533841524573722, + "loss": 1.6527, + "theoretical_loss": 3.8477692419497105, + "tokens_seen": 631898112 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006533681043129389, + "loss": 1.4898, + "theoretical_loss": 3.847728287074654, + "tokens_seen": 631963648 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006533520561685056, + "loss": 1.5432, + "theoretical_loss": 3.847687337635537, + "tokens_seen": 632029184 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006533360080240723, + "loss": 1.5141, + "theoretical_loss": 3.8476463936310736, + "tokens_seen": 632094720 + }, + { + "epoch": 0.19, + "learning_rate": 0.000653319959879639, + "loss": 1.5637, + "theoretical_loss": 3.8476054550599796, + "tokens_seen": 632160256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006533039117352057, + "loss": 1.5922, + "theoretical_loss": 3.8475645219209715, + "tokens_seen": 632225792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006532878635907724, + "loss": 1.5356, + "theoretical_loss": 3.847523594212765, + "tokens_seen": 632291328 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006532718154463391, + "loss": 1.4748, + "theoretical_loss": 3.8474826719340762, + "tokens_seen": 632356864 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.992222249507904, + "objective/train/avg_token_score": 0.007777766324579716, + "objective/train/docs_used": 235545, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.7982854843139648, + "objective/train/original_loss": 1.8125319480895996, + "objective/train/theoretical_loss": 3.847441755083623, + "objective/train/tokens_used": 652882400, + "objective/train/unlikelihood_loss": 0.014013963751494884, + "theoretical_loss": 3.847441755083623, + "tokens_seen": 632422400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006532557673019058, + "loss": 1.6707, + "theoretical_loss": 3.847441755083623, + "tokens_seen": 632422400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006532397191574725, + "loss": 1.5536, + "theoretical_loss": 3.8474008436601226, + "tokens_seen": 632487936 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006532236710130392, + "loss": 1.5419, + "theoretical_loss": 3.8473599376622927, + "tokens_seen": 632553472 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006532076228686059, + "loss": 1.4366, + "theoretical_loss": 3.8473190370888526, + "tokens_seen": 632619008 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006531915747241726, + "loss": 1.4778, + "theoretical_loss": 3.8472781419385207, + "tokens_seen": 632684544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006531755265797393, + "loss": 1.4898, + "theoretical_loss": 3.8472372522100162, + "tokens_seen": 632750080 + }, + { + "epoch": 0.19, + "learning_rate": 0.000653159478435306, + "loss": 1.419, + "theoretical_loss": 3.8471963679020593, + "tokens_seen": 632815616 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006531434302908727, + "loss": 1.5401, + "theoretical_loss": 3.8471554890133692, + "tokens_seen": 632881152 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006531273821464394, + "loss": 1.4624, + "theoretical_loss": 3.847114615542667, + "tokens_seen": 632946688 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006531113340020061, + "loss": 1.5173, + "theoretical_loss": 3.8470737474886745, + "tokens_seen": 633012224 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006530952858575728, + "loss": 1.4458, + "theoretical_loss": 3.847032884850112, + "tokens_seen": 633077760 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006530792377131395, + "loss": 1.5311, + "theoretical_loss": 3.846992027625702, + "tokens_seen": 633143296 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006530631895687062, + "loss": 1.482, + "theoretical_loss": 3.8469511758141666, + "tokens_seen": 633208832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006530471414242729, + "loss": 1.4501, + "theoretical_loss": 3.8469103294142286, + "tokens_seen": 633274368 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006530310932798396, + "loss": 1.4877, + "theoretical_loss": 3.8468694884246117, + "tokens_seen": 633339904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006530150451354063, + "loss": 1.4501, + "theoretical_loss": 3.8468286528440396, + "tokens_seen": 633405440 + }, + { + "epoch": 0.19, + "learning_rate": 0.000652998996990973, + "loss": 1.4594, + "theoretical_loss": 3.846787822671235, + "tokens_seen": 633470976 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006529829488465397, + "loss": 1.4736, + "theoretical_loss": 3.8467469979049245, + "tokens_seen": 633536512 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006529669007021064, + "loss": 1.4843, + "theoretical_loss": 3.8467061785438315, + "tokens_seen": 633602048 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006529508525576731, + "loss": 1.4206, + "theoretical_loss": 3.8466653645866815, + "tokens_seen": 633667584 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006529348044132398, + "loss": 1.5483, + "theoretical_loss": 3.8466245560322005, + "tokens_seen": 633733120 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006529187562688064, + "loss": 1.5672, + "theoretical_loss": 3.8465837528791154, + "tokens_seen": 633798656 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006529027081243732, + "loss": 1.5124, + "theoretical_loss": 3.846542955126152, + "tokens_seen": 633864192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006528866599799398, + "loss": 1.527, + "theoretical_loss": 3.8465021627720377, + "tokens_seen": 633929728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006528706118355066, + "loss": 1.611, + "theoretical_loss": 3.8464613758155, + "tokens_seen": 633995264 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006528545636910732, + "loss": 1.5825, + "theoretical_loss": 3.8464205942552665, + "tokens_seen": 634060800 + }, + { + "epoch": 0.19, + "learning_rate": 0.00065283851554664, + "loss": 1.5978, + "theoretical_loss": 3.8463798180900666, + "tokens_seen": 634126336 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006528224674022066, + "loss": 1.4995, + "theoretical_loss": 3.8463390473186285, + "tokens_seen": 634191872 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006528064192577734, + "loss": 1.4948, + "theoretical_loss": 3.8462982819396814, + "tokens_seen": 634257408 + }, + { + "epoch": 0.19, + "learning_rate": 0.00065279037111334, + "loss": 1.4537, + "theoretical_loss": 3.846257521951955, + "tokens_seen": 634322944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006527743229689067, + "loss": 1.4884, + "theoretical_loss": 3.84621676735418, + "tokens_seen": 634388480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006527582748244734, + "loss": 1.5841, + "theoretical_loss": 3.8461760181450857, + "tokens_seen": 634454016 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006527422266800401, + "loss": 1.4506, + "theoretical_loss": 3.8461352743234043, + "tokens_seen": 634519552 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006527261785356068, + "loss": 1.5113, + "theoretical_loss": 3.8460945358878664, + "tokens_seen": 634585088 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006527101303911735, + "loss": 1.5674, + "theoretical_loss": 3.846053802837204, + "tokens_seen": 634650624 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006526940822467402, + "loss": 1.5244, + "theoretical_loss": 3.8460130751701502, + "tokens_seen": 634716160 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006526780341023069, + "loss": 1.4745, + "theoretical_loss": 3.845972352885437, + "tokens_seen": 634781696 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006526619859578736, + "loss": 1.5003, + "theoretical_loss": 3.845931635981797, + "tokens_seen": 634847232 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006526459378134403, + "loss": 1.4897, + "theoretical_loss": 3.845890924457965, + "tokens_seen": 634912768 + }, + { + "epoch": 0.19, + "learning_rate": 0.000652629889669007, + "loss": 1.4534, + "theoretical_loss": 3.845850218312674, + "tokens_seen": 634978304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006526138415245737, + "loss": 1.6023, + "theoretical_loss": 3.8458095175446587, + "tokens_seen": 635043840 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006525977933801404, + "loss": 1.5018, + "theoretical_loss": 3.8457688221526545, + "tokens_seen": 635109376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006525817452357071, + "loss": 1.4859, + "theoretical_loss": 3.8457281321353953, + "tokens_seen": 635174912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006525656970912738, + "loss": 1.4811, + "theoretical_loss": 3.8456874474916183, + "tokens_seen": 635240448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006525496489468405, + "loss": 1.4795, + "theoretical_loss": 3.8456467682200586, + "tokens_seen": 635305984 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006525336008024072, + "loss": 1.5307, + "theoretical_loss": 3.8456060943194537, + "tokens_seen": 635371520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006525175526579739, + "loss": 1.4666, + "theoretical_loss": 3.845565425788539, + "tokens_seen": 635437056 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006525015045135406, + "loss": 1.5134, + "theoretical_loss": 3.8455247626260536, + "tokens_seen": 635502592 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006524854563691073, + "loss": 1.5811, + "theoretical_loss": 3.845484104830734, + "tokens_seen": 635568128 + }, + { + "epoch": 0.19, + "learning_rate": 0.000652469408224674, + "loss": 1.4893, + "theoretical_loss": 3.84544345240132, + "tokens_seen": 635633664 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.992943286895752, + "objective/train/avg_token_score": 0.007056668866425753, + "objective/train/docs_used": 236526, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.532179355621338, + "objective/train/original_loss": 1.5446350574493408, + "objective/train/theoretical_loss": 3.8454028053365494, + "objective/train/tokens_used": 656159200, + "objective/train/unlikelihood_loss": 0.014889092184603214, + "theoretical_loss": 3.8454028053365494, + "tokens_seen": 635699200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006524533600802407, + "loss": 1.5061, + "theoretical_loss": 3.8454028053365494, + "tokens_seen": 635699200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006524373119358074, + "loss": 1.5357, + "theoretical_loss": 3.8453621636351607, + "tokens_seen": 635764736 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006524212637913741, + "loss": 1.4836, + "theoretical_loss": 3.8453215272958943, + "tokens_seen": 635830272 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006524052156469408, + "loss": 1.5376, + "theoretical_loss": 3.84528089631749, + "tokens_seen": 635895808 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006523891675025075, + "loss": 1.4259, + "theoretical_loss": 3.845240270698688, + "tokens_seen": 635961344 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006523731193580742, + "loss": 1.5008, + "theoretical_loss": 3.845199650438229, + "tokens_seen": 636026880 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006523570712136409, + "loss": 1.5225, + "theoretical_loss": 3.845159035534855, + "tokens_seen": 636092416 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006523410230692076, + "loss": 1.5629, + "theoretical_loss": 3.845118425987307, + "tokens_seen": 636157952 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006523249749247743, + "loss": 1.5629, + "theoretical_loss": 3.845077821794327, + "tokens_seen": 636223488 + }, + { + "epoch": 0.19, + "learning_rate": 0.000652308926780341, + "loss": 1.5816, + "theoretical_loss": 3.845037222954658, + "tokens_seen": 636289024 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006522928786359077, + "loss": 1.5572, + "theoretical_loss": 3.8449966294670426, + "tokens_seen": 636354560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006522768304914744, + "loss": 1.5288, + "theoretical_loss": 3.8449560413302244, + "tokens_seen": 636420096 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006522607823470411, + "loss": 1.6076, + "theoretical_loss": 3.844915458542947, + "tokens_seen": 636485632 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006522447342026078, + "loss": 1.4789, + "theoretical_loss": 3.844874881103955, + "tokens_seen": 636551168 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006522286860581746, + "loss": 1.4558, + "theoretical_loss": 3.844834309011992, + "tokens_seen": 636616704 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006522126379137412, + "loss": 1.4559, + "theoretical_loss": 3.8447937422658045, + "tokens_seen": 636682240 + }, + { + "epoch": 0.19, + "learning_rate": 0.000652196589769308, + "loss": 1.4881, + "theoretical_loss": 3.8447531808641373, + "tokens_seen": 636747776 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006521805416248747, + "loss": 1.4963, + "theoretical_loss": 3.844712624805736, + "tokens_seen": 636813312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006521644934804414, + "loss": 1.7528, + "theoretical_loss": 3.844672074089347, + "tokens_seen": 636878848 + }, + { + "epoch": 0.19, + "learning_rate": 0.000652148445336008, + "loss": 1.4656, + "theoretical_loss": 3.844631528713718, + "tokens_seen": 636944384 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006521323971915748, + "loss": 1.5486, + "theoretical_loss": 3.844590988677595, + "tokens_seen": 637009920 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006521163490471415, + "loss": 1.4505, + "theoretical_loss": 3.844550453979726, + "tokens_seen": 637075456 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006521003009027082, + "loss": 1.5364, + "theoretical_loss": 3.8445099246188597, + "tokens_seen": 637140992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006520842527582749, + "loss": 1.4353, + "theoretical_loss": 3.8444694005937436, + "tokens_seen": 637206528 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006520682046138416, + "loss": 1.5914, + "theoretical_loss": 3.844428881903127, + "tokens_seen": 637272064 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006520521564694083, + "loss": 1.4987, + "theoretical_loss": 3.844388368545759, + "tokens_seen": 637337600 + }, + { + "epoch": 0.19, + "learning_rate": 0.000652036108324975, + "loss": 1.4793, + "theoretical_loss": 3.84434786052039, + "tokens_seen": 637403136 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006520200601805417, + "loss": 1.4935, + "theoretical_loss": 3.8443073578257687, + "tokens_seen": 637468672 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006520040120361084, + "loss": 1.4331, + "theoretical_loss": 3.8442668604606474, + "tokens_seen": 637534208 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006519879638916751, + "loss": 1.5144, + "theoretical_loss": 3.844226368423776, + "tokens_seen": 637599744 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006519719157472418, + "loss": 1.3716, + "theoretical_loss": 3.844185881713906, + "tokens_seen": 637665280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006519558676028085, + "loss": 1.5019, + "theoretical_loss": 3.844145400329789, + "tokens_seen": 637730816 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006519398194583752, + "loss": 1.513, + "theoretical_loss": 3.8441049242701784, + "tokens_seen": 637796352 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006519237713139419, + "loss": 1.5248, + "theoretical_loss": 3.8440644535338255, + "tokens_seen": 637861888 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006519077231695086, + "loss": 1.5013, + "theoretical_loss": 3.844023988119484, + "tokens_seen": 637927424 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006518916750250753, + "loss": 1.4765, + "theoretical_loss": 3.8439835280259076, + "tokens_seen": 637992960 + }, + { + "epoch": 0.19, + "learning_rate": 0.000651875626880642, + "loss": 1.5139, + "theoretical_loss": 3.84394307325185, + "tokens_seen": 638058496 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006518595787362087, + "loss": 1.6049, + "theoretical_loss": 3.8439026237960654, + "tokens_seen": 638124032 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006518435305917754, + "loss": 1.4915, + "theoretical_loss": 3.8438621796573087, + "tokens_seen": 638189568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006518274824473421, + "loss": 1.4309, + "theoretical_loss": 3.843821740834335, + "tokens_seen": 638255104 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006518114343029088, + "loss": 1.4038, + "theoretical_loss": 3.8437813073258997, + "tokens_seen": 638320640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006517953861584755, + "loss": 1.4976, + "theoretical_loss": 3.8437408791307597, + "tokens_seen": 638386176 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006517793380140422, + "loss": 1.5433, + "theoretical_loss": 3.843700456247671, + "tokens_seen": 638451712 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006517632898696089, + "loss": 1.5101, + "theoretical_loss": 3.84366003867539, + "tokens_seen": 638517248 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006517472417251756, + "loss": 1.5875, + "theoretical_loss": 3.8436196264126736, + "tokens_seen": 638582784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006517311935807423, + "loss": 1.5347, + "theoretical_loss": 3.843579219458281, + "tokens_seen": 638648320 + }, + { + "epoch": 0.19, + "learning_rate": 0.000651715145436309, + "loss": 1.6232, + "theoretical_loss": 3.843538817810969, + "tokens_seen": 638713856 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006516990972918757, + "loss": 1.5511, + "theoretical_loss": 3.843498421469497, + "tokens_seen": 638779392 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006516830491474424, + "loss": 1.4648, + "theoretical_loss": 3.843458030432623, + "tokens_seen": 638844928 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006516670010030091, + "loss": 1.5106, + "theoretical_loss": 3.8434176446991075, + "tokens_seen": 638910464 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.9923362731933594, + "objective/train/avg_token_score": 0.007663711439818144, + "objective/train/docs_used": 237858, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.3268431425094604, + "objective/train/original_loss": 1.3387027978897095, + "objective/train/theoretical_loss": 3.8433772642677093, + "objective/train/tokens_used": 659436000, + "objective/train/unlikelihood_loss": 0.022634392604231834, + "theoretical_loss": 3.8433772642677093, + "tokens_seen": 638976000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006516509528585758, + "loss": 1.293, + "theoretical_loss": 3.8433772642677093, + "tokens_seen": 638976000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006516349047141425, + "loss": 1.5248, + "theoretical_loss": 3.8433368891371895, + "tokens_seen": 639041536 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006516188565697092, + "loss": 1.5635, + "theoretical_loss": 3.843296519306308, + "tokens_seen": 639107072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006516028084252759, + "loss": 1.4865, + "theoretical_loss": 3.843256154773826, + "tokens_seen": 639172608 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006515867602808426, + "loss": 1.4952, + "theoretical_loss": 3.843215795538505, + "tokens_seen": 639238144 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006515707121364093, + "loss": 1.5588, + "theoretical_loss": 3.843175441599107, + "tokens_seen": 639303680 + }, + { + "epoch": 0.19, + "learning_rate": 0.000651554663991976, + "loss": 1.6693, + "theoretical_loss": 3.8431350929543946, + "tokens_seen": 639369216 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006515386158475427, + "loss": 1.49, + "theoretical_loss": 3.8430947496031296, + "tokens_seen": 639434752 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006515225677031094, + "loss": 1.3933, + "theoretical_loss": 3.8430544115440752, + "tokens_seen": 639500288 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006515065195586761, + "loss": 1.5138, + "theoretical_loss": 3.8430140787759957, + "tokens_seen": 639565824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006514904714142427, + "loss": 1.5336, + "theoretical_loss": 3.842973751297655, + "tokens_seen": 639631360 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006514744232698095, + "loss": 1.4982, + "theoretical_loss": 3.8429334291078168, + "tokens_seen": 639696896 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006514583751253761, + "loss": 1.4941, + "theoretical_loss": 3.842893112205246, + "tokens_seen": 639762432 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006514423269809429, + "loss": 1.5082, + "theoretical_loss": 3.842852800588709, + "tokens_seen": 639827968 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006514262788365095, + "loss": 1.486, + "theoretical_loss": 3.8428124942569695, + "tokens_seen": 639893504 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006514102306920763, + "loss": 1.4646, + "theoretical_loss": 3.842772193208795, + "tokens_seen": 639959040 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006513941825476429, + "loss": 1.4561, + "theoretical_loss": 3.8427318974429507, + "tokens_seen": 640024576 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006513781344032097, + "loss": 1.4388, + "theoretical_loss": 3.8426916069582044, + "tokens_seen": 640090112 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006513620862587763, + "loss": 1.5242, + "theoretical_loss": 3.842651321753323, + "tokens_seen": 640155648 + }, + { + "epoch": 0.19, + "learning_rate": 0.000651346038114343, + "loss": 1.5806, + "theoretical_loss": 3.8426110418270745, + "tokens_seen": 640221184 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006513299899699097, + "loss": 1.6208, + "theoretical_loss": 3.842570767178227, + "tokens_seen": 640286720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006513139418254764, + "loss": 1.4436, + "theoretical_loss": 3.8425304978055483, + "tokens_seen": 640352256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006512978936810431, + "loss": 1.5294, + "theoretical_loss": 3.8424902337078084, + "tokens_seen": 640417792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006512818455366098, + "loss": 1.5912, + "theoretical_loss": 3.8424499748837753, + "tokens_seen": 640483328 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006512657973921765, + "loss": 1.4521, + "theoretical_loss": 3.8424097213322197, + "tokens_seen": 640548864 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006512497492477432, + "loss": 1.4134, + "theoretical_loss": 3.8423694730519116, + "tokens_seen": 640614400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006512337011033099, + "loss": 1.559, + "theoretical_loss": 3.8423292300416216, + "tokens_seen": 640679936 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006512176529588766, + "loss": 1.4959, + "theoretical_loss": 3.8422889923001207, + "tokens_seen": 640745472 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006512016048144433, + "loss": 1.4898, + "theoretical_loss": 3.84224875982618, + "tokens_seen": 640811008 + }, + { + "epoch": 0.19, + "learning_rate": 0.00065118555667001, + "loss": 1.373, + "theoretical_loss": 3.8422085326185718, + "tokens_seen": 640876544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006511695085255767, + "loss": 1.5806, + "theoretical_loss": 3.842168310676067, + "tokens_seen": 640942080 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006511534603811434, + "loss": 1.4806, + "theoretical_loss": 3.8421280939974407, + "tokens_seen": 641007616 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006511374122367101, + "loss": 1.5687, + "theoretical_loss": 3.8420878825814637, + "tokens_seen": 641073152 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006511213640922768, + "loss": 1.5539, + "theoretical_loss": 3.84204767642691, + "tokens_seen": 641138688 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006511053159478435, + "loss": 1.4957, + "theoretical_loss": 3.8420074755325544, + "tokens_seen": 641204224 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006510892678034102, + "loss": 1.5162, + "theoretical_loss": 3.8419672798971707, + "tokens_seen": 641269760 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006510732196589769, + "loss": 1.5377, + "theoretical_loss": 3.8419270895195323, + "tokens_seen": 641335296 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006510571715145436, + "loss": 1.5097, + "theoretical_loss": 3.8418869043984163, + "tokens_seen": 641400832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006510411233701103, + "loss": 1.5426, + "theoretical_loss": 3.841846724532597, + "tokens_seen": 641466368 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006510250752256771, + "loss": 1.4946, + "theoretical_loss": 3.8418065499208502, + "tokens_seen": 641531904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006510090270812437, + "loss": 1.4872, + "theoretical_loss": 3.8417663805619533, + "tokens_seen": 641597440 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006509929789368105, + "loss": 1.4478, + "theoretical_loss": 3.841726216454682, + "tokens_seen": 641662976 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006509769307923771, + "loss": 1.5853, + "theoretical_loss": 3.8416860575978142, + "tokens_seen": 641728512 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006509608826479439, + "loss": 1.4271, + "theoretical_loss": 3.841645903990127, + "tokens_seen": 641794048 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006509448345035105, + "loss": 1.4481, + "theoretical_loss": 3.841605755630398, + "tokens_seen": 641859584 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006509287863590773, + "loss": 1.5984, + "theoretical_loss": 3.8415656125174067, + "tokens_seen": 641925120 + }, + { + "epoch": 0.19, + "learning_rate": 0.000650912738214644, + "loss": 1.4523, + "theoretical_loss": 3.8415254746499308, + "tokens_seen": 641990656 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006508966900702107, + "loss": 1.4699, + "theoretical_loss": 3.8414853420267496, + "tokens_seen": 642056192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006508806419257773, + "loss": 1.3903, + "theoretical_loss": 3.841445214646644, + "tokens_seen": 642121728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006508645937813441, + "loss": 1.5122, + "theoretical_loss": 3.8414050925083925, + "tokens_seen": 642187264 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.99311363697052, + "objective/train/avg_token_score": 0.006886334158480167, + "objective/train/docs_used": 239059, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5122712850570679, + "objective/train/original_loss": 1.5235824584960938, + "objective/train/theoretical_loss": 3.8413649756107757, + "objective/train/tokens_used": 662712800, + "objective/train/unlikelihood_loss": 0.01607191376388073, + "theoretical_loss": 3.8413649756107757, + "tokens_seen": 642252800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006508485456369107, + "loss": 1.618, + "theoretical_loss": 3.8413649756107757, + "tokens_seen": 642252800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006508324974924775, + "loss": 1.5039, + "theoretical_loss": 3.8413248639525754, + "tokens_seen": 642318336 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006508164493480441, + "loss": 1.5195, + "theoretical_loss": 3.8412847575325717, + "tokens_seen": 642383872 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006508004012036109, + "loss": 1.631, + "theoretical_loss": 3.8412446563495477, + "tokens_seen": 642449408 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006507843530591776, + "loss": 1.6347, + "theoretical_loss": 3.8412045604022835, + "tokens_seen": 642514944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006507683049147443, + "loss": 1.4496, + "theoretical_loss": 3.8411644696895633, + "tokens_seen": 642580480 + }, + { + "epoch": 0.19, + "learning_rate": 0.000650752256770311, + "loss": 1.4561, + "theoretical_loss": 3.8411243842101688, + "tokens_seen": 642646016 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006507362086258777, + "loss": 1.5429, + "theoretical_loss": 3.841084303962884, + "tokens_seen": 642711552 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006507201604814444, + "loss": 1.5013, + "theoretical_loss": 3.841044228946492, + "tokens_seen": 642777088 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006507041123370111, + "loss": 1.4413, + "theoretical_loss": 3.8410041591597777, + "tokens_seen": 642842624 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006506880641925778, + "loss": 1.4806, + "theoretical_loss": 3.840964094601525, + "tokens_seen": 642908160 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006506720160481445, + "loss": 1.5317, + "theoretical_loss": 3.8409240352705187, + "tokens_seen": 642973696 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006506559679037112, + "loss": 1.4883, + "theoretical_loss": 3.840883981165544, + "tokens_seen": 643039232 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006506399197592779, + "loss": 1.5829, + "theoretical_loss": 3.8408439322853876, + "tokens_seen": 643104768 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006506238716148446, + "loss": 1.4269, + "theoretical_loss": 3.8408038886288347, + "tokens_seen": 643170304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006506078234704113, + "loss": 1.5372, + "theoretical_loss": 3.840763850194672, + "tokens_seen": 643235840 + }, + { + "epoch": 0.19, + "learning_rate": 0.000650591775325978, + "loss": 1.5541, + "theoretical_loss": 3.8407238169816864, + "tokens_seen": 643301376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006505757271815447, + "loss": 1.5135, + "theoretical_loss": 3.840683788988666, + "tokens_seen": 643366912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006505596790371114, + "loss": 1.4706, + "theoretical_loss": 3.840643766214397, + "tokens_seen": 643432448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006505436308926781, + "loss": 1.4738, + "theoretical_loss": 3.840603748657669, + "tokens_seen": 643497984 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006505275827482448, + "loss": 1.396, + "theoretical_loss": 3.8405637363172693, + "tokens_seen": 643563520 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006505115346038115, + "loss": 1.3846, + "theoretical_loss": 3.8405237291919883, + "tokens_seen": 643629056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006504954864593782, + "loss": 1.4948, + "theoretical_loss": 3.8404837272806143, + "tokens_seen": 643694592 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006504794383149449, + "loss": 1.5004, + "theoretical_loss": 3.840443730581937, + "tokens_seen": 643760128 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006504633901705116, + "loss": 1.52, + "theoretical_loss": 3.8404037390947474, + "tokens_seen": 643825664 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006504473420260783, + "loss": 1.4566, + "theoretical_loss": 3.8403637528178356, + "tokens_seen": 643891200 + }, + { + "epoch": 0.2, + "learning_rate": 0.000650431293881645, + "loss": 1.483, + "theoretical_loss": 3.8403237717499925, + "tokens_seen": 643956736 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006504152457372117, + "loss": 1.5119, + "theoretical_loss": 3.8402837958900093, + "tokens_seen": 644022272 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006503991975927784, + "loss": 1.4856, + "theoretical_loss": 3.840243825236678, + "tokens_seen": 644087808 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006503831494483451, + "loss": 1.457, + "theoretical_loss": 3.840203859788791, + "tokens_seen": 644153344 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006503671013039118, + "loss": 1.4476, + "theoretical_loss": 3.840163899545141, + "tokens_seen": 644218880 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006503510531594785, + "loss": 1.5144, + "theoretical_loss": 3.8401239445045205, + "tokens_seen": 644284416 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006503350050150452, + "loss": 1.4867, + "theoretical_loss": 3.840083994665723, + "tokens_seen": 644349952 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006503189568706119, + "loss": 1.4447, + "theoretical_loss": 3.840044050027543, + "tokens_seen": 644415488 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006503029087261786, + "loss": 1.4681, + "theoretical_loss": 3.8400041105887732, + "tokens_seen": 644481024 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006502868605817453, + "loss": 1.593, + "theoretical_loss": 3.8399641763482095, + "tokens_seen": 644546560 + }, + { + "epoch": 0.2, + "learning_rate": 0.000650270812437312, + "loss": 1.5381, + "theoretical_loss": 3.8399242473046464, + "tokens_seen": 644612096 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006502547642928787, + "loss": 1.5307, + "theoretical_loss": 3.8398843234568796, + "tokens_seen": 644677632 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006502387161484454, + "loss": 1.5147, + "theoretical_loss": 3.8398444048037046, + "tokens_seen": 644743168 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006502226680040121, + "loss": 1.5018, + "theoretical_loss": 3.8398044913439175, + "tokens_seen": 644808704 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006502066198595788, + "loss": 1.452, + "theoretical_loss": 3.839764583076316, + "tokens_seen": 644874240 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006501905717151455, + "loss": 1.5739, + "theoretical_loss": 3.839724679999695, + "tokens_seen": 644939776 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006501745235707122, + "loss": 1.5844, + "theoretical_loss": 3.8396847821128537, + "tokens_seen": 645005312 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006501584754262789, + "loss": 1.62, + "theoretical_loss": 3.83964488941459, + "tokens_seen": 645070848 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006501424272818456, + "loss": 1.5103, + "theoretical_loss": 3.8396050019037005, + "tokens_seen": 645136384 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006501263791374123, + "loss": 1.4903, + "theoretical_loss": 3.8395651195789853, + "tokens_seen": 645201920 + }, + { + "epoch": 0.2, + "learning_rate": 0.000650110330992979, + "loss": 1.5237, + "theoretical_loss": 3.839525242439243, + "tokens_seen": 645267456 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006500942828485456, + "loss": 1.5056, + "theoretical_loss": 3.8394853704832723, + "tokens_seen": 645332992 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006500782347041124, + "loss": 1.5531, + "theoretical_loss": 3.8394455037098743, + "tokens_seen": 645398528 + }, + { + "epoch": 0.2, + "learning_rate": 0.000650062186559679, + "loss": 1.5179, + "theoretical_loss": 3.8394056421178484, + "tokens_seen": 645464064 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.9884245991706848, + "objective/train/avg_token_score": 0.011575404554605484, + "objective/train/docs_used": 240316, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5184258222579956, + "objective/train/original_loss": 1.5392799377441406, + "objective/train/theoretical_loss": 3.8393657857059953, + "objective/train/tokens_used": 665989600, + "objective/train/unlikelihood_loss": 0.025043992325663567, + "theoretical_loss": 3.8393657857059953, + "tokens_seen": 645529600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006500461384152458, + "loss": 1.4754, + "theoretical_loss": 3.8393657857059953, + "tokens_seen": 645529600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006500300902708124, + "loss": 1.5973, + "theoretical_loss": 3.839325934473116, + "tokens_seen": 645595136 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006500140421263792, + "loss": 1.6838, + "theoretical_loss": 3.839286088418012, + "tokens_seen": 645660672 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006499979939819458, + "loss": 1.4978, + "theoretical_loss": 3.839246247539485, + "tokens_seen": 645726208 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006499819458375126, + "loss": 1.5661, + "theoretical_loss": 3.8392064118363374, + "tokens_seen": 645791744 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006499658976930792, + "loss": 1.5373, + "theoretical_loss": 3.839166581307372, + "tokens_seen": 645857280 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006499498495486459, + "loss": 1.5178, + "theoretical_loss": 3.8391267559513915, + "tokens_seen": 645922816 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006499338014042126, + "loss": 1.5713, + "theoretical_loss": 3.839086935767199, + "tokens_seen": 645988352 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006499177532597793, + "loss": 1.5714, + "theoretical_loss": 3.839047120753599, + "tokens_seen": 646053888 + }, + { + "epoch": 0.2, + "learning_rate": 0.000649901705115346, + "loss": 1.3966, + "theoretical_loss": 3.8390073109093947, + "tokens_seen": 646119424 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006498856569709127, + "loss": 1.5087, + "theoretical_loss": 3.8389675062333923, + "tokens_seen": 646184960 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006498696088264794, + "loss": 1.4944, + "theoretical_loss": 3.838927706724395, + "tokens_seen": 646250496 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006498535606820461, + "loss": 1.4298, + "theoretical_loss": 3.8388879123812094, + "tokens_seen": 646316032 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006498375125376128, + "loss": 1.4769, + "theoretical_loss": 3.8388481232026406, + "tokens_seen": 646381568 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006498214643931796, + "loss": 1.4489, + "theoretical_loss": 3.838808339187495, + "tokens_seen": 646447104 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006498054162487462, + "loss": 1.498, + "theoretical_loss": 3.8387685603345805, + "tokens_seen": 646512640 + }, + { + "epoch": 0.2, + "learning_rate": 0.000649789368104313, + "loss": 1.5269, + "theoretical_loss": 3.838728786642702, + "tokens_seen": 646578176 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006497733199598796, + "loss": 1.5944, + "theoretical_loss": 3.838689018110667, + "tokens_seen": 646643712 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006497572718154464, + "loss": 1.4476, + "theoretical_loss": 3.838649254737285, + "tokens_seen": 646709248 + }, + { + "epoch": 0.2, + "learning_rate": 0.000649741223671013, + "loss": 1.5928, + "theoretical_loss": 3.838609496521363, + "tokens_seen": 646774784 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006497251755265798, + "loss": 1.6662, + "theoretical_loss": 3.8385697434617096, + "tokens_seen": 646840320 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006497091273821464, + "loss": 1.5229, + "theoretical_loss": 3.838529995557134, + "tokens_seen": 646905856 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006496930792377132, + "loss": 1.3988, + "theoretical_loss": 3.8384902528064457, + "tokens_seen": 646971392 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006496770310932798, + "loss": 1.5081, + "theoretical_loss": 3.8384505152084545, + "tokens_seen": 647036928 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006496609829488466, + "loss": 1.5749, + "theoretical_loss": 3.8384107827619696, + "tokens_seen": 647102464 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006496449348044132, + "loss": 1.5457, + "theoretical_loss": 3.8383710554658026, + "tokens_seen": 647168000 + }, + { + "epoch": 0.2, + "learning_rate": 0.00064962888665998, + "loss": 1.4828, + "theoretical_loss": 3.838331333318764, + "tokens_seen": 647233536 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006496128385155466, + "loss": 1.4707, + "theoretical_loss": 3.8382916163196654, + "tokens_seen": 647299072 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006495967903711134, + "loss": 1.4802, + "theoretical_loss": 3.8382519044673176, + "tokens_seen": 647364608 + }, + { + "epoch": 0.2, + "learning_rate": 0.00064958074222668, + "loss": 1.504, + "theoretical_loss": 3.8382121977605346, + "tokens_seen": 647430144 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006495646940822468, + "loss": 1.5652, + "theoretical_loss": 3.8381724961981267, + "tokens_seen": 647495680 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006495486459378134, + "loss": 1.5618, + "theoretical_loss": 3.8381327997789088, + "tokens_seen": 647561216 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006495325977933802, + "loss": 1.6122, + "theoretical_loss": 3.8380931085016927, + "tokens_seen": 647626752 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006495165496489468, + "loss": 1.5541, + "theoretical_loss": 3.8380534223652933, + "tokens_seen": 647692288 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006495005015045136, + "loss": 1.5956, + "theoretical_loss": 3.838013741368523, + "tokens_seen": 647757824 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006494844533600802, + "loss": 1.5402, + "theoretical_loss": 3.837974065510198, + "tokens_seen": 647823360 + }, + { + "epoch": 0.2, + "learning_rate": 0.000649468405215647, + "loss": 1.4822, + "theoretical_loss": 3.837934394789133, + "tokens_seen": 647888896 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006494523570712136, + "loss": 1.4672, + "theoretical_loss": 3.8378947292041428, + "tokens_seen": 647954432 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006494363089267804, + "loss": 1.4794, + "theoretical_loss": 3.8378550687540427, + "tokens_seen": 648019968 + }, + { + "epoch": 0.2, + "learning_rate": 0.000649420260782347, + "loss": 1.5184, + "theoretical_loss": 3.8378154134376494, + "tokens_seen": 648085504 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006494042126379138, + "loss": 1.6112, + "theoretical_loss": 3.837775763253779, + "tokens_seen": 648151040 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006493881644934805, + "loss": 1.5904, + "theoretical_loss": 3.8377361182012484, + "tokens_seen": 648216576 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006493721163490472, + "loss": 1.5798, + "theoretical_loss": 3.8376964782788754, + "tokens_seen": 648282112 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006493560682046139, + "loss": 1.5199, + "theoretical_loss": 3.8376568434854765, + "tokens_seen": 648347648 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006493400200601806, + "loss": 1.5322, + "theoretical_loss": 3.837617213819871, + "tokens_seen": 648413184 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006493239719157473, + "loss": 1.4795, + "theoretical_loss": 3.837577589280876, + "tokens_seen": 648478720 + }, + { + "epoch": 0.2, + "learning_rate": 0.000649307923771314, + "loss": 1.5338, + "theoretical_loss": 3.8375379698673115, + "tokens_seen": 648544256 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006492918756268807, + "loss": 1.468, + "theoretical_loss": 3.837498355577996, + "tokens_seen": 648609792 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006492758274824474, + "loss": 1.563, + "theoretical_loss": 3.837458746411749, + "tokens_seen": 648675328 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006492597793380141, + "loss": 1.4749, + "theoretical_loss": 3.8374191423673913, + "tokens_seen": 648740864 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.9949767589569092, + "objective/train/avg_token_score": 0.0050232852809131145, + "objective/train/docs_used": 241544, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.433426856994629, + "objective/train/original_loss": 1.44157075881958, + "objective/train/theoretical_loss": 3.8373795434437423, + "objective/train/tokens_used": 669266400, + "objective/train/unlikelihood_loss": 0.011114963330328465, + "theoretical_loss": 3.8373795434437423, + "tokens_seen": 648806400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006492437311935808, + "loss": 1.5119, + "theoretical_loss": 3.8373795434437423, + "tokens_seen": 648806400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006492276830491475, + "loss": 1.4715, + "theoretical_loss": 3.8373399496396234, + "tokens_seen": 648871936 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006492116349047142, + "loss": 1.5409, + "theoretical_loss": 3.8373003609538556, + "tokens_seen": 648937472 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006491955867602809, + "loss": 1.5526, + "theoretical_loss": 3.8372607773852603, + "tokens_seen": 649003008 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006491795386158476, + "loss": 1.5995, + "theoretical_loss": 3.8372211989326592, + "tokens_seen": 649068544 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006491634904714143, + "loss": 1.5512, + "theoretical_loss": 3.8371816255948756, + "tokens_seen": 649134080 + }, + { + "epoch": 0.2, + "learning_rate": 0.000649147442326981, + "loss": 1.4539, + "theoretical_loss": 3.837142057370731, + "tokens_seen": 649199616 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006491313941825477, + "loss": 1.5577, + "theoretical_loss": 3.837102494259049, + "tokens_seen": 649265152 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006491153460381144, + "loss": 1.491, + "theoretical_loss": 3.8370629362586532, + "tokens_seen": 649330688 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006490992978936811, + "loss": 1.5907, + "theoretical_loss": 3.8370233833683676, + "tokens_seen": 649396224 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006490832497492478, + "loss": 1.6756, + "theoretical_loss": 3.8369838355870165, + "tokens_seen": 649461760 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006490672016048145, + "loss": 1.497, + "theoretical_loss": 3.836944292913424, + "tokens_seen": 649527296 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006490511534603812, + "loss": 1.4533, + "theoretical_loss": 3.8369047553464153, + "tokens_seen": 649592832 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006490351053159479, + "loss": 1.4747, + "theoretical_loss": 3.836865222884817, + "tokens_seen": 649658368 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006490190571715146, + "loss": 1.5892, + "theoretical_loss": 3.8368256955274527, + "tokens_seen": 649723904 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006490030090270813, + "loss": 1.5166, + "theoretical_loss": 3.8367861732731505, + "tokens_seen": 649789440 + }, + { + "epoch": 0.2, + "learning_rate": 0.000648986960882648, + "loss": 1.433, + "theoretical_loss": 3.836746656120737, + "tokens_seen": 649854976 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006489709127382147, + "loss": 1.5387, + "theoretical_loss": 3.8367071440690372, + "tokens_seen": 649920512 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006489548645937814, + "loss": 1.5061, + "theoretical_loss": 3.836667637116881, + "tokens_seen": 649986048 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006489388164493481, + "loss": 1.4557, + "theoretical_loss": 3.8366281352630947, + "tokens_seen": 650051584 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006489227683049148, + "loss": 1.454, + "theoretical_loss": 3.8365886385065067, + "tokens_seen": 650117120 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006489067201604815, + "loss": 1.4471, + "theoretical_loss": 3.836549146845946, + "tokens_seen": 650182656 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006488906720160482, + "loss": 1.4658, + "theoretical_loss": 3.8365096602802407, + "tokens_seen": 650248192 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006488746238716149, + "loss": 1.644, + "theoretical_loss": 3.83647017880822, + "tokens_seen": 650313728 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006488585757271816, + "loss": 1.4768, + "theoretical_loss": 3.836430702428715, + "tokens_seen": 650379264 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006488425275827483, + "loss": 1.5289, + "theoretical_loss": 3.8363912311405546, + "tokens_seen": 650444800 + }, + { + "epoch": 0.2, + "learning_rate": 0.000648826479438315, + "loss": 1.5204, + "theoretical_loss": 3.8363517649425702, + "tokens_seen": 650510336 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006488104312938817, + "loss": 1.5296, + "theoretical_loss": 3.8363123038335916, + "tokens_seen": 650575872 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006487943831494484, + "loss": 1.4925, + "theoretical_loss": 3.8362728478124506, + "tokens_seen": 650641408 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006487783350050151, + "loss": 1.5511, + "theoretical_loss": 3.8362333968779794, + "tokens_seen": 650706944 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006487622868605818, + "loss": 1.5122, + "theoretical_loss": 3.836193951029008, + "tokens_seen": 650772480 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006487462387161485, + "loss": 1.5819, + "theoretical_loss": 3.8361545102643717, + "tokens_seen": 650838016 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006487301905717152, + "loss": 1.5049, + "theoretical_loss": 3.836115074582901, + "tokens_seen": 650903552 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006487141424272819, + "loss": 1.4828, + "theoretical_loss": 3.8360756439834303, + "tokens_seen": 650969088 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006486980942828485, + "loss": 1.4259, + "theoretical_loss": 3.836036218464792, + "tokens_seen": 651034624 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006486820461384153, + "loss": 1.4, + "theoretical_loss": 3.8359967980258216, + "tokens_seen": 651100160 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006486659979939819, + "loss": 1.4512, + "theoretical_loss": 3.8359573826653524, + "tokens_seen": 651165696 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006486499498495487, + "loss": 1.5609, + "theoretical_loss": 3.8359179723822194, + "tokens_seen": 651231232 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006486339017051153, + "loss": 1.6208, + "theoretical_loss": 3.835878567175257, + "tokens_seen": 651296768 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006486178535606821, + "loss": 1.5279, + "theoretical_loss": 3.8358391670433023, + "tokens_seen": 651362304 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006486018054162487, + "loss": 1.5403, + "theoretical_loss": 3.8357997719851893, + "tokens_seen": 651427840 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006485857572718155, + "loss": 1.542, + "theoretical_loss": 3.835760381999756, + "tokens_seen": 651493376 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006485697091273821, + "loss": 1.603, + "theoretical_loss": 3.835720997085838, + "tokens_seen": 651558912 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006485536609829489, + "loss": 1.561, + "theoretical_loss": 3.835681617242272, + "tokens_seen": 651624448 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006485376128385155, + "loss": 1.5098, + "theoretical_loss": 3.8356422424678964, + "tokens_seen": 651689984 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006485215646940823, + "loss": 1.5677, + "theoretical_loss": 3.835602872761549, + "tokens_seen": 651755520 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006485055165496489, + "loss": 1.5463, + "theoretical_loss": 3.8355635081220667, + "tokens_seen": 651821056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006484894684052157, + "loss": 1.5488, + "theoretical_loss": 3.8355241485482887, + "tokens_seen": 651886592 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006484734202607823, + "loss": 1.4958, + "theoretical_loss": 3.835484794039055, + "tokens_seen": 651952128 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006484573721163491, + "loss": 1.5293, + "theoretical_loss": 3.835445444593203, + "tokens_seen": 652017664 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.9952099323272705, + "objective/train/avg_token_score": 0.004790127277374268, + "objective/train/docs_used": 242832, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5427762269973755, + "objective/train/original_loss": 1.5495846271514893, + "objective/train/theoretical_loss": 3.8354061002095743, + "objective/train/tokens_used": 672543200, + "objective/train/unlikelihood_loss": 0.01030314527451992, + "theoretical_loss": 3.8354061002095743, + "tokens_seen": 652083200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006484413239719157, + "loss": 1.4902, + "theoretical_loss": 3.8354061002095743, + "tokens_seen": 652083200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006484252758274825, + "loss": 1.5647, + "theoretical_loss": 3.835366760887008, + "tokens_seen": 652148736 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006484092276830491, + "loss": 1.4633, + "theoretical_loss": 3.8353274266243442, + "tokens_seen": 652214272 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006483931795386159, + "loss": 1.5072, + "theoretical_loss": 3.835288097420425, + "tokens_seen": 652279808 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006483771313941825, + "loss": 1.619, + "theoretical_loss": 3.83524877327409, + "tokens_seen": 652345344 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006483610832497493, + "loss": 1.5735, + "theoretical_loss": 3.8352094541841817, + "tokens_seen": 652410880 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006483450351053159, + "loss": 1.4672, + "theoretical_loss": 3.8351701401495424, + "tokens_seen": 652476416 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006483289869608827, + "loss": 1.4685, + "theoretical_loss": 3.8351308311690144, + "tokens_seen": 652541952 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006483129388164493, + "loss": 1.5528, + "theoretical_loss": 3.83509152724144, + "tokens_seen": 652607488 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006482968906720161, + "loss": 1.6282, + "theoretical_loss": 3.8350522283656625, + "tokens_seen": 652673024 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006482808425275827, + "loss": 1.54, + "theoretical_loss": 3.8350129345405257, + "tokens_seen": 652738560 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006482647943831495, + "loss": 1.506, + "theoretical_loss": 3.8349736457648733, + "tokens_seen": 652804096 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006482487462387161, + "loss": 1.4683, + "theoretical_loss": 3.83493436203755, + "tokens_seen": 652869632 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006482326980942829, + "loss": 1.5255, + "theoretical_loss": 3.834895083357399, + "tokens_seen": 652935168 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006482166499498495, + "loss": 1.522, + "theoretical_loss": 3.8348558097232672, + "tokens_seen": 653000704 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006482006018054163, + "loss": 1.5857, + "theoretical_loss": 3.834816541133999, + "tokens_seen": 653066240 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006481845536609829, + "loss": 1.4375, + "theoretical_loss": 3.83477727758844, + "tokens_seen": 653131776 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006481685055165497, + "loss": 1.6699, + "theoretical_loss": 3.8347380190854374, + "tokens_seen": 653197312 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006481524573721163, + "loss": 1.5138, + "theoretical_loss": 3.8346987656238367, + "tokens_seen": 653262848 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006481364092276831, + "loss": 1.4905, + "theoretical_loss": 3.8346595172024855, + "tokens_seen": 653328384 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006481203610832497, + "loss": 1.4431, + "theoretical_loss": 3.834620273820231, + "tokens_seen": 653393920 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006481043129388165, + "loss": 1.5876, + "theoretical_loss": 3.834581035475921, + "tokens_seen": 653459456 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006480882647943831, + "loss": 1.509, + "theoretical_loss": 3.8345418021684035, + "tokens_seen": 653524992 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006480722166499499, + "loss": 1.5313, + "theoretical_loss": 3.8345025738965264, + "tokens_seen": 653590528 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006480561685055166, + "loss": 1.517, + "theoretical_loss": 3.8344633506591395, + "tokens_seen": 653656064 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006480401203610833, + "loss": 1.5684, + "theoretical_loss": 3.8344241324550907, + "tokens_seen": 653721600 + }, + { + "epoch": 0.2, + "learning_rate": 0.00064802407221665, + "loss": 1.5588, + "theoretical_loss": 3.8343849192832313, + "tokens_seen": 653787136 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006480080240722167, + "loss": 1.5667, + "theoretical_loss": 3.83434571114241, + "tokens_seen": 653852672 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006479919759277834, + "loss": 1.437, + "theoretical_loss": 3.834306508031478, + "tokens_seen": 653918208 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006479759277833501, + "loss": 1.6113, + "theoretical_loss": 3.8342673099492854, + "tokens_seen": 653983744 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006479598796389168, + "loss": 1.5545, + "theoretical_loss": 3.8342281168946832, + "tokens_seen": 654049280 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006479438314944835, + "loss": 1.5664, + "theoretical_loss": 3.8341889288665234, + "tokens_seen": 654114816 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006479277833500502, + "loss": 1.4923, + "theoretical_loss": 3.8341497458636575, + "tokens_seen": 654180352 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006479117352056169, + "loss": 1.5033, + "theoretical_loss": 3.834110567884938, + "tokens_seen": 654245888 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006478956870611836, + "loss": 1.5958, + "theoretical_loss": 3.8340713949292176, + "tokens_seen": 654311424 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006478796389167503, + "loss": 1.4906, + "theoretical_loss": 3.834032226995349, + "tokens_seen": 654376960 + }, + { + "epoch": 0.2, + "learning_rate": 0.000647863590772317, + "loss": 1.5031, + "theoretical_loss": 3.8339930640821853, + "tokens_seen": 654442496 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006478475426278837, + "loss": 1.4474, + "theoretical_loss": 3.8339539061885812, + "tokens_seen": 654508032 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006478314944834504, + "loss": 1.4523, + "theoretical_loss": 3.83391475331339, + "tokens_seen": 654573568 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006478154463390171, + "loss": 1.5103, + "theoretical_loss": 3.833875605455466, + "tokens_seen": 654639104 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006477993981945838, + "loss": 1.5216, + "theoretical_loss": 3.8338364626136654, + "tokens_seen": 654704640 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006477833500501505, + "loss": 1.4027, + "theoretical_loss": 3.833797324786842, + "tokens_seen": 654770176 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006477673019057172, + "loss": 1.539, + "theoretical_loss": 3.8337581919738524, + "tokens_seen": 654835712 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006477512537612839, + "loss": 1.6014, + "theoretical_loss": 3.8337190641735512, + "tokens_seen": 654901248 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006477352056168506, + "loss": 1.5492, + "theoretical_loss": 3.833679941384797, + "tokens_seen": 654966784 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006477191574724173, + "loss": 1.3878, + "theoretical_loss": 3.833640823606445, + "tokens_seen": 655032320 + }, + { + "epoch": 0.2, + "learning_rate": 0.000647703109327984, + "loss": 1.5002, + "theoretical_loss": 3.8336017108373523, + "tokens_seen": 655097856 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006476870611835507, + "loss": 1.6133, + "theoretical_loss": 3.833562603076377, + "tokens_seen": 655163392 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006476710130391174, + "loss": 1.5895, + "theoretical_loss": 3.833523500322377, + "tokens_seen": 655228928 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006476549648946841, + "loss": 1.4649, + "theoretical_loss": 3.83348440257421, + "tokens_seen": 655294464 + }, + { + "debugging/Compilability": 0.9230769230769231, + "debugging/distinct-1-grams": 0.7560193640250044, + "debugging/entropy-1-grams": 5.2534808493234895, + "debugging/length": 422.2307692307692, + "debugging/num_segments": 13, + "debugging/score": 0.01638830111902192, + "debugging/score_std": 0.019821935735086043, + "epoch": 0.2, + "objective/train/aligned_token_count": 0.9914212226867676, + "objective/train/avg_token_score": 0.008578718639910221, + "objective/train/docs_used": 243976, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4500035047531128, + "objective/train/original_loss": 1.4650835990905762, + "objective/train/theoretical_loss": 3.833445309830735, + "objective/train/tokens_used": 675820000, + "objective/train/unlikelihood_loss": 0.018921513110399246, + "theoretical_loss": 3.833445309830735, + "tokens_seen": 655360000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006476389167502508, + "loss": 1.5687, + "theoretical_loss": 3.833445309830735, + "tokens_seen": 655360000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006476228686058175, + "loss": 1.5475, + "theoretical_loss": 3.833406222090811, + "tokens_seen": 655425536 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006476068204613842, + "loss": 1.4464, + "theoretical_loss": 3.8333671393532973, + "tokens_seen": 655491072 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006475907723169509, + "loss": 1.5395, + "theoretical_loss": 3.8333280616170535, + "tokens_seen": 655556608 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006475747241725176, + "loss": 1.5151, + "theoretical_loss": 3.8332889888809403, + "tokens_seen": 655622144 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006475586760280843, + "loss": 1.4926, + "theoretical_loss": 3.8332499211438176, + "tokens_seen": 655687680 + }, + { + "epoch": 0.2, + "learning_rate": 0.000647542627883651, + "loss": 1.5414, + "theoretical_loss": 3.8332108584045463, + "tokens_seen": 655753216 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006475265797392177, + "loss": 1.5761, + "theoretical_loss": 3.8331718006619884, + "tokens_seen": 655818752 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006475105315947844, + "loss": 1.5093, + "theoretical_loss": 3.833132747915004, + "tokens_seen": 655884288 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006474944834503511, + "loss": 1.4608, + "theoretical_loss": 3.8330937001624568, + "tokens_seen": 655949824 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006474784353059178, + "loss": 1.3735, + "theoretical_loss": 3.833054657403208, + "tokens_seen": 656015360 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006474623871614845, + "loss": 1.5718, + "theoretical_loss": 3.8330156196361207, + "tokens_seen": 656080896 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006474463390170512, + "loss": 1.6025, + "theoretical_loss": 3.8329765868600583, + "tokens_seen": 656146432 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006474302908726179, + "loss": 1.5495, + "theoretical_loss": 3.832937559073884, + "tokens_seen": 656211968 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006474142427281846, + "loss": 1.5342, + "theoretical_loss": 3.8328985362764616, + "tokens_seen": 656277504 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006473981945837513, + "loss": 1.6946, + "theoretical_loss": 3.8328595184666554, + "tokens_seen": 656343040 + }, + { + "epoch": 0.2, + "learning_rate": 0.000647382146439318, + "loss": 1.5367, + "theoretical_loss": 3.8328205056433298, + "tokens_seen": 656408576 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006473660982948847, + "loss": 1.4555, + "theoretical_loss": 3.8327814978053505, + "tokens_seen": 656474112 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006473500501504514, + "loss": 1.5754, + "theoretical_loss": 3.8327424949515816, + "tokens_seen": 656539648 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006473340020060181, + "loss": 1.6319, + "theoretical_loss": 3.83270349708089, + "tokens_seen": 656605184 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006473179538615848, + "loss": 1.529, + "theoretical_loss": 3.832664504192141, + "tokens_seen": 656670720 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006473019057171514, + "loss": 1.5182, + "theoretical_loss": 3.8326255162842022, + "tokens_seen": 656736256 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006472858575727182, + "loss": 1.4531, + "theoretical_loss": 3.832586533355939, + "tokens_seen": 656801792 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006472698094282848, + "loss": 1.4959, + "theoretical_loss": 3.832547555406219, + "tokens_seen": 656867328 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006472537612838516, + "loss": 1.4566, + "theoretical_loss": 3.8325085824339102, + "tokens_seen": 656932864 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006472377131394182, + "loss": 1.5019, + "theoretical_loss": 3.8324696144378803, + "tokens_seen": 656998400 + }, + { + "epoch": 0.2, + "learning_rate": 0.000647221664994985, + "loss": 1.3906, + "theoretical_loss": 3.8324306514169977, + "tokens_seen": 657063936 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006472056168505516, + "loss": 1.508, + "theoretical_loss": 3.832391693370131, + "tokens_seen": 657129472 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006471895687061184, + "loss": 1.588, + "theoretical_loss": 3.832352740296149, + "tokens_seen": 657195008 + }, + { + "epoch": 0.2, + "learning_rate": 0.000647173520561685, + "loss": 1.5328, + "theoretical_loss": 3.8323137921939217, + "tokens_seen": 657260544 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006471574724172518, + "loss": 1.411, + "theoretical_loss": 3.832274849062318, + "tokens_seen": 657326080 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006471414242728184, + "loss": 1.4741, + "theoretical_loss": 3.8322359109002093, + "tokens_seen": 657391616 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006471253761283852, + "loss": 1.4736, + "theoretical_loss": 3.8321969777064653, + "tokens_seen": 657457152 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006471093279839518, + "loss": 1.4827, + "theoretical_loss": 3.8321580494799568, + "tokens_seen": 657522688 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006470932798395186, + "loss": 1.6265, + "theoretical_loss": 3.8321191262195553, + "tokens_seen": 657588224 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006470772316950852, + "loss": 1.5582, + "theoretical_loss": 3.832080207924133, + "tokens_seen": 657653760 + }, + { + "epoch": 0.2, + "learning_rate": 0.000647061183550652, + "loss": 1.4944, + "theoretical_loss": 3.832041294592561, + "tokens_seen": 657719296 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006470451354062186, + "loss": 1.4774, + "theoretical_loss": 3.832002386223712, + "tokens_seen": 657784832 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006470290872617854, + "loss": 1.4389, + "theoretical_loss": 3.831963482816459, + "tokens_seen": 657850368 + }, + { + "epoch": 0.2, + "learning_rate": 0.000647013039117352, + "loss": 1.5274, + "theoretical_loss": 3.8319245843696743, + "tokens_seen": 657915904 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006469969909729188, + "loss": 1.5622, + "theoretical_loss": 3.8318856908822325, + "tokens_seen": 657981440 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006469809428284854, + "loss": 1.4891, + "theoretical_loss": 3.831846802353007, + "tokens_seen": 658046976 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006469648946840522, + "loss": 1.5829, + "theoretical_loss": 3.8318079187808713, + "tokens_seen": 658112512 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006469488465396188, + "loss": 1.5605, + "theoretical_loss": 3.831769040164701, + "tokens_seen": 658178048 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006469327983951856, + "loss": 1.47, + "theoretical_loss": 3.8317301665033705, + "tokens_seen": 658243584 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006469167502507522, + "loss": 1.5676, + "theoretical_loss": 3.8316912977957553, + "tokens_seen": 658309120 + }, + { + "epoch": 0.2, + "learning_rate": 0.000646900702106319, + "loss": 1.5085, + "theoretical_loss": 3.8316524340407314, + "tokens_seen": 658374656 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006468846539618856, + "loss": 1.51, + "theoretical_loss": 3.8316135752371743, + "tokens_seen": 658440192 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006468686058174524, + "loss": 1.5061, + "theoretical_loss": 3.8315747213839604, + "tokens_seen": 658505728 + }, + { + "epoch": 0.2, + "learning_rate": 0.000646852557673019, + "loss": 1.4625, + "theoretical_loss": 3.831535872479967, + "tokens_seen": 658571264 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.98899245262146, + "objective/train/avg_token_score": 0.011007587425410748, + "objective/train/docs_used": 245288, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4003214836120605, + "objective/train/original_loss": 1.4168421030044556, + "objective/train/theoretical_loss": 3.831497028524071, + "objective/train/tokens_used": 679096800, + "objective/train/unlikelihood_loss": 0.028428860008716583, + "theoretical_loss": 3.831497028524071, + "tokens_seen": 658636800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006468365095285858, + "loss": 1.4802, + "theoretical_loss": 3.831497028524071, + "tokens_seen": 658636800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006468204613841524, + "loss": 1.5012, + "theoretical_loss": 3.83145818951515, + "tokens_seen": 658702336 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006468044132397192, + "loss": 1.4637, + "theoretical_loss": 3.8314193554520806, + "tokens_seen": 658767872 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006467883650952858, + "loss": 1.5203, + "theoretical_loss": 3.831380526333743, + "tokens_seen": 658833408 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006467723169508526, + "loss": 1.5198, + "theoretical_loss": 3.8313417021590155, + "tokens_seen": 658898944 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006467562688064192, + "loss": 1.5497, + "theoretical_loss": 3.831302882926776, + "tokens_seen": 658964480 + }, + { + "epoch": 0.2, + "learning_rate": 0.000646740220661986, + "loss": 1.4729, + "theoretical_loss": 3.831264068635905, + "tokens_seen": 659030016 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006467241725175526, + "loss": 1.5069, + "theoretical_loss": 3.831225259285281, + "tokens_seen": 659095552 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006467081243731194, + "loss": 1.5185, + "theoretical_loss": 3.8311864548737855, + "tokens_seen": 659161088 + }, + { + "epoch": 0.2, + "learning_rate": 0.000646692076228686, + "loss": 1.5547, + "theoretical_loss": 3.831147655400298, + "tokens_seen": 659226624 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006466760280842528, + "loss": 1.5695, + "theoretical_loss": 3.831108860863699, + "tokens_seen": 659292160 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006466599799398195, + "loss": 1.5656, + "theoretical_loss": 3.8310700712628707, + "tokens_seen": 659357696 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006466439317953862, + "loss": 1.5632, + "theoretical_loss": 3.8310312865966942, + "tokens_seen": 659423232 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006466278836509529, + "loss": 1.5344, + "theoretical_loss": 3.830992506864051, + "tokens_seen": 659488768 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006466118355065196, + "loss": 1.5674, + "theoretical_loss": 3.830953732063824, + "tokens_seen": 659554304 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006465957873620863, + "loss": 1.5804, + "theoretical_loss": 3.830914962194895, + "tokens_seen": 659619840 + }, + { + "epoch": 0.2, + "learning_rate": 0.000646579739217653, + "loss": 1.619, + "theoretical_loss": 3.830876197256148, + "tokens_seen": 659685376 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006465636910732197, + "loss": 1.5122, + "theoretical_loss": 3.8308374372464655, + "tokens_seen": 659750912 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006465476429287864, + "loss": 1.5267, + "theoretical_loss": 3.830798682164732, + "tokens_seen": 659816448 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006465315947843531, + "loss": 1.4962, + "theoretical_loss": 3.8307599320098307, + "tokens_seen": 659881984 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006465155466399198, + "loss": 1.5829, + "theoretical_loss": 3.8307211867806465, + "tokens_seen": 659947520 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006464994984954865, + "loss": 1.4204, + "theoretical_loss": 3.8306824464760645, + "tokens_seen": 660013056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006464834503510532, + "loss": 1.4877, + "theoretical_loss": 3.83064371109497, + "tokens_seen": 660078592 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006464674022066199, + "loss": 1.5387, + "theoretical_loss": 3.830604980636248, + "tokens_seen": 660144128 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006464513540621866, + "loss": 1.5577, + "theoretical_loss": 3.8305662550987845, + "tokens_seen": 660209664 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006464353059177533, + "loss": 1.5395, + "theoretical_loss": 3.8305275344814658, + "tokens_seen": 660275200 + }, + { + "epoch": 0.2, + "learning_rate": 0.00064641925777332, + "loss": 1.4254, + "theoretical_loss": 3.8304888187831785, + "tokens_seen": 660340736 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006464032096288867, + "loss": 1.5081, + "theoretical_loss": 3.8304501080028097, + "tokens_seen": 660406272 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006463871614844534, + "loss": 1.5545, + "theoretical_loss": 3.830411402139247, + "tokens_seen": 660471808 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006463711133400201, + "loss": 1.5228, + "theoretical_loss": 3.830372701191377, + "tokens_seen": 660537344 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006463550651955868, + "loss": 1.4992, + "theoretical_loss": 3.8303340051580896, + "tokens_seen": 660602880 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006463390170511535, + "loss": 1.547, + "theoretical_loss": 3.8302953140382714, + "tokens_seen": 660668416 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006463229689067202, + "loss": 1.4422, + "theoretical_loss": 3.8302566278308126, + "tokens_seen": 660733952 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006463069207622869, + "loss": 1.4515, + "theoretical_loss": 3.830217946534602, + "tokens_seen": 660799488 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006462908726178536, + "loss": 1.4824, + "theoretical_loss": 3.830179270148528, + "tokens_seen": 660865024 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006462748244734203, + "loss": 1.6566, + "theoretical_loss": 3.8301405986714823, + "tokens_seen": 660930560 + }, + { + "epoch": 0.2, + "learning_rate": 0.000646258776328987, + "loss": 1.5127, + "theoretical_loss": 3.8301019321023544, + "tokens_seen": 660996096 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006462427281845538, + "loss": 1.5099, + "theoretical_loss": 3.830063270440034, + "tokens_seen": 661061632 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006462266800401204, + "loss": 1.4875, + "theoretical_loss": 3.8300246136834133, + "tokens_seen": 661127168 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006462106318956872, + "loss": 1.3977, + "theoretical_loss": 3.829985961831383, + "tokens_seen": 661192704 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006461945837512538, + "loss": 1.4889, + "theoretical_loss": 3.829947314882835, + "tokens_seen": 661258240 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006461785356068206, + "loss": 1.4258, + "theoretical_loss": 3.829908672836661, + "tokens_seen": 661323776 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006461624874623872, + "loss": 1.5435, + "theoretical_loss": 3.829870035691754, + "tokens_seen": 661389312 + }, + { + "epoch": 0.2, + "learning_rate": 0.000646146439317954, + "loss": 1.572, + "theoretical_loss": 3.829831403447007, + "tokens_seen": 661454848 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006461303911735206, + "loss": 1.4439, + "theoretical_loss": 3.8297927761013115, + "tokens_seen": 661520384 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006461143430290874, + "loss": 1.5621, + "theoretical_loss": 3.829754153653563, + "tokens_seen": 661585920 + }, + { + "epoch": 0.2, + "learning_rate": 0.000646098294884654, + "loss": 1.5284, + "theoretical_loss": 3.829715536102654, + "tokens_seen": 661651456 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006460822467402208, + "loss": 1.5201, + "theoretical_loss": 3.8296769234474795, + "tokens_seen": 661716992 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006460661985957874, + "loss": 1.4393, + "theoretical_loss": 3.829638315686933, + "tokens_seen": 661782528 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006460501504513542, + "loss": 1.5897, + "theoretical_loss": 3.8295997128199106, + "tokens_seen": 661848064 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.9894727468490601, + "objective/train/avg_token_score": 0.010527306236326694, + "objective/train/docs_used": 246530, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5516411066055298, + "objective/train/original_loss": 1.5682170391082764, + "objective/train/theoretical_loss": 3.8295611148453075, + "objective/train/tokens_used": 682373600, + "objective/train/unlikelihood_loss": 0.02196403220295906, + "theoretical_loss": 3.8295611148453075, + "tokens_seen": 661913600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006460341023069208, + "loss": 1.6194, + "theoretical_loss": 3.8295611148453075, + "tokens_seen": 661913600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006460180541624875, + "loss": 1.4588, + "theoretical_loss": 3.8295225217620183, + "tokens_seen": 661979136 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006460020060180542, + "loss": 1.5068, + "theoretical_loss": 3.82948393356894, + "tokens_seen": 662044672 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006459859578736209, + "loss": 1.5694, + "theoretical_loss": 3.829445350264969, + "tokens_seen": 662110208 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006459699097291876, + "loss": 1.6112, + "theoretical_loss": 3.8294067718490012, + "tokens_seen": 662175744 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006459538615847543, + "loss": 1.62, + "theoretical_loss": 3.829368198319934, + "tokens_seen": 662241280 + }, + { + "epoch": 0.2, + "learning_rate": 0.000645937813440321, + "loss": 1.5225, + "theoretical_loss": 3.829329629676665, + "tokens_seen": 662306816 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006459217652958877, + "loss": 1.5316, + "theoretical_loss": 3.829291065918092, + "tokens_seen": 662372352 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006459057171514543, + "loss": 1.4395, + "theoretical_loss": 3.8292525070431136, + "tokens_seen": 662437888 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006458896690070211, + "loss": 1.4762, + "theoretical_loss": 3.829213953050627, + "tokens_seen": 662503424 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006458736208625877, + "loss": 1.4401, + "theoretical_loss": 3.8291754039395323, + "tokens_seen": 662568960 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006458575727181545, + "loss": 1.3932, + "theoretical_loss": 3.8291368597087283, + "tokens_seen": 662634496 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006458415245737211, + "loss": 1.4885, + "theoretical_loss": 3.8290983203571147, + "tokens_seen": 662700032 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006458254764292879, + "loss": 1.596, + "theoretical_loss": 3.829059785883591, + "tokens_seen": 662765568 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006458094282848545, + "loss": 1.4333, + "theoretical_loss": 3.8290212562870583, + "tokens_seen": 662831104 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006457933801404213, + "loss": 1.3687, + "theoretical_loss": 3.8289827315664158, + "tokens_seen": 662896640 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006457773319959879, + "loss": 1.5097, + "theoretical_loss": 3.8289442117205663, + "tokens_seen": 662962176 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006457612838515547, + "loss": 1.5543, + "theoretical_loss": 3.82890569674841, + "tokens_seen": 663027712 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006457452357071213, + "loss": 1.5155, + "theoretical_loss": 3.828867186648849, + "tokens_seen": 663093248 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006457291875626881, + "loss": 1.6646, + "theoretical_loss": 3.8288286814207853, + "tokens_seen": 663158784 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006457131394182547, + "loss": 1.5294, + "theoretical_loss": 3.828790181063121, + "tokens_seen": 663224320 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006456970912738215, + "loss": 1.4848, + "theoretical_loss": 3.828751685574759, + "tokens_seen": 663289856 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006456810431293881, + "loss": 1.4891, + "theoretical_loss": 3.828713194954604, + "tokens_seen": 663355392 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006456649949849549, + "loss": 1.4766, + "theoretical_loss": 3.8286747092015574, + "tokens_seen": 663420928 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006456489468405215, + "loss": 1.5545, + "theoretical_loss": 3.828636228314523, + "tokens_seen": 663486464 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006456328986960883, + "loss": 1.5663, + "theoretical_loss": 3.8285977522924073, + "tokens_seen": 663552000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006456168505516549, + "loss": 1.6191, + "theoretical_loss": 3.8285592811341127, + "tokens_seen": 663617536 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006456008024072217, + "loss": 1.468, + "theoretical_loss": 3.8285208148385443, + "tokens_seen": 663683072 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006455847542627883, + "loss": 1.4585, + "theoretical_loss": 3.8284823534046084, + "tokens_seen": 663748608 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006455687061183551, + "loss": 1.4949, + "theoretical_loss": 3.8284438968312102, + "tokens_seen": 663814144 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006455526579739217, + "loss": 1.6263, + "theoretical_loss": 3.828405445117255, + "tokens_seen": 663879680 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006455366098294885, + "loss": 1.5039, + "theoretical_loss": 3.82836699826165, + "tokens_seen": 663945216 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006455205616850551, + "loss": 1.5084, + "theoretical_loss": 3.828328556263302, + "tokens_seen": 664010752 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006455045135406219, + "loss": 1.4684, + "theoretical_loss": 3.8282901191211174, + "tokens_seen": 664076288 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006454884653961885, + "loss": 1.4871, + "theoretical_loss": 3.8282516868340037, + "tokens_seen": 664141824 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006454724172517553, + "loss": 1.5663, + "theoretical_loss": 3.8282132594008687, + "tokens_seen": 664207360 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006454563691073219, + "loss": 1.4843, + "theoretical_loss": 3.828174836820621, + "tokens_seen": 664272896 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006454403209628887, + "loss": 1.6141, + "theoretical_loss": 3.8281364190921683, + "tokens_seen": 664338432 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006454242728184553, + "loss": 1.5146, + "theoretical_loss": 3.82809800621442, + "tokens_seen": 664403968 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006454082246740221, + "loss": 1.3764, + "theoretical_loss": 3.8280595981862846, + "tokens_seen": 664469504 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006453921765295887, + "loss": 1.5178, + "theoretical_loss": 3.8280211950066723, + "tokens_seen": 664535040 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006453761283851555, + "loss": 1.5519, + "theoretical_loss": 3.827982796674493, + "tokens_seen": 664600576 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006453600802407221, + "loss": 1.4805, + "theoretical_loss": 3.8279444031886563, + "tokens_seen": 664666112 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006453440320962889, + "loss": 1.5431, + "theoretical_loss": 3.827906014548073, + "tokens_seen": 664731648 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006453279839518556, + "loss": 1.4319, + "theoretical_loss": 3.827867630751654, + "tokens_seen": 664797184 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006453119358074223, + "loss": 1.5404, + "theoretical_loss": 3.827829251798311, + "tokens_seen": 664862720 + }, + { + "epoch": 0.2, + "learning_rate": 0.000645295887662989, + "loss": 1.4478, + "theoretical_loss": 3.8277908776869554, + "tokens_seen": 664928256 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006452798395185557, + "loss": 1.5426, + "theoretical_loss": 3.8277525084164994, + "tokens_seen": 664993792 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006452637913741224, + "loss": 1.5215, + "theoretical_loss": 3.8277141439858546, + "tokens_seen": 665059328 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006452477432296891, + "loss": 1.6613, + "theoretical_loss": 3.827675784393934, + "tokens_seen": 665124864 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.9918051362037659, + "objective/train/avg_token_score": 0.008194856345653534, + "objective/train/docs_used": 247583, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6224291324615479, + "objective/train/original_loss": 1.6371939182281494, + "objective/train/theoretical_loss": 3.827637429639651, + "objective/train/tokens_used": 685650400, + "objective/train/unlikelihood_loss": 0.01554748322814703, + "theoretical_loss": 3.827637429639651, + "tokens_seen": 665190400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006452316950852558, + "loss": 1.5474, + "theoretical_loss": 3.827637429639651, + "tokens_seen": 665190400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006452156469408225, + "loss": 1.4332, + "theoretical_loss": 3.827599079721918, + "tokens_seen": 665255936 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006451995987963892, + "loss": 1.5256, + "theoretical_loss": 3.8275607346396505, + "tokens_seen": 665321472 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006451835506519559, + "loss": 1.6016, + "theoretical_loss": 3.827522394391761, + "tokens_seen": 665387008 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006451675025075226, + "loss": 1.4799, + "theoretical_loss": 3.8274840589771655, + "tokens_seen": 665452544 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006451514543630893, + "loss": 1.4416, + "theoretical_loss": 3.8274457283947765, + "tokens_seen": 665518080 + }, + { + "epoch": 0.2, + "learning_rate": 0.000645135406218656, + "loss": 1.4847, + "theoretical_loss": 3.8274074026435114, + "tokens_seen": 665583616 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006451193580742227, + "loss": 1.4821, + "theoretical_loss": 3.8273690817222845, + "tokens_seen": 665649152 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006451033099297894, + "loss": 1.3762, + "theoretical_loss": 3.8273307656300117, + "tokens_seen": 665714688 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006450872617853561, + "loss": 1.4404, + "theoretical_loss": 3.8272924543656095, + "tokens_seen": 665780224 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006450712136409228, + "loss": 1.512, + "theoretical_loss": 3.8272541479279942, + "tokens_seen": 665845760 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006450551654964895, + "loss": 1.5032, + "theoretical_loss": 3.827215846316083, + "tokens_seen": 665911296 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006450391173520563, + "loss": 1.4848, + "theoretical_loss": 3.827177549528793, + "tokens_seen": 665976832 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006450230692076229, + "loss": 1.5054, + "theoretical_loss": 3.8271392575650416, + "tokens_seen": 666042368 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006450070210631897, + "loss": 1.4688, + "theoretical_loss": 3.827100970423747, + "tokens_seen": 666107904 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006449909729187563, + "loss": 1.547, + "theoretical_loss": 3.827062688103827, + "tokens_seen": 666173440 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006449749247743231, + "loss": 1.5198, + "theoretical_loss": 3.827024410604201, + "tokens_seen": 666238976 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006449588766298897, + "loss": 1.5149, + "theoretical_loss": 3.8269861379237877, + "tokens_seen": 666304512 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006449428284854565, + "loss": 1.4703, + "theoretical_loss": 3.826947870061506, + "tokens_seen": 666370048 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006449267803410231, + "loss": 1.4989, + "theoretical_loss": 3.826909607016276, + "tokens_seen": 666435584 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006449107321965899, + "loss": 1.5317, + "theoretical_loss": 3.8268713487870176, + "tokens_seen": 666501120 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006448946840521565, + "loss": 1.4126, + "theoretical_loss": 3.8268330953726513, + "tokens_seen": 666566656 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006448786359077233, + "loss": 1.534, + "theoretical_loss": 3.826794846772098, + "tokens_seen": 666632192 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006448625877632899, + "loss": 1.5943, + "theoretical_loss": 3.826756602984278, + "tokens_seen": 666697728 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006448465396188567, + "loss": 1.4661, + "theoretical_loss": 3.826718364008114, + "tokens_seen": 666763264 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006448304914744233, + "loss": 1.5589, + "theoretical_loss": 3.826680129842526, + "tokens_seen": 666828800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006448144433299901, + "loss": 1.5488, + "theoretical_loss": 3.826641900486438, + "tokens_seen": 666894336 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006447983951855567, + "loss": 1.4774, + "theoretical_loss": 3.8266036759387716, + "tokens_seen": 666959872 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006447823470411235, + "loss": 1.444, + "theoretical_loss": 3.8265654561984492, + "tokens_seen": 667025408 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006447662988966901, + "loss": 1.6231, + "theoretical_loss": 3.8265272412643947, + "tokens_seen": 667090944 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006447502507522569, + "loss": 1.4049, + "theoretical_loss": 3.8264890311355315, + "tokens_seen": 667156480 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006447342026078235, + "loss": 1.4284, + "theoretical_loss": 3.8264508258107828, + "tokens_seen": 667222016 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006447181544633903, + "loss": 1.5329, + "theoretical_loss": 3.8264126252890738, + "tokens_seen": 667287552 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006447021063189569, + "loss": 1.4895, + "theoretical_loss": 3.826374429569328, + "tokens_seen": 667353088 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006446860581745237, + "loss": 1.4977, + "theoretical_loss": 3.8263362386504713, + "tokens_seen": 667418624 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006446700100300903, + "loss": 1.5492, + "theoretical_loss": 3.826298052531428, + "tokens_seen": 667484160 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006446539618856571, + "loss": 1.5472, + "theoretical_loss": 3.8262598712111244, + "tokens_seen": 667549696 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006446379137412237, + "loss": 1.489, + "theoretical_loss": 3.8262216946884866, + "tokens_seen": 667615232 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006446218655967904, + "loss": 1.5038, + "theoretical_loss": 3.8261835229624395, + "tokens_seen": 667680768 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006446058174523571, + "loss": 1.4554, + "theoretical_loss": 3.8261453560319114, + "tokens_seen": 667746304 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006445897693079238, + "loss": 1.4097, + "theoretical_loss": 3.8261071938958278, + "tokens_seen": 667811840 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006445737211634905, + "loss": 1.557, + "theoretical_loss": 3.826069036553118, + "tokens_seen": 667877376 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006445576730190572, + "loss": 1.4863, + "theoretical_loss": 3.8260308840027073, + "tokens_seen": 667942912 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006445416248746239, + "loss": 1.5152, + "theoretical_loss": 3.825992736243525, + "tokens_seen": 668008448 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006445255767301906, + "loss": 1.4464, + "theoretical_loss": 3.8259545932745, + "tokens_seen": 668073984 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006445095285857573, + "loss": 1.4857, + "theoretical_loss": 3.8259164550945597, + "tokens_seen": 668139520 + }, + { + "epoch": 0.2, + "learning_rate": 0.000644493480441324, + "loss": 1.5826, + "theoretical_loss": 3.8258783217026338, + "tokens_seen": 668205056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006444774322968906, + "loss": 1.4921, + "theoretical_loss": 3.825840193097652, + "tokens_seen": 668270592 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006444613841524574, + "loss": 1.5208, + "theoretical_loss": 3.8258020692785433, + "tokens_seen": 668336128 + }, + { + "epoch": 0.2, + "learning_rate": 0.000644445336008024, + "loss": 1.4895, + "theoretical_loss": 3.825763950244238, + "tokens_seen": 668401664 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.9928392767906189, + "objective/train/avg_token_score": 0.007160807494074106, + "objective/train/docs_used": 248744, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.555258870124817, + "objective/train/original_loss": 1.564308762550354, + "objective/train/theoretical_loss": 3.8257258359936666, + "objective/train/tokens_used": 688927200, + "objective/train/unlikelihood_loss": 0.020795905962586403, + "theoretical_loss": 3.8257258359936666, + "tokens_seen": 668467200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006444292878635908, + "loss": 1.5986, + "theoretical_loss": 3.8257258359936666, + "tokens_seen": 668467200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006444132397191574, + "loss": 1.4579, + "theoretical_loss": 3.82568772652576, + "tokens_seen": 668532736 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006443971915747242, + "loss": 1.5532, + "theoretical_loss": 3.8256496218394496, + "tokens_seen": 668598272 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006443811434302908, + "loss": 1.4589, + "theoretical_loss": 3.825611521933666, + "tokens_seen": 668663808 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006443650952858576, + "loss": 1.4168, + "theoretical_loss": 3.8255734268073414, + "tokens_seen": 668729344 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006443490471414242, + "loss": 1.5588, + "theoretical_loss": 3.8255353364594082, + "tokens_seen": 668794880 + }, + { + "epoch": 0.2, + "learning_rate": 0.000644332998996991, + "loss": 1.4249, + "theoretical_loss": 3.825497250888799, + "tokens_seen": 668860416 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006443169508525576, + "loss": 1.5356, + "theoretical_loss": 3.8254591700944456, + "tokens_seen": 668925952 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006443009027081244, + "loss": 1.456, + "theoretical_loss": 3.825421094075282, + "tokens_seen": 668991488 + }, + { + "epoch": 0.2, + "learning_rate": 0.000644284854563691, + "loss": 1.4451, + "theoretical_loss": 3.8253830228302417, + "tokens_seen": 669057024 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006442688064192578, + "loss": 1.4934, + "theoretical_loss": 3.825344956358258, + "tokens_seen": 669122560 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006442527582748244, + "loss": 1.5436, + "theoretical_loss": 3.825306894658266, + "tokens_seen": 669188096 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006442367101303912, + "loss": 1.4732, + "theoretical_loss": 3.8252688377292, + "tokens_seen": 669253632 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006442206619859578, + "loss": 1.5316, + "theoretical_loss": 3.8252307855699943, + "tokens_seen": 669319168 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006442046138415246, + "loss": 1.4934, + "theoretical_loss": 3.8251927381795845, + "tokens_seen": 669384704 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006441885656970912, + "loss": 1.4976, + "theoretical_loss": 3.825154695556906, + "tokens_seen": 669450240 + }, + { + "epoch": 0.2, + "learning_rate": 0.000644172517552658, + "loss": 1.4848, + "theoretical_loss": 3.825116657700894, + "tokens_seen": 669515776 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006441564694082246, + "loss": 1.5841, + "theoretical_loss": 3.8250786246104864, + "tokens_seen": 669581312 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006441404212637914, + "loss": 1.4814, + "theoretical_loss": 3.825040596284619, + "tokens_seen": 669646848 + }, + { + "epoch": 0.2, + "learning_rate": 0.000644124373119358, + "loss": 1.4928, + "theoretical_loss": 3.8250025727222283, + "tokens_seen": 669712384 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006441083249749248, + "loss": 1.5629, + "theoretical_loss": 3.8249645539222525, + "tokens_seen": 669777920 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006440922768304914, + "loss": 1.4689, + "theoretical_loss": 3.824926539883628, + "tokens_seen": 669843456 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006440762286860582, + "loss": 1.4628, + "theoretical_loss": 3.8248885306052935, + "tokens_seen": 669908992 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006440601805416248, + "loss": 1.5698, + "theoretical_loss": 3.8248505260861876, + "tokens_seen": 669974528 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006440441323971916, + "loss": 1.4687, + "theoretical_loss": 3.824812526325248, + "tokens_seen": 670040064 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006440280842527582, + "loss": 1.4871, + "theoretical_loss": 3.824774531321414, + "tokens_seen": 670105600 + }, + { + "epoch": 0.2, + "learning_rate": 0.000644012036108325, + "loss": 1.4336, + "theoretical_loss": 3.8247365410736256, + "tokens_seen": 670171136 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006439959879638916, + "loss": 1.5868, + "theoretical_loss": 3.824698555580822, + "tokens_seen": 670236672 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006439799398194584, + "loss": 1.5304, + "theoretical_loss": 3.8246605748419427, + "tokens_seen": 670302208 + }, + { + "epoch": 0.2, + "learning_rate": 0.000643963891675025, + "loss": 1.4528, + "theoretical_loss": 3.824622598855929, + "tokens_seen": 670367744 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006439478435305918, + "loss": 1.2988, + "theoretical_loss": 3.824584627621721, + "tokens_seen": 670433280 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006439317953861585, + "loss": 1.4011, + "theoretical_loss": 3.8245466611382595, + "tokens_seen": 670498816 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006439157472417252, + "loss": 1.4783, + "theoretical_loss": 3.824508699404486, + "tokens_seen": 670564352 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006438996990972919, + "loss": 1.5529, + "theoretical_loss": 3.8244707424193427, + "tokens_seen": 670629888 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006438836509528586, + "loss": 1.4972, + "theoretical_loss": 3.8244327901817705, + "tokens_seen": 670695424 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006438676028084253, + "loss": 1.5504, + "theoretical_loss": 3.8243948426907135, + "tokens_seen": 670760960 + }, + { + "epoch": 0.2, + "learning_rate": 0.000643851554663992, + "loss": 1.5181, + "theoretical_loss": 3.824356899945113, + "tokens_seen": 670826496 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006438355065195588, + "loss": 1.556, + "theoretical_loss": 3.824318961943912, + "tokens_seen": 670892032 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006438194583751254, + "loss": 1.4416, + "theoretical_loss": 3.8242810286860545, + "tokens_seen": 670957568 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006438034102306922, + "loss": 1.4747, + "theoretical_loss": 3.8242431001704844, + "tokens_seen": 671023104 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006437873620862588, + "loss": 1.5739, + "theoretical_loss": 3.8242051763961458, + "tokens_seen": 671088640 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006437713139418256, + "loss": 1.5047, + "theoretical_loss": 3.824167257361982, + "tokens_seen": 671154176 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006437552657973922, + "loss": 1.4234, + "theoretical_loss": 3.8241293430669385, + "tokens_seen": 671219712 + }, + { + "epoch": 0.2, + "learning_rate": 0.000643739217652959, + "loss": 1.4595, + "theoretical_loss": 3.824091433509961, + "tokens_seen": 671285248 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006437231695085256, + "loss": 1.4629, + "theoretical_loss": 3.824053528689994, + "tokens_seen": 671350784 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006437071213640924, + "loss": 1.5404, + "theoretical_loss": 3.8240156286059834, + "tokens_seen": 671416320 + }, + { + "epoch": 0.2, + "learning_rate": 0.000643691073219659, + "loss": 1.5359, + "theoretical_loss": 3.823977733256875, + "tokens_seen": 671481856 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006436750250752258, + "loss": 1.5575, + "theoretical_loss": 3.8239398426416162, + "tokens_seen": 671547392 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006436589769307924, + "loss": 1.5827, + "theoretical_loss": 3.823901956759153, + "tokens_seen": 671612928 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006436429287863592, + "loss": 1.5461, + "theoretical_loss": 3.8238640756084328, + "tokens_seen": 671678464 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.9925973415374756, + "objective/train/avg_token_score": 0.007402685005217791, + "objective/train/docs_used": 249880, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.3932018280029297, + "objective/train/original_loss": 1.4044262170791626, + "objective/train/theoretical_loss": 3.823826199188403, + "objective/train/tokens_used": 692204000, + "objective/train/unlikelihood_loss": 0.01837296225130558, + "theoretical_loss": 3.823826199188403, + "tokens_seen": 671744000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006436268806419258, + "loss": 1.4277, + "theoretical_loss": 3.823826199188403, + "tokens_seen": 671744000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006436108324974926, + "loss": 1.3776, + "theoretical_loss": 3.8237883274980113, + "tokens_seen": 671809536 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006435947843530592, + "loss": 1.4398, + "theoretical_loss": 3.823750460536206, + "tokens_seen": 671875072 + }, + { + "epoch": 0.2, + "learning_rate": 0.000643578736208626, + "loss": 1.4132, + "theoretical_loss": 3.8237125983019347, + "tokens_seen": 671940608 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006435626880641926, + "loss": 1.5426, + "theoretical_loss": 3.8236747407941474, + "tokens_seen": 672006144 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006435466399197594, + "loss": 1.4813, + "theoretical_loss": 3.8236368880117926, + "tokens_seen": 672071680 + }, + { + "epoch": 0.2, + "learning_rate": 0.000643530591775326, + "loss": 1.4053, + "theoretical_loss": 3.82359903995382, + "tokens_seen": 672137216 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006435145436308928, + "loss": 1.4694, + "theoretical_loss": 3.823561196619179, + "tokens_seen": 672202752 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006434984954864594, + "loss": 1.532, + "theoretical_loss": 3.82352335800682, + "tokens_seen": 672268288 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006434824473420262, + "loss": 1.5191, + "theoretical_loss": 3.8234855241156938, + "tokens_seen": 672333824 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006434663991975928, + "loss": 1.4415, + "theoretical_loss": 3.8234476949447505, + "tokens_seen": 672399360 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006434503510531596, + "loss": 1.5385, + "theoretical_loss": 3.8234098704929416, + "tokens_seen": 672464896 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006434343029087262, + "loss": 1.4023, + "theoretical_loss": 3.823372050759218, + "tokens_seen": 672530432 + }, + { + "epoch": 0.2, + "learning_rate": 0.000643418254764293, + "loss": 1.5361, + "theoretical_loss": 3.823334235742533, + "tokens_seen": 672595968 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006434022066198596, + "loss": 1.5172, + "theoretical_loss": 3.8232964254418373, + "tokens_seen": 672661504 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006433861584754264, + "loss": 1.5284, + "theoretical_loss": 3.823258619856084, + "tokens_seen": 672727040 + }, + { + "epoch": 0.2, + "learning_rate": 0.000643370110330993, + "loss": 1.5671, + "theoretical_loss": 3.8232208189842254, + "tokens_seen": 672792576 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006433540621865598, + "loss": 1.4225, + "theoretical_loss": 3.8231830228252157, + "tokens_seen": 672858112 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006433380140421264, + "loss": 1.5691, + "theoretical_loss": 3.8231452313780068, + "tokens_seen": 672923648 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006433219658976932, + "loss": 1.462, + "theoretical_loss": 3.823107444641554, + "tokens_seen": 672989184 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006433059177532598, + "loss": 1.5335, + "theoretical_loss": 3.8230696626148113, + "tokens_seen": 673054720 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006432898696088266, + "loss": 1.4471, + "theoretical_loss": 3.823031885296732, + "tokens_seen": 673120256 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006432738214643932, + "loss": 1.6201, + "theoretical_loss": 3.822994112686272, + "tokens_seen": 673185792 + }, + { + "epoch": 0.2, + "learning_rate": 0.00064325777331996, + "loss": 1.5352, + "theoretical_loss": 3.8229563447823858, + "tokens_seen": 673251328 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006432417251755266, + "loss": 1.405, + "theoretical_loss": 3.8229185815840294, + "tokens_seen": 673316864 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006432256770310933, + "loss": 1.4913, + "theoretical_loss": 3.8228808230901583, + "tokens_seen": 673382400 + }, + { + "epoch": 0.2, + "learning_rate": 0.00064320962888666, + "loss": 1.4995, + "theoretical_loss": 3.822843069299729, + "tokens_seen": 673447936 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006431935807422267, + "loss": 1.5284, + "theoretical_loss": 3.822805320211698, + "tokens_seen": 673513472 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006431775325977934, + "loss": 1.5426, + "theoretical_loss": 3.8227675758250212, + "tokens_seen": 673579008 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006431614844533601, + "loss": 1.5036, + "theoretical_loss": 3.822729836138657, + "tokens_seen": 673644544 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006431454363089268, + "loss": 1.4059, + "theoretical_loss": 3.8226921011515618, + "tokens_seen": 673710080 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006431293881644935, + "loss": 1.4335, + "theoretical_loss": 3.822654370862694, + "tokens_seen": 673775616 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006431133400200602, + "loss": 1.4575, + "theoretical_loss": 3.8226166452710117, + "tokens_seen": 673841152 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006430972918756269, + "loss": 1.5576, + "theoretical_loss": 3.822578924375473, + "tokens_seen": 673906688 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006430812437311935, + "loss": 1.5212, + "theoretical_loss": 3.8225412081750374, + "tokens_seen": 673972224 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006430651955867603, + "loss": 1.466, + "theoretical_loss": 3.8225034966686637, + "tokens_seen": 674037760 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006430491474423269, + "loss": 1.5341, + "theoretical_loss": 3.822465789855311, + "tokens_seen": 674103296 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006430330992978937, + "loss": 1.5525, + "theoretical_loss": 3.82242808773394, + "tokens_seen": 674168832 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006430170511534603, + "loss": 1.4994, + "theoretical_loss": 3.82239039030351, + "tokens_seen": 674234368 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006430010030090271, + "loss": 1.4931, + "theoretical_loss": 3.822352697562982, + "tokens_seen": 674299904 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006429849548645937, + "loss": 1.3756, + "theoretical_loss": 3.8223150095113163, + "tokens_seen": 674365440 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006429689067201605, + "loss": 1.5411, + "theoretical_loss": 3.822277326147475, + "tokens_seen": 674430976 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006429528585757271, + "loss": 1.445, + "theoretical_loss": 3.822239647470418, + "tokens_seen": 674496512 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006429368104312939, + "loss": 1.4562, + "theoretical_loss": 3.822201973479108, + "tokens_seen": 674562048 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006429207622868605, + "loss": 1.4927, + "theoretical_loss": 3.8221643041725075, + "tokens_seen": 674627584 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006429047141424273, + "loss": 1.5053, + "theoretical_loss": 3.8221266395495785, + "tokens_seen": 674693120 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006428886659979939, + "loss": 1.552, + "theoretical_loss": 3.822088979609284, + "tokens_seen": 674758656 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006428726178535607, + "loss": 1.4112, + "theoretical_loss": 3.8220513243505865, + "tokens_seen": 674824192 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006428565697091273, + "loss": 1.4264, + "theoretical_loss": 3.8220136737724504, + "tokens_seen": 674889728 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006428405215646941, + "loss": 1.3882, + "theoretical_loss": 3.8219760278738386, + "tokens_seen": 674955264 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.992687463760376, + "objective/train/avg_token_score": 0.007312524598091841, + "objective/train/docs_used": 251058, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4895726442337036, + "objective/train/original_loss": 1.502542495727539, + "objective/train/theoretical_loss": 3.821938386653716, + "objective/train/tokens_used": 695480800, + "objective/train/unlikelihood_loss": 0.017061451449990273, + "theoretical_loss": 3.821938386653716, + "tokens_seen": 675020800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006428244734202607, + "loss": 1.4683, + "theoretical_loss": 3.821938386653716, + "tokens_seen": 675020800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006428084252758275, + "loss": 1.4648, + "theoretical_loss": 3.8219007501110465, + "tokens_seen": 675086336 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006427923771313941, + "loss": 1.4961, + "theoretical_loss": 3.8218631182447944, + "tokens_seen": 675151872 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006427763289869609, + "loss": 1.4931, + "theoretical_loss": 3.821825491053926, + "tokens_seen": 675217408 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006427602808425275, + "loss": 1.4421, + "theoretical_loss": 3.821787868537406, + "tokens_seen": 675282944 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006427442326980943, + "loss": 1.4865, + "theoretical_loss": 3.8217502506942003, + "tokens_seen": 675348480 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006427281845536609, + "loss": 1.4273, + "theoretical_loss": 3.821712637523275, + "tokens_seen": 675414016 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006427121364092277, + "loss": 1.433, + "theoretical_loss": 3.8216750290235963, + "tokens_seen": 675479552 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006426960882647943, + "loss": 1.4519, + "theoretical_loss": 3.8216374251941314, + "tokens_seen": 675545088 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006426800401203611, + "loss": 1.5023, + "theoretical_loss": 3.8215998260338466, + "tokens_seen": 675610624 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006426639919759277, + "loss": 1.5663, + "theoretical_loss": 3.8215622315417104, + "tokens_seen": 675676160 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006426479438314945, + "loss": 1.5037, + "theoretical_loss": 3.8215246417166893, + "tokens_seen": 675741696 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006426318956870613, + "loss": 1.4299, + "theoretical_loss": 3.821487056557752, + "tokens_seen": 675807232 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006426158475426279, + "loss": 1.4457, + "theoretical_loss": 3.8214494760638673, + "tokens_seen": 675872768 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006425997993981947, + "loss": 1.4072, + "theoretical_loss": 3.8214119002340032, + "tokens_seen": 675938304 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006425837512537613, + "loss": 1.5301, + "theoretical_loss": 3.821374329067129, + "tokens_seen": 676003840 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006425677031093281, + "loss": 1.4392, + "theoretical_loss": 3.8213367625622143, + "tokens_seen": 676069376 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006425516549648947, + "loss": 1.423, + "theoretical_loss": 3.8212992007182285, + "tokens_seen": 676134912 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006425356068204615, + "loss": 1.5415, + "theoretical_loss": 3.8212616435341413, + "tokens_seen": 676200448 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006425195586760281, + "loss": 1.4785, + "theoretical_loss": 3.821224091008924, + "tokens_seen": 676265984 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006425035105315949, + "loss": 1.4934, + "theoretical_loss": 3.821186543141547, + "tokens_seen": 676331520 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006424874623871615, + "loss": 1.4728, + "theoretical_loss": 3.8211489999309807, + "tokens_seen": 676397056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006424714142427283, + "loss": 1.5119, + "theoretical_loss": 3.821111461376196, + "tokens_seen": 676462592 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006424553660982949, + "loss": 1.5296, + "theoretical_loss": 3.8210739274761663, + "tokens_seen": 676528128 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006424393179538617, + "loss": 1.488, + "theoretical_loss": 3.8210363982298623, + "tokens_seen": 676593664 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006424232698094283, + "loss": 1.5306, + "theoretical_loss": 3.820998873636257, + "tokens_seen": 676659200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006424072216649951, + "loss": 1.5095, + "theoretical_loss": 3.8209613536943223, + "tokens_seen": 676724736 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006423911735205617, + "loss": 1.4114, + "theoretical_loss": 3.8209238384030315, + "tokens_seen": 676790272 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006423751253761285, + "loss": 1.5143, + "theoretical_loss": 3.820886327761358, + "tokens_seen": 676855808 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006423590772316951, + "loss": 1.5268, + "theoretical_loss": 3.820848821768275, + "tokens_seen": 676921344 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006423430290872619, + "loss": 1.3521, + "theoretical_loss": 3.8208113204227576, + "tokens_seen": 676986880 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006423269809428285, + "loss": 1.4537, + "theoretical_loss": 3.8207738237237785, + "tokens_seen": 677052416 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006423109327983953, + "loss": 1.4937, + "theoretical_loss": 3.8207363316703136, + "tokens_seen": 677117952 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006422948846539619, + "loss": 1.3494, + "theoretical_loss": 3.820698844261337, + "tokens_seen": 677183488 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006422788365095287, + "loss": 1.5492, + "theoretical_loss": 3.8206613614958242, + "tokens_seen": 677249024 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006422627883650953, + "loss": 1.5065, + "theoretical_loss": 3.820623883372751, + "tokens_seen": 677314560 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006422467402206621, + "loss": 1.493, + "theoretical_loss": 3.820586409891093, + "tokens_seen": 677380096 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006422306920762287, + "loss": 1.5057, + "theoretical_loss": 3.820548941049826, + "tokens_seen": 677445632 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006422146439317955, + "loss": 1.4121, + "theoretical_loss": 3.820511476847928, + "tokens_seen": 677511168 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006421985957873621, + "loss": 1.4947, + "theoretical_loss": 3.8204740172843747, + "tokens_seen": 677576704 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006421825476429289, + "loss": 1.5014, + "theoretical_loss": 3.820436562358143, + "tokens_seen": 677642240 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006421664994984955, + "loss": 1.5554, + "theoretical_loss": 3.8203991120682117, + "tokens_seen": 677707776 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006421504513540623, + "loss": 1.4545, + "theoretical_loss": 3.8203616664135573, + "tokens_seen": 677773312 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006421344032096289, + "loss": 1.4888, + "theoretical_loss": 3.8203242253931586, + "tokens_seen": 677838848 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006421183550651957, + "loss": 1.3907, + "theoretical_loss": 3.820286789005994, + "tokens_seen": 677904384 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006421023069207623, + "loss": 1.461, + "theoretical_loss": 3.820249357251043, + "tokens_seen": 677969920 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006420862587763291, + "loss": 1.4907, + "theoretical_loss": 3.8202119301272837, + "tokens_seen": 678035456 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006420702106318957, + "loss": 1.4713, + "theoretical_loss": 3.8201745076336957, + "tokens_seen": 678100992 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006420541624874625, + "loss": 1.5481, + "theoretical_loss": 3.8201370897692595, + "tokens_seen": 678166528 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006420381143430291, + "loss": 1.5346, + "theoretical_loss": 3.820099676532955, + "tokens_seen": 678232064 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.9941136837005615, + "objective/train/avg_token_score": 0.005886353552341461, + "objective/train/docs_used": 252282, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4191356897354126, + "objective/train/original_loss": 1.428138017654419, + "objective/train/theoretical_loss": 3.8200622679237615, + "objective/train/tokens_used": 698757600, + "objective/train/unlikelihood_loss": 0.014388896524906158, + "theoretical_loss": 3.8200622679237615, + "tokens_seen": 678297600 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006420220661985959, + "loss": 1.4054, + "theoretical_loss": 3.8200622679237615, + "tokens_seen": 678297600 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006420060180541625, + "loss": 1.4767, + "theoretical_loss": 3.8200248639406613, + "tokens_seen": 678363136 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006419899699097293, + "loss": 1.3771, + "theoretical_loss": 3.819987464582635, + "tokens_seen": 678428672 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006419739217652959, + "loss": 1.5532, + "theoretical_loss": 3.8199500698486633, + "tokens_seen": 678494208 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006419578736208627, + "loss": 1.486, + "theoretical_loss": 3.8199126797377287, + "tokens_seen": 678559744 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006419418254764293, + "loss": 1.5911, + "theoretical_loss": 3.8198752942488134, + "tokens_seen": 678625280 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006419257773319961, + "loss": 1.5387, + "theoretical_loss": 3.8198379133808995, + "tokens_seen": 678690816 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006419097291875627, + "loss": 1.5376, + "theoretical_loss": 3.819800537132969, + "tokens_seen": 678756352 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006418936810431295, + "loss": 1.5831, + "theoretical_loss": 3.8197631655040056, + "tokens_seen": 678821888 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006418776328986961, + "loss": 1.5521, + "theoretical_loss": 3.8197257984929927, + "tokens_seen": 678887424 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006418615847542629, + "loss": 1.5078, + "theoretical_loss": 3.8196884360989136, + "tokens_seen": 678952960 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006418455366098295, + "loss": 1.553, + "theoretical_loss": 3.819651078320753, + "tokens_seen": 679018496 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006418294884653962, + "loss": 1.5666, + "theoretical_loss": 3.819613725157494, + "tokens_seen": 679084032 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006418134403209629, + "loss": 1.5529, + "theoretical_loss": 3.819576376608122, + "tokens_seen": 679149568 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006417973921765296, + "loss": 1.6353, + "theoretical_loss": 3.8195390326716225, + "tokens_seen": 679215104 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006417813440320963, + "loss": 1.3871, + "theoretical_loss": 3.8195016933469796, + "tokens_seen": 679280640 + }, + { + "epoch": 0.21, + "learning_rate": 0.000641765295887663, + "loss": 1.4691, + "theoretical_loss": 3.819464358633179, + "tokens_seen": 679346176 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006417492477432297, + "loss": 1.4524, + "theoretical_loss": 3.819427028529208, + "tokens_seen": 679411712 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006417331995987964, + "loss": 1.4833, + "theoretical_loss": 3.8193897030340516, + "tokens_seen": 679477248 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006417171514543631, + "loss": 1.6252, + "theoretical_loss": 3.819352382146696, + "tokens_seen": 679542784 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006417011033099298, + "loss": 1.4729, + "theoretical_loss": 3.819315065866129, + "tokens_seen": 679608320 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006416850551654964, + "loss": 1.4928, + "theoretical_loss": 3.8192777541913374, + "tokens_seen": 679673856 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006416690070210632, + "loss": 1.4431, + "theoretical_loss": 3.819240447121309, + "tokens_seen": 679739392 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006416529588766298, + "loss": 1.5307, + "theoretical_loss": 3.8192031446550314, + "tokens_seen": 679804928 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006416369107321966, + "loss": 1.4661, + "theoretical_loss": 3.819165846791493, + "tokens_seen": 679870464 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006416208625877632, + "loss": 1.5321, + "theoretical_loss": 3.8191285535296813, + "tokens_seen": 679936000 + }, + { + "epoch": 0.21, + "learning_rate": 0.00064160481444333, + "loss": 1.5044, + "theoretical_loss": 3.819091264868586, + "tokens_seen": 680001536 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006415887662988966, + "loss": 1.4914, + "theoretical_loss": 3.8190539808071966, + "tokens_seen": 680067072 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006415727181544634, + "loss": 1.4186, + "theoretical_loss": 3.819016701344501, + "tokens_seen": 680132608 + }, + { + "epoch": 0.21, + "learning_rate": 0.00064155667001003, + "loss": 1.4959, + "theoretical_loss": 3.8189794264794905, + "tokens_seen": 680198144 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006415406218655968, + "loss": 1.5647, + "theoretical_loss": 3.8189421562111545, + "tokens_seen": 680263680 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006415245737211634, + "loss": 1.4, + "theoretical_loss": 3.8189048905384837, + "tokens_seen": 680329216 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006415085255767302, + "loss": 1.4692, + "theoretical_loss": 3.8188676294604686, + "tokens_seen": 680394752 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006414924774322968, + "loss": 1.5395, + "theoretical_loss": 3.8188303729761, + "tokens_seen": 680460288 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006414764292878636, + "loss": 1.4202, + "theoretical_loss": 3.818793121084369, + "tokens_seen": 680525824 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006414603811434302, + "loss": 1.5191, + "theoretical_loss": 3.818755873784268, + "tokens_seen": 680591360 + }, + { + "epoch": 0.21, + "learning_rate": 0.000641444332998997, + "loss": 1.5417, + "theoretical_loss": 3.8187186310747885, + "tokens_seen": 680656896 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006414282848545638, + "loss": 1.5026, + "theoretical_loss": 3.8186813929549235, + "tokens_seen": 680722432 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006414122367101304, + "loss": 1.4149, + "theoretical_loss": 3.818644159423665, + "tokens_seen": 680787968 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006413961885656972, + "loss": 1.5168, + "theoretical_loss": 3.818606930480006, + "tokens_seen": 680853504 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006413801404212638, + "loss": 1.521, + "theoretical_loss": 3.8185697061229398, + "tokens_seen": 680919040 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006413640922768306, + "loss": 1.3762, + "theoretical_loss": 3.8185324863514594, + "tokens_seen": 680984576 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006413480441323972, + "loss": 1.5796, + "theoretical_loss": 3.8184952711645597, + "tokens_seen": 681050112 + }, + { + "epoch": 0.21, + "learning_rate": 0.000641331995987964, + "loss": 1.5134, + "theoretical_loss": 3.8184580605612344, + "tokens_seen": 681115648 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006413159478435306, + "loss": 1.3987, + "theoretical_loss": 3.818420854540478, + "tokens_seen": 681181184 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006412998996990974, + "loss": 1.5274, + "theoretical_loss": 3.818383653101286, + "tokens_seen": 681246720 + }, + { + "epoch": 0.21, + "learning_rate": 0.000641283851554664, + "loss": 1.4804, + "theoretical_loss": 3.8183464562426526, + "tokens_seen": 681312256 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006412678034102308, + "loss": 1.5369, + "theoretical_loss": 3.8183092639635734, + "tokens_seen": 681377792 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006412517552657974, + "loss": 1.526, + "theoretical_loss": 3.8182720762630447, + "tokens_seen": 681443328 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006412357071213642, + "loss": 1.5236, + "theoretical_loss": 3.8182348931400627, + "tokens_seen": 681508864 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.9955111742019653, + "objective/train/avg_token_score": 0.004488764330744743, + "objective/train/docs_used": 253498, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5036208629608154, + "objective/train/original_loss": 1.5109496116638184, + "objective/train/theoretical_loss": 3.818197714593623, + "objective/train/tokens_used": 702034400, + "objective/train/unlikelihood_loss": 0.011985911056399345, + "theoretical_loss": 3.818197714593623, + "tokens_seen": 681574400 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006412196589769308, + "loss": 1.5269, + "theoretical_loss": 3.818197714593623, + "tokens_seen": 681574400 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006412036108324976, + "loss": 1.469, + "theoretical_loss": 3.818160540622723, + "tokens_seen": 681639936 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006411875626880642, + "loss": 1.6036, + "theoretical_loss": 3.8181233712263594, + "tokens_seen": 681705472 + }, + { + "epoch": 0.21, + "learning_rate": 0.000641171514543631, + "loss": 1.5075, + "theoretical_loss": 3.81808620640353, + "tokens_seen": 681771008 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006411554663991976, + "loss": 1.4833, + "theoretical_loss": 3.8180490461532317, + "tokens_seen": 681836544 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006411394182547644, + "loss": 1.5559, + "theoretical_loss": 3.818011890474464, + "tokens_seen": 681902080 + }, + { + "epoch": 0.21, + "learning_rate": 0.000641123370110331, + "loss": 1.4815, + "theoretical_loss": 3.817974739366224, + "tokens_seen": 681967616 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006411073219658978, + "loss": 1.5141, + "theoretical_loss": 3.8179375928275103, + "tokens_seen": 682033152 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006410912738214644, + "loss": 1.5604, + "theoretical_loss": 3.817900450857322, + "tokens_seen": 682098688 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006410752256770312, + "loss": 1.454, + "theoretical_loss": 3.817863313454659, + "tokens_seen": 682164224 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006410591775325978, + "loss": 1.5651, + "theoretical_loss": 3.8178261806185207, + "tokens_seen": 682229760 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006410431293881646, + "loss": 1.563, + "theoretical_loss": 3.817789052347906, + "tokens_seen": 682295296 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006410270812437312, + "loss": 1.5608, + "theoretical_loss": 3.8177519286418162, + "tokens_seen": 682360832 + }, + { + "epoch": 0.21, + "learning_rate": 0.000641011033099298, + "loss": 1.5135, + "theoretical_loss": 3.817714809499252, + "tokens_seen": 682426368 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006409949849548646, + "loss": 1.5743, + "theoretical_loss": 3.8176776949192135, + "tokens_seen": 682491904 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006409789368104314, + "loss": 1.519, + "theoretical_loss": 3.8176405849007016, + "tokens_seen": 682557440 + }, + { + "epoch": 0.21, + "learning_rate": 0.000640962888665998, + "loss": 1.4239, + "theoretical_loss": 3.8176034794427185, + "tokens_seen": 682622976 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006409468405215648, + "loss": 1.5316, + "theoretical_loss": 3.8175663785442655, + "tokens_seen": 682688512 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006409307923771314, + "loss": 1.5733, + "theoretical_loss": 3.8175292822043456, + "tokens_seen": 682754048 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006409147442326982, + "loss": 1.5837, + "theoretical_loss": 3.81749219042196, + "tokens_seen": 682819584 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006408986960882648, + "loss": 1.3869, + "theoretical_loss": 3.8174551031961124, + "tokens_seen": 682885120 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006408826479438316, + "loss": 1.4922, + "theoretical_loss": 3.817418020525805, + "tokens_seen": 682950656 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006408665997993982, + "loss": 1.4616, + "theoretical_loss": 3.817380942410042, + "tokens_seen": 683016192 + }, + { + "epoch": 0.21, + "learning_rate": 0.000640850551654965, + "loss": 1.4406, + "theoretical_loss": 3.8173438688478267, + "tokens_seen": 683081728 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006408345035105316, + "loss": 1.4749, + "theoretical_loss": 3.817306799838163, + "tokens_seen": 683147264 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006408184553660984, + "loss": 1.5213, + "theoretical_loss": 3.817269735380055, + "tokens_seen": 683212800 + }, + { + "epoch": 0.21, + "learning_rate": 0.000640802407221665, + "loss": 1.4876, + "theoretical_loss": 3.817232675472508, + "tokens_seen": 683278336 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006407863590772318, + "loss": 1.5321, + "theoretical_loss": 3.817195620114526, + "tokens_seen": 683343872 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006407703109327984, + "loss": 1.5599, + "theoretical_loss": 3.817158569305115, + "tokens_seen": 683409408 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006407542627883652, + "loss": 1.4521, + "theoretical_loss": 3.817121523043281, + "tokens_seen": 683474944 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006407382146439318, + "loss": 1.6757, + "theoretical_loss": 3.8170844813280285, + "tokens_seen": 683540480 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006407221664994986, + "loss": 1.5215, + "theoretical_loss": 3.817047444158364, + "tokens_seen": 683606016 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006407061183550652, + "loss": 1.4359, + "theoretical_loss": 3.8170104115332943, + "tokens_seen": 683671552 + }, + { + "epoch": 0.21, + "learning_rate": 0.000640690070210632, + "loss": 1.5009, + "theoretical_loss": 3.816973383451827, + "tokens_seen": 683737088 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006406740220661986, + "loss": 1.5027, + "theoretical_loss": 3.816936359912968, + "tokens_seen": 683802624 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006406579739217654, + "loss": 1.5575, + "theoretical_loss": 3.8168993409157252, + "tokens_seen": 683868160 + }, + { + "epoch": 0.21, + "learning_rate": 0.000640641925777332, + "loss": 1.4909, + "theoretical_loss": 3.8168623264591064, + "tokens_seen": 683933696 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006406258776328988, + "loss": 1.4943, + "theoretical_loss": 3.8168253165421198, + "tokens_seen": 683999232 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006406098294884654, + "loss": 1.6141, + "theoretical_loss": 3.816788311163773, + "tokens_seen": 684064768 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006405937813440322, + "loss": 1.4882, + "theoretical_loss": 3.8167513103230757, + "tokens_seen": 684130304 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006405777331995988, + "loss": 1.4507, + "theoretical_loss": 3.816714314019036, + "tokens_seen": 684195840 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006405616850551656, + "loss": 1.6223, + "theoretical_loss": 3.816677322250664, + "tokens_seen": 684261376 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006405456369107322, + "loss": 1.3812, + "theoretical_loss": 3.8166403350169684, + "tokens_seen": 684326912 + }, + { + "epoch": 0.21, + "learning_rate": 0.000640529588766299, + "loss": 1.4349, + "theoretical_loss": 3.81660335231696, + "tokens_seen": 684392448 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006405135406218656, + "loss": 1.5241, + "theoretical_loss": 3.816566374149648, + "tokens_seen": 684457984 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006404974924774324, + "loss": 1.4592, + "theoretical_loss": 3.816529400514044, + "tokens_seen": 684523520 + }, + { + "epoch": 0.21, + "learning_rate": 0.000640481444332999, + "loss": 1.4983, + "theoretical_loss": 3.8164924314091584, + "tokens_seen": 684589056 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006404653961885658, + "loss": 1.4374, + "theoretical_loss": 3.816455466834003, + "tokens_seen": 684654592 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006404493480441324, + "loss": 1.4967, + "theoretical_loss": 3.8164185067875875, + "tokens_seen": 684720128 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006404332998996991, + "loss": 1.5004, + "theoretical_loss": 3.8163815512689254, + "tokens_seen": 684785664 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.9944674372673035, + "objective/train/avg_token_score": 0.00553258927538991, + "objective/train/docs_used": 254674, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4801346063613892, + "objective/train/original_loss": 1.488675594329834, + "objective/train/theoretical_loss": 3.816344600277029, + "objective/train/tokens_used": 705311200, + "objective/train/unlikelihood_loss": 0.013560554943978786, + "theoretical_loss": 3.816344600277029, + "tokens_seen": 684851200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006404172517552658, + "loss": 1.4844, + "theoretical_loss": 3.816344600277029, + "tokens_seen": 684851200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006404012036108325, + "loss": 1.5177, + "theoretical_loss": 3.8163076538109086, + "tokens_seen": 684916736 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006403851554663992, + "loss": 1.5374, + "theoretical_loss": 3.816270711869579, + "tokens_seen": 684982272 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006403691073219659, + "loss": 1.4371, + "theoretical_loss": 3.8162337744520523, + "tokens_seen": 685047808 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006403530591775326, + "loss": 1.447, + "theoretical_loss": 3.816196841557342, + "tokens_seen": 685113344 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006403370110330993, + "loss": 1.4495, + "theoretical_loss": 3.816159913184462, + "tokens_seen": 685178880 + }, + { + "epoch": 0.21, + "learning_rate": 0.000640320962888666, + "loss": 1.5148, + "theoretical_loss": 3.8161229893324258, + "tokens_seen": 685244416 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006403049147442327, + "loss": 1.5458, + "theoretical_loss": 3.8160860700002477, + "tokens_seen": 685309952 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006402888665997993, + "loss": 1.4688, + "theoretical_loss": 3.816049155186943, + "tokens_seen": 685375488 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006402728184553661, + "loss": 1.5091, + "theoretical_loss": 3.8160122448915255, + "tokens_seen": 685441024 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006402567703109327, + "loss": 1.4751, + "theoretical_loss": 3.815975339113011, + "tokens_seen": 685506560 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006402407221664995, + "loss": 1.5949, + "theoretical_loss": 3.8159384378504146, + "tokens_seen": 685572096 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006402246740220662, + "loss": 1.4439, + "theoretical_loss": 3.8159015411027526, + "tokens_seen": 685637632 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006402086258776329, + "loss": 1.4959, + "theoretical_loss": 3.815864648869041, + "tokens_seen": 685703168 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006401925777331996, + "loss": 1.5312, + "theoretical_loss": 3.815827761148296, + "tokens_seen": 685768704 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006401765295887663, + "loss": 1.4599, + "theoretical_loss": 3.8157908779395346, + "tokens_seen": 685834240 + }, + { + "epoch": 0.21, + "learning_rate": 0.000640160481444333, + "loss": 1.485, + "theoretical_loss": 3.8157539992417737, + "tokens_seen": 685899776 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006401444332998997, + "loss": 1.6482, + "theoretical_loss": 3.8157171250540305, + "tokens_seen": 685965312 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006401283851554664, + "loss": 1.5556, + "theoretical_loss": 3.815680255375323, + "tokens_seen": 686030848 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006401123370110331, + "loss": 1.3829, + "theoretical_loss": 3.815643390204669, + "tokens_seen": 686096384 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006400962888665998, + "loss": 1.5054, + "theoretical_loss": 3.8156065295410864, + "tokens_seen": 686161920 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006400802407221665, + "loss": 1.5056, + "theoretical_loss": 3.8155696733835947, + "tokens_seen": 686227456 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006400641925777332, + "loss": 1.5663, + "theoretical_loss": 3.8155328217312117, + "tokens_seen": 686292992 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006400481444332999, + "loss": 1.5447, + "theoretical_loss": 3.815495974582957, + "tokens_seen": 686358528 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006400320962888667, + "loss": 1.399, + "theoretical_loss": 3.8154591319378506, + "tokens_seen": 686424064 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006400160481444333, + "loss": 1.5581, + "theoretical_loss": 3.815422293794912, + "tokens_seen": 686489600 + }, + { + "epoch": 0.21, + "learning_rate": 0.00064, + "loss": 1.4782, + "theoretical_loss": 3.8153854601531605, + "tokens_seen": 686555136 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006399839518555667, + "loss": 1.5016, + "theoretical_loss": 3.815348631011618, + "tokens_seen": 686620672 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006399679037111335, + "loss": 1.4375, + "theoretical_loss": 3.8153118063693037, + "tokens_seen": 686686208 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006399518555667001, + "loss": 1.4504, + "theoretical_loss": 3.8152749862252398, + "tokens_seen": 686751744 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006399358074222669, + "loss": 1.4771, + "theoretical_loss": 3.8152381705784473, + "tokens_seen": 686817280 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006399197592778335, + "loss": 1.4899, + "theoretical_loss": 3.815201359427948, + "tokens_seen": 686882816 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006399037111334003, + "loss": 1.5337, + "theoretical_loss": 3.815164552772763, + "tokens_seen": 686948352 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006398876629889669, + "loss": 1.4488, + "theoretical_loss": 3.8151277506119152, + "tokens_seen": 687013888 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006398716148445337, + "loss": 1.4455, + "theoretical_loss": 3.8150909529444275, + "tokens_seen": 687079424 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006398555667001003, + "loss": 1.464, + "theoretical_loss": 3.815054159769322, + "tokens_seen": 687144960 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006398395185556671, + "loss": 1.5831, + "theoretical_loss": 3.8150173710856228, + "tokens_seen": 687210496 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006398234704112337, + "loss": 1.4358, + "theoretical_loss": 3.8149805868923528, + "tokens_seen": 687276032 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006398074222668005, + "loss": 1.4234, + "theoretical_loss": 3.814943807188536, + "tokens_seen": 687341568 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006397913741223671, + "loss": 1.5162, + "theoretical_loss": 3.8149070319731955, + "tokens_seen": 687407104 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006397753259779339, + "loss": 1.6448, + "theoretical_loss": 3.8148702612453573, + "tokens_seen": 687472640 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006397592778335005, + "loss": 1.5654, + "theoretical_loss": 3.814833495004045, + "tokens_seen": 687538176 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006397432296890673, + "loss": 1.4493, + "theoretical_loss": 3.8147967332482846, + "tokens_seen": 687603712 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006397271815446339, + "loss": 1.6487, + "theoretical_loss": 3.8147599759771, + "tokens_seen": 687669248 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006397111334002007, + "loss": 1.498, + "theoretical_loss": 3.814723223189518, + "tokens_seen": 687734784 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006396950852557673, + "loss": 1.4644, + "theoretical_loss": 3.8146864748845637, + "tokens_seen": 687800320 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006396790371113341, + "loss": 1.3672, + "theoretical_loss": 3.814649731061264, + "tokens_seen": 687865856 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006396629889669007, + "loss": 1.4863, + "theoretical_loss": 3.8146129917186453, + "tokens_seen": 687931392 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006396469408224675, + "loss": 1.4238, + "theoretical_loss": 3.8145762568557338, + "tokens_seen": 687996928 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006396308926780341, + "loss": 1.5468, + "theoretical_loss": 3.814539526471558, + "tokens_seen": 688062464 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.9902955889701843, + "objective/train/avg_token_score": 0.00970442034304142, + "objective/train/docs_used": 255827, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4563153982162476, + "objective/train/original_loss": 1.4676415920257568, + "objective/train/theoretical_loss": 3.8145028005651436, + "objective/train/tokens_used": 708588000, + "objective/train/unlikelihood_loss": 0.028517017140984535, + "theoretical_loss": 3.8145028005651436, + "tokens_seen": 688128000 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006396148445336009, + "loss": 1.4957, + "theoretical_loss": 3.8145028005651436, + "tokens_seen": 688128000 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006395987963891675, + "loss": 1.486, + "theoretical_loss": 3.8144660791355194, + "tokens_seen": 688193536 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006395827482447343, + "loss": 1.5243, + "theoretical_loss": 3.8144293621817136, + "tokens_seen": 688259072 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006395667001003009, + "loss": 1.5095, + "theoretical_loss": 3.8143926497027536, + "tokens_seen": 688324608 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006395506519558677, + "loss": 1.4952, + "theoretical_loss": 3.814355941697669, + "tokens_seen": 688390144 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006395346038114343, + "loss": 1.4759, + "theoretical_loss": 3.814319238165489, + "tokens_seen": 688455680 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006395185556670011, + "loss": 1.5602, + "theoretical_loss": 3.8142825391052417, + "tokens_seen": 688521216 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006395025075225677, + "loss": 1.494, + "theoretical_loss": 3.8142458445159573, + "tokens_seen": 688586752 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006394864593781345, + "loss": 1.4869, + "theoretical_loss": 3.8142091543966656, + "tokens_seen": 688652288 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006394704112337011, + "loss": 1.4039, + "theoretical_loss": 3.814172468746397, + "tokens_seen": 688717824 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006394543630892679, + "loss": 1.5549, + "theoretical_loss": 3.8141357875641813, + "tokens_seen": 688783360 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006394383149448345, + "loss": 1.457, + "theoretical_loss": 3.81409911084905, + "tokens_seen": 688848896 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006394222668004013, + "loss": 1.518, + "theoretical_loss": 3.8140624386000344, + "tokens_seen": 688914432 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006394062186559679, + "loss": 1.4634, + "theoretical_loss": 3.8140257708161647, + "tokens_seen": 688979968 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006393901705115347, + "loss": 1.5428, + "theoretical_loss": 3.8139891074964734, + "tokens_seen": 689045504 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006393741223671013, + "loss": 1.4946, + "theoretical_loss": 3.8139524486399927, + "tokens_seen": 689111040 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006393580742226681, + "loss": 1.5052, + "theoretical_loss": 3.813915794245754, + "tokens_seen": 689176576 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006393420260782347, + "loss": 1.3937, + "theoretical_loss": 3.813879144312791, + "tokens_seen": 689242112 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006393259779338015, + "loss": 1.513, + "theoretical_loss": 3.8138424988401365, + "tokens_seen": 689307648 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006393099297893681, + "loss": 1.6331, + "theoretical_loss": 3.8138058578268224, + "tokens_seen": 689373184 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006392938816449349, + "loss": 1.4328, + "theoretical_loss": 3.8137692212718832, + "tokens_seen": 689438720 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006392778335005015, + "loss": 1.5189, + "theoretical_loss": 3.8137325891743528, + "tokens_seen": 689504256 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006392617853560683, + "loss": 1.5136, + "theoretical_loss": 3.8136959615332646, + "tokens_seen": 689569792 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006392457372116349, + "loss": 1.4702, + "theoretical_loss": 3.813659338347654, + "tokens_seen": 689635328 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006392296890672017, + "loss": 1.5417, + "theoretical_loss": 3.8136227196165544, + "tokens_seen": 689700864 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006392136409227683, + "loss": 1.5133, + "theoretical_loss": 3.813586105339002, + "tokens_seen": 689766400 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006391975927783351, + "loss": 1.5307, + "theoretical_loss": 3.813549495514031, + "tokens_seen": 689831936 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006391815446339017, + "loss": 1.4878, + "theoretical_loss": 3.8135128901406783, + "tokens_seen": 689897472 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006391654964894685, + "loss": 1.5073, + "theoretical_loss": 3.813476289217979, + "tokens_seen": 689963008 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006391494483450351, + "loss": 1.4121, + "theoretical_loss": 3.813439692744969, + "tokens_seen": 690028544 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006391334002006019, + "loss": 1.5978, + "theoretical_loss": 3.813403100720686, + "tokens_seen": 690094080 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006391173520561685, + "loss": 1.4741, + "theoretical_loss": 3.813366513144165, + "tokens_seen": 690159616 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006391013039117353, + "loss": 1.4612, + "theoretical_loss": 3.8133299300144445, + "tokens_seen": 690225152 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006390852557673019, + "loss": 1.4996, + "theoretical_loss": 3.813293351330562, + "tokens_seen": 690290688 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006390692076228687, + "loss": 1.5502, + "theoretical_loss": 3.8132567770915546, + "tokens_seen": 690356224 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006390531594784353, + "loss": 1.5872, + "theoretical_loss": 3.8132202072964603, + "tokens_seen": 690421760 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006390371113340021, + "loss": 1.4665, + "theoretical_loss": 3.8131836419443172, + "tokens_seen": 690487296 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006390210631895687, + "loss": 1.4447, + "theoretical_loss": 3.8131470810341646, + "tokens_seen": 690552832 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006390050150451354, + "loss": 1.5262, + "theoretical_loss": 3.8131105245650403, + "tokens_seen": 690618368 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006389889669007021, + "loss": 1.5399, + "theoretical_loss": 3.8130739725359852, + "tokens_seen": 690683904 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006389729187562688, + "loss": 1.5769, + "theoretical_loss": 3.8130374249460375, + "tokens_seen": 690749440 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006389568706118355, + "loss": 1.5363, + "theoretical_loss": 3.8130008817942374, + "tokens_seen": 690814976 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006389408224674022, + "loss": 1.4357, + "theoretical_loss": 3.8129643430796243, + "tokens_seen": 690880512 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006389247743229689, + "loss": 1.3892, + "theoretical_loss": 3.8129278088012395, + "tokens_seen": 690946048 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006389087261785356, + "loss": 1.4392, + "theoretical_loss": 3.8128912789581237, + "tokens_seen": 691011584 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006388926780341023, + "loss": 1.6105, + "theoretical_loss": 3.8128547535493174, + "tokens_seen": 691077120 + }, + { + "epoch": 0.21, + "learning_rate": 0.000638876629889669, + "loss": 1.5767, + "theoretical_loss": 3.8128182325738624, + "tokens_seen": 691142656 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006388605817452357, + "loss": 1.6202, + "theoretical_loss": 3.8127817160307993, + "tokens_seen": 691208192 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006388445336008024, + "loss": 1.4065, + "theoretical_loss": 3.812745203919171, + "tokens_seen": 691273728 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006388284854563691, + "loss": 1.494, + "theoretical_loss": 3.8127086962380194, + "tokens_seen": 691339264 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.9958645701408386, + "objective/train/avg_token_score": 0.004135389346629381, + "objective/train/docs_used": 257040, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4831666946411133, + "objective/train/original_loss": 1.489225149154663, + "objective/train/theoretical_loss": 3.812672192986387, + "objective/train/tokens_used": 711864800, + "objective/train/unlikelihood_loss": 0.010791855864226818, + "theoretical_loss": 3.812672192986387, + "tokens_seen": 691404800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006388124373119358, + "loss": 1.5045, + "theoretical_loss": 3.812672192986387, + "tokens_seen": 691404800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006387963891675025, + "loss": 1.3748, + "theoretical_loss": 3.8126356941633164, + "tokens_seen": 691470336 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006387803410230692, + "loss": 1.4997, + "theoretical_loss": 3.8125991997678503, + "tokens_seen": 691535872 + }, + { + "epoch": 0.21, + "learning_rate": 0.000638764292878636, + "loss": 1.4152, + "theoretical_loss": 3.8125627097990327, + "tokens_seen": 691601408 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006387482447342026, + "loss": 1.4659, + "theoretical_loss": 3.8125262242559073, + "tokens_seen": 691666944 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006387321965897693, + "loss": 1.5046, + "theoretical_loss": 3.812489743137518, + "tokens_seen": 691732480 + }, + { + "epoch": 0.21, + "learning_rate": 0.000638716148445336, + "loss": 1.4175, + "theoretical_loss": 3.8124532664429087, + "tokens_seen": 691798016 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006387001003009027, + "loss": 1.3962, + "theoretical_loss": 3.8124167941711242, + "tokens_seen": 691863552 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006386840521564694, + "loss": 1.5368, + "theoretical_loss": 3.8123803263212097, + "tokens_seen": 691929088 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006386680040120362, + "loss": 1.5451, + "theoretical_loss": 3.8123438628922095, + "tokens_seen": 691994624 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006386519558676028, + "loss": 1.451, + "theoretical_loss": 3.8123074038831697, + "tokens_seen": 692060160 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006386359077231696, + "loss": 1.5469, + "theoretical_loss": 3.8122709492931364, + "tokens_seen": 692125696 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006386198595787362, + "loss": 1.45, + "theoretical_loss": 3.8122344991211543, + "tokens_seen": 692191232 + }, + { + "epoch": 0.21, + "learning_rate": 0.000638603811434303, + "loss": 1.5139, + "theoretical_loss": 3.8121980533662714, + "tokens_seen": 692256768 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006385877632898696, + "loss": 1.3888, + "theoretical_loss": 3.8121616120275332, + "tokens_seen": 692322304 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006385717151454364, + "loss": 1.5522, + "theoretical_loss": 3.812125175103987, + "tokens_seen": 692387840 + }, + { + "epoch": 0.21, + "learning_rate": 0.000638555667001003, + "loss": 1.4672, + "theoretical_loss": 3.81208874259468, + "tokens_seen": 692453376 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006385396188565698, + "loss": 1.4873, + "theoretical_loss": 3.8120523144986596, + "tokens_seen": 692518912 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006385235707121364, + "loss": 1.5068, + "theoretical_loss": 3.8120158908149735, + "tokens_seen": 692584448 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006385075225677032, + "loss": 1.6086, + "theoretical_loss": 3.8119794715426707, + "tokens_seen": 692649984 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006384914744232698, + "loss": 1.5061, + "theoretical_loss": 3.8119430566807986, + "tokens_seen": 692715520 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006384754262788366, + "loss": 1.5388, + "theoretical_loss": 3.8119066462284064, + "tokens_seen": 692781056 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006384593781344032, + "loss": 1.4794, + "theoretical_loss": 3.8118702401845423, + "tokens_seen": 692846592 + }, + { + "epoch": 0.21, + "learning_rate": 0.00063844332998997, + "loss": 1.4743, + "theoretical_loss": 3.8118338385482575, + "tokens_seen": 692912128 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006384272818455366, + "loss": 1.5386, + "theoretical_loss": 3.8117974413185993, + "tokens_seen": 692977664 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006384112337011034, + "loss": 1.4929, + "theoretical_loss": 3.8117610484946187, + "tokens_seen": 693043200 + }, + { + "epoch": 0.21, + "learning_rate": 0.00063839518555667, + "loss": 1.5882, + "theoretical_loss": 3.8117246600753667, + "tokens_seen": 693108736 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006383791374122368, + "loss": 1.6196, + "theoretical_loss": 3.8116882760598925, + "tokens_seen": 693174272 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006383630892678034, + "loss": 1.4827, + "theoretical_loss": 3.811651896447247, + "tokens_seen": 693239808 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006383470411233702, + "loss": 1.51, + "theoretical_loss": 3.8116155212364817, + "tokens_seen": 693305344 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006383309929789368, + "loss": 1.4475, + "theoretical_loss": 3.811579150426648, + "tokens_seen": 693370880 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006383149448345036, + "loss": 1.5412, + "theoretical_loss": 3.811542784016797, + "tokens_seen": 693436416 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006382988966900702, + "loss": 1.5476, + "theoretical_loss": 3.811506422005981, + "tokens_seen": 693501952 + }, + { + "epoch": 0.21, + "learning_rate": 0.000638282848545637, + "loss": 1.5104, + "theoretical_loss": 3.8114700643932524, + "tokens_seen": 693567488 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006382668004012036, + "loss": 1.4571, + "theoretical_loss": 3.8114337111776635, + "tokens_seen": 693633024 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006382507522567704, + "loss": 1.5409, + "theoretical_loss": 3.8113973623582673, + "tokens_seen": 693698560 + }, + { + "epoch": 0.21, + "learning_rate": 0.000638234704112337, + "loss": 1.5036, + "theoretical_loss": 3.8113610179341166, + "tokens_seen": 693764096 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006382186559679038, + "loss": 1.5515, + "theoretical_loss": 3.811324677904265, + "tokens_seen": 693829632 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006382026078234704, + "loss": 1.5009, + "theoretical_loss": 3.811288342267767, + "tokens_seen": 693895168 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006381865596790372, + "loss": 1.5348, + "theoretical_loss": 3.811252011023675, + "tokens_seen": 693960704 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006381705115346038, + "loss": 1.4175, + "theoretical_loss": 3.811215684171044, + "tokens_seen": 694026240 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006381544633901706, + "loss": 1.5803, + "theoretical_loss": 3.8111793617089296, + "tokens_seen": 694091776 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006381384152457372, + "loss": 1.5206, + "theoretical_loss": 3.8111430436363856, + "tokens_seen": 694157312 + }, + { + "epoch": 0.21, + "learning_rate": 0.000638122367101304, + "loss": 1.6707, + "theoretical_loss": 3.8111067299524666, + "tokens_seen": 694222848 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006381063189568706, + "loss": 1.4848, + "theoretical_loss": 3.81107042065623, + "tokens_seen": 694288384 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006380902708124374, + "loss": 1.4166, + "theoretical_loss": 3.811034115746729, + "tokens_seen": 694353920 + }, + { + "epoch": 0.21, + "learning_rate": 0.000638074222668004, + "loss": 1.4851, + "theoretical_loss": 3.810997815223022, + "tokens_seen": 694419456 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006380581745235708, + "loss": 1.601, + "theoretical_loss": 3.810961519084164, + "tokens_seen": 694484992 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006380421263791374, + "loss": 1.4328, + "theoretical_loss": 3.8109252273292125, + "tokens_seen": 694550528 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006380260782347042, + "loss": 1.4269, + "theoretical_loss": 3.8108889399572234, + "tokens_seen": 694616064 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.9930875301361084, + "objective/train/avg_token_score": 0.0069124954752624035, + "objective/train/docs_used": 258341, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5183489322662354, + "objective/train/original_loss": 1.5304009914398193, + "objective/train/theoretical_loss": 3.810852656967255, + "objective/train/tokens_used": 715141600, + "objective/train/unlikelihood_loss": 0.017739487811923027, + "theoretical_loss": 3.810852656967255, + "tokens_seen": 694681600 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006380100300902708, + "loss": 1.5983, + "theoretical_loss": 3.810852656967255, + "tokens_seen": 694681600 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006379939819458376, + "loss": 1.4976, + "theoretical_loss": 3.8108163783583637, + "tokens_seen": 694747136 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006379779338014042, + "loss": 1.5329, + "theoretical_loss": 3.8107801041296083, + "tokens_seen": 694812672 + }, + { + "epoch": 0.21, + "learning_rate": 0.000637961885656971, + "loss": 1.575, + "theoretical_loss": 3.8107438342800464, + "tokens_seen": 694878208 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006379458375125376, + "loss": 1.5117, + "theoretical_loss": 3.810707568808736, + "tokens_seen": 694943744 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006379297893681044, + "loss": 1.4922, + "theoretical_loss": 3.8106713077147365, + "tokens_seen": 695009280 + }, + { + "epoch": 0.21, + "learning_rate": 0.000637913741223671, + "loss": 1.5566, + "theoretical_loss": 3.810635050997107, + "tokens_seen": 695074816 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006378976930792378, + "loss": 1.4173, + "theoretical_loss": 3.810598798654906, + "tokens_seen": 695140352 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006378816449348044, + "loss": 1.561, + "theoretical_loss": 3.8105625506871936, + "tokens_seen": 695205888 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006378655967903712, + "loss": 1.4951, + "theoretical_loss": 3.810526307093029, + "tokens_seen": 695271424 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006378495486459378, + "loss": 1.4136, + "theoretical_loss": 3.810490067871473, + "tokens_seen": 695336960 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006378335005015046, + "loss": 1.5663, + "theoretical_loss": 3.810453833021586, + "tokens_seen": 695402496 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006378174523570713, + "loss": 1.4537, + "theoretical_loss": 3.810417602542428, + "tokens_seen": 695468032 + }, + { + "epoch": 0.21, + "learning_rate": 0.000637801404212638, + "loss": 1.47, + "theoretical_loss": 3.8103813764330607, + "tokens_seen": 695533568 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006377853560682047, + "loss": 1.3991, + "theoretical_loss": 3.8103451546925453, + "tokens_seen": 695599104 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006377693079237714, + "loss": 1.6217, + "theoretical_loss": 3.8103089373199435, + "tokens_seen": 695664640 + }, + { + "epoch": 0.21, + "learning_rate": 0.000637753259779338, + "loss": 1.3926, + "theoretical_loss": 3.8102727243143164, + "tokens_seen": 695730176 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006377372116349048, + "loss": 1.4945, + "theoretical_loss": 3.8102365156747267, + "tokens_seen": 695795712 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006377211634904714, + "loss": 1.5367, + "theoretical_loss": 3.810200311400237, + "tokens_seen": 695861248 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006377051153460382, + "loss": 1.4627, + "theoretical_loss": 3.8101641114899105, + "tokens_seen": 695926784 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006376890672016048, + "loss": 1.5098, + "theoretical_loss": 3.810127915942809, + "tokens_seen": 695992320 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006376730190571716, + "loss": 1.3962, + "theoretical_loss": 3.8100917247579966, + "tokens_seen": 696057856 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006376569709127382, + "loss": 1.4408, + "theoretical_loss": 3.8100555379345367, + "tokens_seen": 696123392 + }, + { + "epoch": 0.21, + "learning_rate": 0.000637640922768305, + "loss": 1.5439, + "theoretical_loss": 3.810019355471493, + "tokens_seen": 696188928 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006376248746238716, + "loss": 1.4965, + "theoretical_loss": 3.80998317736793, + "tokens_seen": 696254464 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006376088264794383, + "loss": 1.5107, + "theoretical_loss": 3.809947003622913, + "tokens_seen": 696320000 + }, + { + "epoch": 0.21, + "learning_rate": 0.000637592778335005, + "loss": 1.5593, + "theoretical_loss": 3.8099108342355046, + "tokens_seen": 696385536 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006375767301905717, + "loss": 1.4667, + "theoretical_loss": 3.809874669204772, + "tokens_seen": 696451072 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006375606820461384, + "loss": 1.5133, + "theoretical_loss": 3.8098385085297792, + "tokens_seen": 696516608 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006375446339017051, + "loss": 1.4683, + "theoretical_loss": 3.809802352209592, + "tokens_seen": 696582144 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006375285857572718, + "loss": 1.4175, + "theoretical_loss": 3.809766200243277, + "tokens_seen": 696647680 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006375125376128385, + "loss": 1.5697, + "theoretical_loss": 3.8097300526299, + "tokens_seen": 696713216 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006374964894684052, + "loss": 1.6231, + "theoretical_loss": 3.8096939093685274, + "tokens_seen": 696778752 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006374804413239719, + "loss": 1.4079, + "theoretical_loss": 3.8096577704582266, + "tokens_seen": 696844288 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006374643931795386, + "loss": 1.5173, + "theoretical_loss": 3.8096216358980635, + "tokens_seen": 696909824 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006374483450351053, + "loss": 1.5931, + "theoretical_loss": 3.809585505687106, + "tokens_seen": 696975360 + }, + { + "epoch": 0.21, + "learning_rate": 0.000637432296890672, + "loss": 1.5382, + "theoretical_loss": 3.809549379824422, + "tokens_seen": 697040896 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006374162487462387, + "loss": 1.4723, + "theoretical_loss": 3.80951325830908, + "tokens_seen": 697106432 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006374002006018054, + "loss": 1.419, + "theoretical_loss": 3.809477141140147, + "tokens_seen": 697171968 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006373841524573721, + "loss": 1.4199, + "theoretical_loss": 3.8094410283166917, + "tokens_seen": 697237504 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006373681043129388, + "loss": 1.4774, + "theoretical_loss": 3.809404919837784, + "tokens_seen": 697303040 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006373520561685055, + "loss": 1.5987, + "theoretical_loss": 3.8093688157024914, + "tokens_seen": 697368576 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006373360080240722, + "loss": 1.5156, + "theoretical_loss": 3.8093327159098846, + "tokens_seen": 697434112 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006373199598796389, + "loss": 1.4698, + "theoretical_loss": 3.8092966204590324, + "tokens_seen": 697499648 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006373039117352057, + "loss": 1.6584, + "theoretical_loss": 3.8092605293490056, + "tokens_seen": 697565184 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006372878635907723, + "loss": 1.5381, + "theoretical_loss": 3.809224442578873, + "tokens_seen": 697630720 + }, + { + "epoch": 0.21, + "learning_rate": 0.000637271815446339, + "loss": 1.5487, + "theoretical_loss": 3.809188360147707, + "tokens_seen": 697696256 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006372557673019057, + "loss": 1.4342, + "theoretical_loss": 3.8091522820545762, + "tokens_seen": 697761792 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006372397191574725, + "loss": 1.6151, + "theoretical_loss": 3.809116208298554, + "tokens_seen": 697827328 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006372236710130391, + "loss": 1.5682, + "theoretical_loss": 3.80908013887871, + "tokens_seen": 697892864 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.9903514981269836, + "objective/train/avg_token_score": 0.009648504666984081, + "objective/train/docs_used": 259086, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5222419500350952, + "objective/train/original_loss": 1.5385329723358154, + "objective/train/theoretical_loss": 3.8090440737941176, + "objective/train/tokens_used": 718418400, + "objective/train/unlikelihood_loss": 0.024524051696062088, + "theoretical_loss": 3.8090440737941176, + "tokens_seen": 697958400 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006372076228686059, + "loss": 1.4854, + "theoretical_loss": 3.8090440737941176, + "tokens_seen": 697958400 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006371915747241725, + "loss": 1.4636, + "theoretical_loss": 3.8090080130438464, + "tokens_seen": 698023936 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006371755265797393, + "loss": 1.5365, + "theoretical_loss": 3.80897195662697, + "tokens_seen": 698089472 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006371594784353059, + "loss": 1.3847, + "theoretical_loss": 3.8089359045425617, + "tokens_seen": 698155008 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006371434302908727, + "loss": 1.4152, + "theoretical_loss": 3.808899856789693, + "tokens_seen": 698220544 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006371273821464393, + "loss": 1.585, + "theoretical_loss": 3.808863813367437, + "tokens_seen": 698286080 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006371113340020061, + "loss": 1.5177, + "theoretical_loss": 3.808827774274868, + "tokens_seen": 698351616 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006370952858575727, + "loss": 1.4663, + "theoretical_loss": 3.808791739511059, + "tokens_seen": 698417152 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006370792377131395, + "loss": 1.5427, + "theoretical_loss": 3.808755709075084, + "tokens_seen": 698482688 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006370631895687061, + "loss": 1.495, + "theoretical_loss": 3.8087196829660175, + "tokens_seen": 698548224 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006370471414242729, + "loss": 1.5801, + "theoretical_loss": 3.8086836611829336, + "tokens_seen": 698613760 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006370310932798395, + "loss": 1.3625, + "theoretical_loss": 3.8086476437249077, + "tokens_seen": 698679296 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006370150451354063, + "loss": 1.5351, + "theoretical_loss": 3.808611630591014, + "tokens_seen": 698744832 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006369989969909729, + "loss": 1.5185, + "theoretical_loss": 3.808575621780329, + "tokens_seen": 698810368 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006369829488465397, + "loss": 1.552, + "theoretical_loss": 3.8085396172919275, + "tokens_seen": 698875904 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006369669007021063, + "loss": 1.561, + "theoretical_loss": 3.8085036171248854, + "tokens_seen": 698941440 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006369508525576731, + "loss": 1.5598, + "theoretical_loss": 3.8084676212782793, + "tokens_seen": 699006976 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006369348044132397, + "loss": 1.6057, + "theoretical_loss": 3.808431629751186, + "tokens_seen": 699072512 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006369187562688065, + "loss": 1.4666, + "theoretical_loss": 3.8083956425426813, + "tokens_seen": 699138048 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006369027081243731, + "loss": 1.4997, + "theoretical_loss": 3.808359659651843, + "tokens_seen": 699203584 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006368866599799399, + "loss": 1.4711, + "theoretical_loss": 3.8083236810777485, + "tokens_seen": 699269120 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006368706118355065, + "loss": 1.4436, + "theoretical_loss": 3.808287706819475, + "tokens_seen": 699334656 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006368545636910733, + "loss": 1.4959, + "theoretical_loss": 3.808251736876101, + "tokens_seen": 699400192 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006368385155466399, + "loss": 1.5358, + "theoretical_loss": 3.8082157712467044, + "tokens_seen": 699465728 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006368224674022067, + "loss": 1.5356, + "theoretical_loss": 3.8081798099303636, + "tokens_seen": 699531264 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006368064192577733, + "loss": 1.52, + "theoretical_loss": 3.808143852926157, + "tokens_seen": 699596800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006367903711133401, + "loss": 1.514, + "theoretical_loss": 3.8081079002331646, + "tokens_seen": 699662336 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006367743229689067, + "loss": 1.4704, + "theoretical_loss": 3.808071951850465, + "tokens_seen": 699727872 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006367582748244735, + "loss": 1.56, + "theoretical_loss": 3.8080360077771385, + "tokens_seen": 699793408 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006367422266800401, + "loss": 1.4236, + "theoretical_loss": 3.808000068012264, + "tokens_seen": 699858944 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006367261785356069, + "loss": 1.4907, + "theoretical_loss": 3.8079641325549223, + "tokens_seen": 699924480 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006367101303911735, + "loss": 1.5714, + "theoretical_loss": 3.807928201404194, + "tokens_seen": 699990016 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006366940822467403, + "loss": 1.3841, + "theoretical_loss": 3.8078922745591597, + "tokens_seen": 700055552 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006366780341023069, + "loss": 1.2319, + "theoretical_loss": 3.8078563520189004, + "tokens_seen": 700121088 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006366619859578737, + "loss": 1.5188, + "theoretical_loss": 3.8078204337824975, + "tokens_seen": 700186624 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006366459378134403, + "loss": 1.5345, + "theoretical_loss": 3.807784519849032, + "tokens_seen": 700252160 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006366298896690071, + "loss": 1.5246, + "theoretical_loss": 3.807748610217587, + "tokens_seen": 700317696 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006366138415245738, + "loss": 1.5402, + "theoretical_loss": 3.8077127048872437, + "tokens_seen": 700383232 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006365977933801405, + "loss": 1.5166, + "theoretical_loss": 3.8076768038570847, + "tokens_seen": 700448768 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006365817452357072, + "loss": 1.5369, + "theoretical_loss": 3.807640907126193, + "tokens_seen": 700514304 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006365656970912739, + "loss": 1.447, + "theoretical_loss": 3.8076050146936513, + "tokens_seen": 700579840 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006365496489468406, + "loss": 1.5621, + "theoretical_loss": 3.807569126558543, + "tokens_seen": 700645376 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006365336008024073, + "loss": 1.422, + "theoretical_loss": 3.807533242719952, + "tokens_seen": 700710912 + }, + { + "epoch": 0.21, + "learning_rate": 0.000636517552657974, + "loss": 1.5393, + "theoretical_loss": 3.807497363176962, + "tokens_seen": 700776448 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006365015045135407, + "loss": 1.4829, + "theoretical_loss": 3.8074614879286566, + "tokens_seen": 700841984 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006364854563691074, + "loss": 1.4888, + "theoretical_loss": 3.8074256169741205, + "tokens_seen": 700907520 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006364694082246741, + "loss": 1.5154, + "theoretical_loss": 3.8073897503124385, + "tokens_seen": 700973056 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006364533600802408, + "loss": 1.475, + "theoretical_loss": 3.807353887942696, + "tokens_seen": 701038592 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006364373119358075, + "loss": 1.4242, + "theoretical_loss": 3.8073180298639775, + "tokens_seen": 701104128 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006364212637913742, + "loss": 1.4709, + "theoretical_loss": 3.8072821760753683, + "tokens_seen": 701169664 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.9933174848556519, + "objective/train/avg_token_score": 0.006682520266622305, + "objective/train/docs_used": 260260, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5085208415985107, + "objective/train/original_loss": 1.5184680223464966, + "objective/train/theoretical_loss": 3.8072463265759557, + "objective/train/tokens_used": 721695200, + "objective/train/unlikelihood_loss": 0.017049826681613922, + "theoretical_loss": 3.8072463265759557, + "tokens_seen": 701235200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006364052156469409, + "loss": 1.5092, + "theoretical_loss": 3.8072463265759557, + "tokens_seen": 701235200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006363891675025076, + "loss": 1.5128, + "theoretical_loss": 3.8072104813648244, + "tokens_seen": 701300736 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006363731193580743, + "loss": 1.474, + "theoretical_loss": 3.8071746404410614, + "tokens_seen": 701366272 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006363570712136409, + "loss": 1.466, + "theoretical_loss": 3.807138803803753, + "tokens_seen": 701431808 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006363410230692077, + "loss": 1.5129, + "theoretical_loss": 3.807102971451986, + "tokens_seen": 701497344 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006363249749247743, + "loss": 1.5099, + "theoretical_loss": 3.807067143384849, + "tokens_seen": 701562880 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006363089267803411, + "loss": 1.4932, + "theoretical_loss": 3.8070313196014274, + "tokens_seen": 701628416 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006362928786359077, + "loss": 1.5411, + "theoretical_loss": 3.8069955001008102, + "tokens_seen": 701693952 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006362768304914745, + "loss": 1.5482, + "theoretical_loss": 3.8069596848820852, + "tokens_seen": 701759488 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006362607823470411, + "loss": 1.5117, + "theoretical_loss": 3.806923873944341, + "tokens_seen": 701825024 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006362447342026079, + "loss": 1.5063, + "theoretical_loss": 3.8068880672866667, + "tokens_seen": 701890560 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006362286860581745, + "loss": 1.5945, + "theoretical_loss": 3.8068522649081498, + "tokens_seen": 701956096 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006362126379137412, + "loss": 1.5369, + "theoretical_loss": 3.80681646680788, + "tokens_seen": 702021632 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006361965897693079, + "loss": 1.4618, + "theoretical_loss": 3.8067806729849467, + "tokens_seen": 702087168 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006361805416248746, + "loss": 1.4629, + "theoretical_loss": 3.8067448834384408, + "tokens_seen": 702152704 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006361644934804413, + "loss": 1.295, + "theoretical_loss": 3.8067090981674507, + "tokens_seen": 702218240 + }, + { + "epoch": 0.21, + "learning_rate": 0.000636148445336008, + "loss": 1.5254, + "theoretical_loss": 3.806673317171067, + "tokens_seen": 702283776 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006361323971915747, + "loss": 1.4963, + "theoretical_loss": 3.8066375404483814, + "tokens_seen": 702349312 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006361163490471414, + "loss": 1.4287, + "theoretical_loss": 3.806601767998483, + "tokens_seen": 702414848 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006361003009027081, + "loss": 1.4981, + "theoretical_loss": 3.806565999820465, + "tokens_seen": 702480384 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006360842527582748, + "loss": 1.4712, + "theoretical_loss": 3.806530235913417, + "tokens_seen": 702545920 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006360682046138415, + "loss": 1.4845, + "theoretical_loss": 3.8064944762764314, + "tokens_seen": 702611456 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006360521564694082, + "loss": 1.6338, + "theoretical_loss": 3.8064587209086005, + "tokens_seen": 702676992 + }, + { + "epoch": 0.21, + "learning_rate": 0.000636036108324975, + "loss": 1.5375, + "theoretical_loss": 3.8064229698090157, + "tokens_seen": 702742528 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006360200601805416, + "loss": 1.5119, + "theoretical_loss": 3.80638722297677, + "tokens_seen": 702808064 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006360040120361083, + "loss": 1.4942, + "theoretical_loss": 3.8063514804109557, + "tokens_seen": 702873600 + }, + { + "epoch": 0.21, + "learning_rate": 0.000635987963891675, + "loss": 1.5623, + "theoretical_loss": 3.806315742110667, + "tokens_seen": 702939136 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006359719157472417, + "loss": 1.5242, + "theoretical_loss": 3.806280008074996, + "tokens_seen": 703004672 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006359558676028084, + "loss": 1.5307, + "theoretical_loss": 3.806244278303037, + "tokens_seen": 703070208 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006359398194583752, + "loss": 1.5217, + "theoretical_loss": 3.806208552793884, + "tokens_seen": 703135744 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006359237713139418, + "loss": 1.5514, + "theoretical_loss": 3.8061728315466308, + "tokens_seen": 703201280 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006359077231695086, + "loss": 1.4309, + "theoretical_loss": 3.8061371145603715, + "tokens_seen": 703266816 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006358916750250752, + "loss": 1.485, + "theoretical_loss": 3.806101401834202, + "tokens_seen": 703332352 + }, + { + "epoch": 0.21, + "learning_rate": 0.000635875626880642, + "loss": 1.4576, + "theoretical_loss": 3.8060656933672163, + "tokens_seen": 703397888 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006358595787362086, + "loss": 1.5456, + "theoretical_loss": 3.8060299891585094, + "tokens_seen": 703463424 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006358435305917754, + "loss": 1.4328, + "theoretical_loss": 3.8059942892071783, + "tokens_seen": 703528960 + }, + { + "epoch": 0.21, + "learning_rate": 0.000635827482447342, + "loss": 1.5553, + "theoretical_loss": 3.8059585935123175, + "tokens_seen": 703594496 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006358114343029088, + "loss": 1.3878, + "theoretical_loss": 3.8059229020730236, + "tokens_seen": 703660032 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006357953861584754, + "loss": 1.4689, + "theoretical_loss": 3.805887214888393, + "tokens_seen": 703725568 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006357793380140422, + "loss": 1.5991, + "theoretical_loss": 3.8058515319575226, + "tokens_seen": 703791104 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006357632898696088, + "loss": 1.5125, + "theoretical_loss": 3.8058158532795088, + "tokens_seen": 703856640 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006357472417251756, + "loss": 1.6152, + "theoretical_loss": 3.805780178853449, + "tokens_seen": 703922176 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006357311935807422, + "loss": 1.4395, + "theoretical_loss": 3.8057445086784405, + "tokens_seen": 703987712 + }, + { + "epoch": 0.21, + "learning_rate": 0.000635715145436309, + "loss": 1.458, + "theoretical_loss": 3.8057088427535817, + "tokens_seen": 704053248 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006356990972918756, + "loss": 1.5028, + "theoretical_loss": 3.80567318107797, + "tokens_seen": 704118784 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006356830491474424, + "loss": 1.539, + "theoretical_loss": 3.8056375236507045, + "tokens_seen": 704184320 + }, + { + "epoch": 0.21, + "learning_rate": 0.000635667001003009, + "loss": 1.4909, + "theoretical_loss": 3.805601870470883, + "tokens_seen": 704249856 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006356509528585758, + "loss": 1.4898, + "theoretical_loss": 3.805566221537604, + "tokens_seen": 704315392 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006356349047141424, + "loss": 1.4226, + "theoretical_loss": 3.805530576849968, + "tokens_seen": 704380928 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006356188565697092, + "loss": 1.5259, + "theoretical_loss": 3.8054949364070736, + "tokens_seen": 704446464 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.9950438737869263, + "objective/train/avg_token_score": 0.004956121556460857, + "objective/train/docs_used": 261464, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.356468677520752, + "objective/train/original_loss": 1.3638042211532593, + "objective/train/theoretical_loss": 3.8054593002080206, + "objective/train/tokens_used": 724972000, + "objective/train/unlikelihood_loss": 0.015025044791400433, + "theoretical_loss": 3.8054593002080206, + "tokens_seen": 704512000 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006356028084252758, + "loss": 1.4215, + "theoretical_loss": 3.8054593002080206, + "tokens_seen": 704512000 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006355867602808426, + "loss": 1.4244, + "theoretical_loss": 3.805423668251909, + "tokens_seen": 704577536 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006355707121364092, + "loss": 1.5515, + "theoretical_loss": 3.8053880405378386, + "tokens_seen": 704643072 + }, + { + "epoch": 0.21, + "learning_rate": 0.000635554663991976, + "loss": 1.5377, + "theoretical_loss": 3.8053524170649107, + "tokens_seen": 704708608 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006355386158475426, + "loss": 1.521, + "theoretical_loss": 3.805316797832225, + "tokens_seen": 704774144 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006355225677031094, + "loss": 1.4457, + "theoretical_loss": 3.805281182838884, + "tokens_seen": 704839680 + }, + { + "epoch": 0.21, + "learning_rate": 0.000635506519558676, + "loss": 1.6264, + "theoretical_loss": 3.8052455720839875, + "tokens_seen": 704905216 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006354904714142428, + "loss": 1.5108, + "theoretical_loss": 3.8052099655666387, + "tokens_seen": 704970752 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006354744232698094, + "loss": 1.5529, + "theoretical_loss": 3.8051743632859383, + "tokens_seen": 705036288 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006354583751253762, + "loss": 1.4921, + "theoretical_loss": 3.8051387652409887, + "tokens_seen": 705101824 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006354423269809428, + "loss": 1.5347, + "theoretical_loss": 3.8051031714308925, + "tokens_seen": 705167360 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006354262788365096, + "loss": 1.4112, + "theoretical_loss": 3.8050675818547526, + "tokens_seen": 705232896 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006354102306920763, + "loss": 1.4915, + "theoretical_loss": 3.8050319965116715, + "tokens_seen": 705298432 + }, + { + "epoch": 0.21, + "learning_rate": 0.000635394182547643, + "loss": 1.4036, + "theoretical_loss": 3.804996415400753, + "tokens_seen": 705363968 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006353781344032097, + "loss": 1.3939, + "theoretical_loss": 3.8049608385211, + "tokens_seen": 705429504 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006353620862587764, + "loss": 1.4165, + "theoretical_loss": 3.804925265871817, + "tokens_seen": 705495040 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006353460381143431, + "loss": 1.5486, + "theoretical_loss": 3.804889697452007, + "tokens_seen": 705560576 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006353299899699098, + "loss": 1.5145, + "theoretical_loss": 3.8048541332607755, + "tokens_seen": 705626112 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006353139418254765, + "loss": 1.5154, + "theoretical_loss": 3.8048185732972266, + "tokens_seen": 705691648 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006352978936810432, + "loss": 1.4164, + "theoretical_loss": 3.8047830175604656, + "tokens_seen": 705757184 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006352818455366099, + "loss": 1.5288, + "theoretical_loss": 3.804747466049597, + "tokens_seen": 705822720 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006352657973921766, + "loss": 1.5536, + "theoretical_loss": 3.804711918763726, + "tokens_seen": 705888256 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006352497492477433, + "loss": 1.4619, + "theoretical_loss": 3.8046763757019595, + "tokens_seen": 705953792 + }, + { + "epoch": 0.21, + "learning_rate": 0.00063523370110331, + "loss": 1.441, + "theoretical_loss": 3.8046408368634026, + "tokens_seen": 706019328 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006352176529588767, + "loss": 1.501, + "theoretical_loss": 3.804605302247162, + "tokens_seen": 706084864 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006352016048144434, + "loss": 1.5441, + "theoretical_loss": 3.8045697718523437, + "tokens_seen": 706150400 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006351855566700101, + "loss": 1.5511, + "theoretical_loss": 3.804534245678055, + "tokens_seen": 706215936 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006351695085255768, + "loss": 1.5404, + "theoretical_loss": 3.8044987237234027, + "tokens_seen": 706281472 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006351534603811435, + "loss": 1.4519, + "theoretical_loss": 3.804463205987494, + "tokens_seen": 706347008 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006351374122367102, + "loss": 1.4955, + "theoretical_loss": 3.804427692469437, + "tokens_seen": 706412544 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006351213640922769, + "loss": 1.4417, + "theoretical_loss": 3.8043921831683396, + "tokens_seen": 706478080 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006351053159478436, + "loss": 1.5224, + "theoretical_loss": 3.804356678083309, + "tokens_seen": 706543616 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006350892678034103, + "loss": 1.5571, + "theoretical_loss": 3.8043211772134544, + "tokens_seen": 706609152 + }, + { + "epoch": 0.21, + "learning_rate": 0.000635073219658977, + "loss": 1.4947, + "theoretical_loss": 3.804285680557885, + "tokens_seen": 706674688 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006350571715145437, + "loss": 1.47, + "theoretical_loss": 3.8042501881157085, + "tokens_seen": 706740224 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006350411233701104, + "loss": 1.4818, + "theoretical_loss": 3.804214699886035, + "tokens_seen": 706805760 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006350250752256771, + "loss": 1.5884, + "theoretical_loss": 3.8041792158679737, + "tokens_seen": 706871296 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006350090270812438, + "loss": 1.5461, + "theoretical_loss": 3.804143736060635, + "tokens_seen": 706936832 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006349929789368105, + "loss": 1.4883, + "theoretical_loss": 3.8041082604631278, + "tokens_seen": 707002368 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006349769307923772, + "loss": 1.4492, + "theoretical_loss": 3.804072789074563, + "tokens_seen": 707067904 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006349608826479438, + "loss": 1.5642, + "theoretical_loss": 3.8040373218940515, + "tokens_seen": 707133440 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006349448345035106, + "loss": 1.5554, + "theoretical_loss": 3.804001858920704, + "tokens_seen": 707198976 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006349287863590772, + "loss": 1.4217, + "theoretical_loss": 3.8039664001536315, + "tokens_seen": 707264512 + }, + { + "epoch": 0.21, + "learning_rate": 0.000634912738214644, + "loss": 1.475, + "theoretical_loss": 3.8039309455919454, + "tokens_seen": 707330048 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006348966900702106, + "loss": 1.4022, + "theoretical_loss": 3.803895495234757, + "tokens_seen": 707395584 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006348806419257774, + "loss": 1.5103, + "theoretical_loss": 3.8038600490811794, + "tokens_seen": 707461120 + }, + { + "epoch": 0.21, + "learning_rate": 0.000634864593781344, + "loss": 1.5347, + "theoretical_loss": 3.8038246071303234, + "tokens_seen": 707526656 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006348485456369108, + "loss": 1.4685, + "theoretical_loss": 3.803789169381303, + "tokens_seen": 707592192 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006348324974924774, + "loss": 1.4745, + "theoretical_loss": 3.8037537358332294, + "tokens_seen": 707657728 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006348164493480441, + "loss": 1.5079, + "theoretical_loss": 3.8037183064852162, + "tokens_seen": 707723264 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.9892262816429138, + "objective/train/avg_token_score": 0.010773678310215473, + "objective/train/docs_used": 262675, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.671102523803711, + "objective/train/original_loss": 1.69081449508667, + "objective/train/theoretical_loss": 3.8036828813363774, + "objective/train/tokens_used": 728248800, + "objective/train/unlikelihood_loss": 0.02226361446082592, + "theoretical_loss": 3.8036828813363774, + "tokens_seen": 707788800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006348004012036108, + "loss": 1.6043, + "theoretical_loss": 3.8036828813363774, + "tokens_seen": 707788800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006347843530591775, + "loss": 1.4197, + "theoretical_loss": 3.8036474603858252, + "tokens_seen": 707854336 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006347683049147442, + "loss": 1.6044, + "theoretical_loss": 3.803612043632675, + "tokens_seen": 707919872 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006347522567703109, + "loss": 1.478, + "theoretical_loss": 3.80357663107604, + "tokens_seen": 707985408 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006347362086258776, + "loss": 1.3873, + "theoretical_loss": 3.803541222715034, + "tokens_seen": 708050944 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006347201604814443, + "loss": 1.5534, + "theoretical_loss": 3.803505818548773, + "tokens_seen": 708116480 + }, + { + "epoch": 0.21, + "learning_rate": 0.000634704112337011, + "loss": 1.5795, + "theoretical_loss": 3.803470418576371, + "tokens_seen": 708182016 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006346880641925777, + "loss": 1.5145, + "theoretical_loss": 3.8034350227969433, + "tokens_seen": 708247552 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006346720160481444, + "loss": 1.5241, + "theoretical_loss": 3.8033996312096052, + "tokens_seen": 708313088 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006346559679037111, + "loss": 1.5239, + "theoretical_loss": 3.803364243813473, + "tokens_seen": 708378624 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006346399197592778, + "loss": 1.5492, + "theoretical_loss": 3.803328860607662, + "tokens_seen": 708444160 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006346238716148445, + "loss": 1.5436, + "theoretical_loss": 3.8032934815912895, + "tokens_seen": 708509696 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006346078234704112, + "loss": 1.4753, + "theoretical_loss": 3.8032581067634705, + "tokens_seen": 708575232 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006345917753259779, + "loss": 1.5222, + "theoretical_loss": 3.803222736123323, + "tokens_seen": 708640768 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006345757271815446, + "loss": 1.4048, + "theoretical_loss": 3.8031873696699634, + "tokens_seen": 708706304 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006345596790371113, + "loss": 1.5791, + "theoretical_loss": 3.8031520074025096, + "tokens_seen": 708771840 + }, + { + "epoch": 0.21, + "learning_rate": 0.000634543630892678, + "loss": 1.3368, + "theoretical_loss": 3.8031166493200788, + "tokens_seen": 708837376 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006345275827482447, + "loss": 1.5097, + "theoretical_loss": 3.803081295421788, + "tokens_seen": 708902912 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006345115346038115, + "loss": 1.4492, + "theoretical_loss": 3.8030459457067574, + "tokens_seen": 708968448 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006344954864593781, + "loss": 1.6251, + "theoretical_loss": 3.803010600174103, + "tokens_seen": 709033984 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006344794383149449, + "loss": 1.5353, + "theoretical_loss": 3.8029752588229453, + "tokens_seen": 709099520 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006344633901705115, + "loss": 1.4717, + "theoretical_loss": 3.8029399216524027, + "tokens_seen": 709165056 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006344473420260783, + "loss": 1.4756, + "theoretical_loss": 3.802904588661594, + "tokens_seen": 709230592 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006344312938816449, + "loss": 1.5931, + "theoretical_loss": 3.802869259849639, + "tokens_seen": 709296128 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006344152457372117, + "loss": 1.5329, + "theoretical_loss": 3.8028339352156575, + "tokens_seen": 709361664 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006343991975927783, + "loss": 1.3919, + "theoretical_loss": 3.802798614758769, + "tokens_seen": 709427200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006343831494483451, + "loss": 1.6008, + "theoretical_loss": 3.8027632984780944, + "tokens_seen": 709492736 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006343671013039117, + "loss": 1.517, + "theoretical_loss": 3.8027279863727537, + "tokens_seen": 709558272 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006343510531594785, + "loss": 1.4949, + "theoretical_loss": 3.8026926784418675, + "tokens_seen": 709623808 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006343350050150451, + "loss": 1.5848, + "theoretical_loss": 3.802657374684558, + "tokens_seen": 709689344 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006343189568706119, + "loss": 1.5217, + "theoretical_loss": 3.8026220750999453, + "tokens_seen": 709754880 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006343029087261785, + "loss": 1.4718, + "theoretical_loss": 3.802586779687152, + "tokens_seen": 709820416 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006342868605817453, + "loss": 1.5871, + "theoretical_loss": 3.8025514884452987, + "tokens_seen": 709885952 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006342708124373119, + "loss": 1.44, + "theoretical_loss": 3.8025162013735088, + "tokens_seen": 709951488 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006342547642928787, + "loss": 1.5682, + "theoretical_loss": 3.802480918470904, + "tokens_seen": 710017024 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006342387161484453, + "loss": 1.4482, + "theoretical_loss": 3.8024456397366073, + "tokens_seen": 710082560 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006342226680040121, + "loss": 1.4955, + "theoretical_loss": 3.802410365169741, + "tokens_seen": 710148096 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006342066198595788, + "loss": 1.6055, + "theoretical_loss": 3.802375094769429, + "tokens_seen": 710213632 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006341905717151455, + "loss": 1.6195, + "theoretical_loss": 3.8023398285347945, + "tokens_seen": 710279168 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006341745235707122, + "loss": 1.5441, + "theoretical_loss": 3.8023045664649606, + "tokens_seen": 710344704 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006341584754262789, + "loss": 1.5235, + "theoretical_loss": 3.802269308559052, + "tokens_seen": 710410240 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006341424272818456, + "loss": 1.4564, + "theoretical_loss": 3.8022340548161933, + "tokens_seen": 710475776 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006341263791374123, + "loss": 1.6357, + "theoretical_loss": 3.8021988052355082, + "tokens_seen": 710541312 + }, + { + "epoch": 0.22, + "learning_rate": 0.000634110330992979, + "loss": 1.5608, + "theoretical_loss": 3.8021635598161216, + "tokens_seen": 710606848 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006340942828485457, + "loss": 1.5612, + "theoretical_loss": 3.802128318557159, + "tokens_seen": 710672384 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006340782347041124, + "loss": 1.6111, + "theoretical_loss": 3.802093081457745, + "tokens_seen": 710737920 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006340621865596791, + "loss": 1.5672, + "theoretical_loss": 3.8020578485170056, + "tokens_seen": 710803456 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006340461384152458, + "loss": 1.4429, + "theoretical_loss": 3.8020226197340667, + "tokens_seen": 710868992 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006340300902708125, + "loss": 1.5802, + "theoretical_loss": 3.801987395108055, + "tokens_seen": 710934528 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006340140421263792, + "loss": 1.5132, + "theoretical_loss": 3.8019521746380955, + "tokens_seen": 711000064 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.991804838180542, + "objective/train/avg_token_score": 0.008195140399038792, + "objective/train/docs_used": 263840, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.577251672744751, + "objective/train/original_loss": 1.5889029502868652, + "objective/train/theoretical_loss": 3.801916958323315, + "objective/train/tokens_used": 731525600, + "objective/train/unlikelihood_loss": 0.02007972076535225, + "theoretical_loss": 3.801916958323315, + "tokens_seen": 711065600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006339979939819459, + "loss": 1.5294, + "theoretical_loss": 3.801916958323315, + "tokens_seen": 711065600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006339819458375126, + "loss": 1.4076, + "theoretical_loss": 3.8018817461628416, + "tokens_seen": 711131136 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006339658976930793, + "loss": 1.5164, + "theoretical_loss": 3.801846538155802, + "tokens_seen": 711196672 + }, + { + "epoch": 0.22, + "learning_rate": 0.000633949849548646, + "loss": 1.5794, + "theoretical_loss": 3.801811334301323, + "tokens_seen": 711262208 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006339338014042127, + "loss": 1.4974, + "theoretical_loss": 3.8017761345985326, + "tokens_seen": 711327744 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006339177532597794, + "loss": 1.5542, + "theoretical_loss": 3.8017409390465584, + "tokens_seen": 711393280 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006339017051153461, + "loss": 1.565, + "theoretical_loss": 3.8017057476445295, + "tokens_seen": 711458816 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006338856569709128, + "loss": 1.5792, + "theoretical_loss": 3.8016705603915737, + "tokens_seen": 711524352 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006338696088264795, + "loss": 1.5126, + "theoretical_loss": 3.8016353772868197, + "tokens_seen": 711589888 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006338535606820462, + "loss": 1.4134, + "theoretical_loss": 3.801600198329397, + "tokens_seen": 711655424 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006338375125376129, + "loss": 1.4444, + "theoretical_loss": 3.8015650235184344, + "tokens_seen": 711720960 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006338214643931796, + "loss": 1.492, + "theoretical_loss": 3.8015298528530614, + "tokens_seen": 711786496 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006338054162487463, + "loss": 1.5075, + "theoretical_loss": 3.801494686332408, + "tokens_seen": 711852032 + }, + { + "epoch": 0.22, + "learning_rate": 0.000633789368104313, + "loss": 1.4797, + "theoretical_loss": 3.8014595239556046, + "tokens_seen": 711917568 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006337733199598797, + "loss": 1.617, + "theoretical_loss": 3.8014243657217808, + "tokens_seen": 711983104 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006337572718154464, + "loss": 1.4931, + "theoretical_loss": 3.8013892116300676, + "tokens_seen": 712048640 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006337412236710131, + "loss": 1.4501, + "theoretical_loss": 3.8013540616795956, + "tokens_seen": 712114176 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006337251755265798, + "loss": 1.4917, + "theoretical_loss": 3.801318915869496, + "tokens_seen": 712179712 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006337091273821465, + "loss": 1.4748, + "theoretical_loss": 3.8012837741989003, + "tokens_seen": 712245248 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006336930792377132, + "loss": 1.5176, + "theoretical_loss": 3.80124863666694, + "tokens_seen": 712310784 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006336770310932799, + "loss": 1.4012, + "theoretical_loss": 3.8012135032727468, + "tokens_seen": 712376320 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006336609829488466, + "loss": 1.4642, + "theoretical_loss": 3.8011783740154534, + "tokens_seen": 712441856 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006336449348044133, + "loss": 1.4627, + "theoretical_loss": 3.801143248894192, + "tokens_seen": 712507392 + }, + { + "epoch": 0.22, + "learning_rate": 0.00063362888665998, + "loss": 1.486, + "theoretical_loss": 3.801108127908095, + "tokens_seen": 712572928 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006336128385155467, + "loss": 1.4319, + "theoretical_loss": 3.801073011056295, + "tokens_seen": 712638464 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006335967903711134, + "loss": 1.5937, + "theoretical_loss": 3.801037898337926, + "tokens_seen": 712704000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006335807422266801, + "loss": 1.3899, + "theoretical_loss": 3.801002789752121, + "tokens_seen": 712769536 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006335646940822468, + "loss": 1.59, + "theoretical_loss": 3.800967685298014, + "tokens_seen": 712835072 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006335486459378135, + "loss": 1.4217, + "theoretical_loss": 3.8009325849747384, + "tokens_seen": 712900608 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006335325977933801, + "loss": 1.5479, + "theoretical_loss": 3.8008974887814295, + "tokens_seen": 712966144 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006335165496489469, + "loss": 1.6097, + "theoretical_loss": 3.8008623967172204, + "tokens_seen": 713031680 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006335005015045135, + "loss": 1.4386, + "theoretical_loss": 3.800827308781247, + "tokens_seen": 713097216 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006334844533600803, + "loss": 1.4096, + "theoretical_loss": 3.8007922249726445, + "tokens_seen": 713162752 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006334684052156469, + "loss": 1.4193, + "theoretical_loss": 3.8007571452905466, + "tokens_seen": 713228288 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006334523570712137, + "loss": 1.5355, + "theoretical_loss": 3.800722069734091, + "tokens_seen": 713293824 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006334363089267803, + "loss": 1.57, + "theoretical_loss": 3.800686998302411, + "tokens_seen": 713359360 + }, + { + "epoch": 0.22, + "learning_rate": 0.000633420260782347, + "loss": 1.4883, + "theoretical_loss": 3.8006519309946447, + "tokens_seen": 713424896 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006334042126379137, + "loss": 1.5217, + "theoretical_loss": 3.8006168678099277, + "tokens_seen": 713490432 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006333881644934804, + "loss": 1.5178, + "theoretical_loss": 3.800581808747397, + "tokens_seen": 713555968 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006333721163490471, + "loss": 1.3412, + "theoretical_loss": 3.800546753806189, + "tokens_seen": 713621504 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006333560682046138, + "loss": 1.5002, + "theoretical_loss": 3.8005117029854403, + "tokens_seen": 713687040 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006333400200601805, + "loss": 1.5213, + "theoretical_loss": 3.8004766562842893, + "tokens_seen": 713752576 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006333239719157472, + "loss": 1.4245, + "theoretical_loss": 3.800441613701873, + "tokens_seen": 713818112 + }, + { + "epoch": 0.22, + "learning_rate": 0.000633307923771314, + "loss": 1.5487, + "theoretical_loss": 3.8004065752373295, + "tokens_seen": 713883648 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006332918756268806, + "loss": 1.5465, + "theoretical_loss": 3.800371540889797, + "tokens_seen": 713949184 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006332758274824473, + "loss": 1.4757, + "theoretical_loss": 3.800336510658414, + "tokens_seen": 714014720 + }, + { + "epoch": 0.22, + "learning_rate": 0.000633259779338014, + "loss": 1.511, + "theoretical_loss": 3.8003014845423184, + "tokens_seen": 714080256 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006332437311935807, + "loss": 1.5428, + "theoretical_loss": 3.80026646254065, + "tokens_seen": 714145792 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006332276830491474, + "loss": 1.4933, + "theoretical_loss": 3.800231444652548, + "tokens_seen": 714211328 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006332116349047141, + "loss": 1.5453, + "theoretical_loss": 3.800196430877151, + "tokens_seen": 714276864 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.9922759532928467, + "objective/train/avg_token_score": 0.007723982445895672, + "objective/train/docs_used": 265036, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.3930925130844116, + "objective/train/original_loss": 1.403379201889038, + "objective/train/theoretical_loss": 3.8001614212135997, + "objective/train/tokens_used": 734802400, + "objective/train/unlikelihood_loss": 0.024823136627674103, + "theoretical_loss": 3.8001614212135997, + "tokens_seen": 714342400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006331955867602808, + "loss": 1.5009, + "theoretical_loss": 3.8001614212135997, + "tokens_seen": 714342400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006331795386158476, + "loss": 1.514, + "theoretical_loss": 3.8001264156610333, + "tokens_seen": 714407936 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006331634904714142, + "loss": 1.4153, + "theoretical_loss": 3.800091414218592, + "tokens_seen": 714473472 + }, + { + "epoch": 0.22, + "learning_rate": 0.000633147442326981, + "loss": 1.5258, + "theoretical_loss": 3.800056416885417, + "tokens_seen": 714539008 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006331313941825476, + "loss": 1.4236, + "theoretical_loss": 3.8000214236606484, + "tokens_seen": 714604544 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006331153460381144, + "loss": 1.4484, + "theoretical_loss": 3.7999864345434275, + "tokens_seen": 714670080 + }, + { + "epoch": 0.22, + "learning_rate": 0.000633099297893681, + "loss": 1.4764, + "theoretical_loss": 3.7999514495328954, + "tokens_seen": 714735616 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006330832497492478, + "loss": 1.4628, + "theoretical_loss": 3.7999164686281937, + "tokens_seen": 714801152 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006330672016048144, + "loss": 1.5306, + "theoretical_loss": 3.7998814918284642, + "tokens_seen": 714866688 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006330511534603812, + "loss": 1.4784, + "theoretical_loss": 3.7998465191328483, + "tokens_seen": 714932224 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006330351053159478, + "loss": 1.4152, + "theoretical_loss": 3.7998115505404892, + "tokens_seen": 714997760 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006330190571715146, + "loss": 1.6655, + "theoretical_loss": 3.7997765860505295, + "tokens_seen": 715063296 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006330030090270813, + "loss": 1.501, + "theoretical_loss": 3.7997416256621106, + "tokens_seen": 715128832 + }, + { + "epoch": 0.22, + "learning_rate": 0.000632986960882648, + "loss": 1.5475, + "theoretical_loss": 3.7997066693743773, + "tokens_seen": 715194368 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006329709127382147, + "loss": 1.4729, + "theoretical_loss": 3.799671717186472, + "tokens_seen": 715259904 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006329548645937814, + "loss": 1.4758, + "theoretical_loss": 3.799636769097538, + "tokens_seen": 715325440 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006329388164493481, + "loss": 1.5465, + "theoretical_loss": 3.7996018251067203, + "tokens_seen": 715390976 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006329227683049148, + "loss": 1.5239, + "theoretical_loss": 3.799566885213162, + "tokens_seen": 715456512 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006329067201604815, + "loss": 1.5164, + "theoretical_loss": 3.799531949416007, + "tokens_seen": 715522048 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006328906720160482, + "loss": 1.4343, + "theoretical_loss": 3.799497017714401, + "tokens_seen": 715587584 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006328746238716149, + "loss": 1.4175, + "theoretical_loss": 3.7994620901074887, + "tokens_seen": 715653120 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006328585757271816, + "loss": 1.5593, + "theoretical_loss": 3.7994271665944144, + "tokens_seen": 715718656 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006328425275827483, + "loss": 1.4415, + "theoretical_loss": 3.799392247174324, + "tokens_seen": 715784192 + }, + { + "epoch": 0.22, + "learning_rate": 0.000632826479438315, + "loss": 1.4388, + "theoretical_loss": 3.799357331846364, + "tokens_seen": 715849728 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006328104312938817, + "loss": 1.5316, + "theoretical_loss": 3.7993224206096787, + "tokens_seen": 715915264 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006327943831494484, + "loss": 1.5473, + "theoretical_loss": 3.7992875134634154, + "tokens_seen": 715980800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006327783350050151, + "loss": 1.4998, + "theoretical_loss": 3.79925261040672, + "tokens_seen": 716046336 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006327622868605818, + "loss": 1.4774, + "theoretical_loss": 3.7992177114387387, + "tokens_seen": 716111872 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006327462387161485, + "loss": 1.4801, + "theoretical_loss": 3.799182816558619, + "tokens_seen": 716177408 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006327301905717152, + "loss": 1.4645, + "theoretical_loss": 3.7991479257655083, + "tokens_seen": 716242944 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006327141424272819, + "loss": 1.4399, + "theoretical_loss": 3.7991130390585535, + "tokens_seen": 716308480 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006326980942828486, + "loss": 1.3607, + "theoretical_loss": 3.7990781564369023, + "tokens_seen": 716374016 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006326820461384153, + "loss": 1.4679, + "theoretical_loss": 3.7990432778997034, + "tokens_seen": 716439552 + }, + { + "epoch": 0.22, + "learning_rate": 0.000632665997993982, + "loss": 1.5297, + "theoretical_loss": 3.799008403446104, + "tokens_seen": 716505088 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006326499498495487, + "loss": 1.4663, + "theoretical_loss": 3.798973533075253, + "tokens_seen": 716570624 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006326339017051154, + "loss": 1.5027, + "theoretical_loss": 3.798938666786299, + "tokens_seen": 716636160 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006326178535606821, + "loss": 1.6376, + "theoretical_loss": 3.798903804578391, + "tokens_seen": 716701696 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006326018054162488, + "loss": 1.4919, + "theoretical_loss": 3.7988689464506784, + "tokens_seen": 716767232 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006325857572718155, + "loss": 1.4259, + "theoretical_loss": 3.798834092402311, + "tokens_seen": 716832768 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006325697091273822, + "loss": 1.4808, + "theoretical_loss": 3.7987992424324366, + "tokens_seen": 716898304 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006325536609829489, + "loss": 1.5557, + "theoretical_loss": 3.7987643965402076, + "tokens_seen": 716963840 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006325376128385156, + "loss": 1.5124, + "theoretical_loss": 3.7987295547247735, + "tokens_seen": 717029376 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006325215646940823, + "loss": 1.5621, + "theoretical_loss": 3.7986947169852834, + "tokens_seen": 717094912 + }, + { + "epoch": 0.22, + "learning_rate": 0.000632505516549649, + "loss": 1.576, + "theoretical_loss": 3.79865988332089, + "tokens_seen": 717160448 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006324894684052157, + "loss": 1.5547, + "theoretical_loss": 3.798625053730743, + "tokens_seen": 717225984 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006324734202607824, + "loss": 1.5295, + "theoretical_loss": 3.7985902282139943, + "tokens_seen": 717291520 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006324573721163491, + "loss": 1.5305, + "theoretical_loss": 3.7985554067697946, + "tokens_seen": 717357056 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006324413239719158, + "loss": 1.5112, + "theoretical_loss": 3.798520589397297, + "tokens_seen": 717422592 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006324252758274825, + "loss": 1.6593, + "theoretical_loss": 3.798485776095652, + "tokens_seen": 717488128 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006324092276830492, + "loss": 1.446, + "theoretical_loss": 3.7984509668640136, + "tokens_seen": 717553664 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.9918718934059143, + "objective/train/avg_token_score": 0.008128096349537373, + "objective/train/docs_used": 266305, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5197114944458008, + "objective/train/original_loss": 1.5322251319885254, + "objective/train/theoretical_loss": 3.7984161617015326, + "objective/train/tokens_used": 738079200, + "objective/train/unlikelihood_loss": 0.020104652270674706, + "theoretical_loss": 3.7984161617015326, + "tokens_seen": 717619200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006323931795386159, + "loss": 1.4545, + "theoretical_loss": 3.7984161617015326, + "tokens_seen": 717619200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006323771313941826, + "loss": 1.5272, + "theoretical_loss": 3.7983813606073626, + "tokens_seen": 717684736 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006323610832497493, + "loss": 1.5153, + "theoretical_loss": 3.7983465635806564, + "tokens_seen": 717750272 + }, + { + "epoch": 0.22, + "learning_rate": 0.000632345035105316, + "loss": 1.6194, + "theoretical_loss": 3.7983117706205674, + "tokens_seen": 717815808 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006323289869608827, + "loss": 1.6122, + "theoretical_loss": 3.798276981726249, + "tokens_seen": 717881344 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006323129388164494, + "loss": 1.4788, + "theoretical_loss": 3.798242196896856, + "tokens_seen": 717946880 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006322968906720161, + "loss": 1.5346, + "theoretical_loss": 3.798207416131541, + "tokens_seen": 718012416 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006322808425275828, + "loss": 1.4964, + "theoretical_loss": 3.798172639429459, + "tokens_seen": 718077952 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006322647943831495, + "loss": 1.4691, + "theoretical_loss": 3.798137866789764, + "tokens_seen": 718143488 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006322487462387162, + "loss": 1.5082, + "theoretical_loss": 3.798103098211611, + "tokens_seen": 718209024 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006322326980942829, + "loss": 1.4794, + "theoretical_loss": 3.798068333694156, + "tokens_seen": 718274560 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006322166499498496, + "loss": 1.5233, + "theoretical_loss": 3.798033573236553, + "tokens_seen": 718340096 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006322006018054163, + "loss": 1.4364, + "theoretical_loss": 3.797998816837959, + "tokens_seen": 718405632 + }, + { + "epoch": 0.22, + "learning_rate": 0.000632184553660983, + "loss": 1.512, + "theoretical_loss": 3.7979640644975277, + "tokens_seen": 718471168 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006321685055165497, + "loss": 1.4458, + "theoretical_loss": 3.797929316214417, + "tokens_seen": 718536704 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006321524573721164, + "loss": 1.3913, + "theoretical_loss": 3.797894571987783, + "tokens_seen": 718602240 + }, + { + "epoch": 0.22, + "learning_rate": 0.000632136409227683, + "loss": 1.4614, + "theoretical_loss": 3.7978598318167816, + "tokens_seen": 718667776 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006321203610832498, + "loss": 1.486, + "theoretical_loss": 3.7978250957005697, + "tokens_seen": 718733312 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006321043129388164, + "loss": 1.4996, + "theoretical_loss": 3.7977903636383044, + "tokens_seen": 718798848 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006320882647943832, + "loss": 1.4207, + "theoretical_loss": 3.7977556356291435, + "tokens_seen": 718864384 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006320722166499498, + "loss": 1.5471, + "theoretical_loss": 3.7977209116722443, + "tokens_seen": 718929920 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006320561685055166, + "loss": 1.4746, + "theoretical_loss": 3.797686191766765, + "tokens_seen": 718995456 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006320401203610832, + "loss": 1.4725, + "theoretical_loss": 3.797651475911863, + "tokens_seen": 719060992 + }, + { + "epoch": 0.22, + "learning_rate": 0.00063202407221665, + "loss": 1.5065, + "theoretical_loss": 3.7976167641066962, + "tokens_seen": 719126528 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006320080240722166, + "loss": 1.4283, + "theoretical_loss": 3.7975820563504246, + "tokens_seen": 719192064 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006319919759277833, + "loss": 1.6017, + "theoretical_loss": 3.797547352642206, + "tokens_seen": 719257600 + }, + { + "epoch": 0.22, + "learning_rate": 0.00063197592778335, + "loss": 1.4729, + "theoretical_loss": 3.7975126529812, + "tokens_seen": 719323136 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006319598796389167, + "loss": 1.4496, + "theoretical_loss": 3.797477957366566, + "tokens_seen": 719388672 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006319438314944834, + "loss": 1.5014, + "theoretical_loss": 3.797443265797463, + "tokens_seen": 719454208 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006319277833500501, + "loss": 1.5991, + "theoretical_loss": 3.7974085782730516, + "tokens_seen": 719519744 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006319117352056168, + "loss": 1.3858, + "theoretical_loss": 3.797373894792491, + "tokens_seen": 719585280 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006318956870611835, + "loss": 1.478, + "theoretical_loss": 3.797339215354942, + "tokens_seen": 719650816 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006318796389167502, + "loss": 1.553, + "theoretical_loss": 3.7973045399595655, + "tokens_seen": 719716352 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006318635907723169, + "loss": 1.5035, + "theoretical_loss": 3.797269868605522, + "tokens_seen": 719781888 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006318475426278836, + "loss": 1.4688, + "theoretical_loss": 3.797235201291973, + "tokens_seen": 719847424 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006318314944834504, + "loss": 1.5204, + "theoretical_loss": 3.797200538018079, + "tokens_seen": 719912960 + }, + { + "epoch": 0.22, + "learning_rate": 0.000631815446339017, + "loss": 1.4848, + "theoretical_loss": 3.7971658787830025, + "tokens_seen": 719978496 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006317993981945838, + "loss": 1.5176, + "theoretical_loss": 3.7971312235859047, + "tokens_seen": 720044032 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006317833500501505, + "loss": 1.5734, + "theoretical_loss": 3.797096572425948, + "tokens_seen": 720109568 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006317673019057172, + "loss": 1.5416, + "theoretical_loss": 3.7970619253022946, + "tokens_seen": 720175104 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006317512537612839, + "loss": 1.4205, + "theoretical_loss": 3.797027282214107, + "tokens_seen": 720240640 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006317352056168506, + "loss": 1.4199, + "theoretical_loss": 3.7969926431605483, + "tokens_seen": 720306176 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006317191574724173, + "loss": 1.4146, + "theoretical_loss": 3.796958008140782, + "tokens_seen": 720371712 + }, + { + "epoch": 0.22, + "learning_rate": 0.000631703109327984, + "loss": 1.4893, + "theoretical_loss": 3.7969233771539708, + "tokens_seen": 720437248 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006316870611835507, + "loss": 1.5245, + "theoretical_loss": 3.796888750199278, + "tokens_seen": 720502784 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006316710130391174, + "loss": 1.4767, + "theoretical_loss": 3.796854127275868, + "tokens_seen": 720568320 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006316549648946841, + "loss": 1.5378, + "theoretical_loss": 3.796819508382905, + "tokens_seen": 720633856 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006316389167502508, + "loss": 1.4694, + "theoretical_loss": 3.796784893519553, + "tokens_seen": 720699392 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006316228686058175, + "loss": 1.5834, + "theoretical_loss": 3.796750282684977, + "tokens_seen": 720764928 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006316068204613842, + "loss": 1.3717, + "theoretical_loss": 3.7967156758783416, + "tokens_seen": 720830464 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.9948531985282898, + "objective/train/avg_token_score": 0.005146845243871212, + "objective/train/docs_used": 267624, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.394271731376648, + "objective/train/original_loss": 1.402082920074463, + "objective/train/theoretical_loss": 3.7966810730988114, + "objective/train/tokens_used": 741356000, + "objective/train/unlikelihood_loss": 0.014200781472027302, + "theoretical_loss": 3.7966810730988114, + "tokens_seen": 720896000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006315907723169509, + "loss": 1.3855, + "theoretical_loss": 3.7966810730988114, + "tokens_seen": 720896000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006315747241725176, + "loss": 1.425, + "theoretical_loss": 3.7966464743455526, + "tokens_seen": 720961536 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006315586760280843, + "loss": 1.477, + "theoretical_loss": 3.79661187961773, + "tokens_seen": 721027072 + }, + { + "epoch": 0.22, + "learning_rate": 0.000631542627883651, + "loss": 1.4091, + "theoretical_loss": 3.79657728891451, + "tokens_seen": 721092608 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006315265797392177, + "loss": 1.4938, + "theoretical_loss": 3.796542702235059, + "tokens_seen": 721158144 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006315105315947844, + "loss": 1.4551, + "theoretical_loss": 3.7965081195785424, + "tokens_seen": 721223680 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006314944834503511, + "loss": 1.4953, + "theoretical_loss": 3.7964735409441275, + "tokens_seen": 721289216 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006314784353059178, + "loss": 1.4312, + "theoretical_loss": 3.7964389663309808, + "tokens_seen": 721354752 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006314623871614845, + "loss": 1.544, + "theoretical_loss": 3.796404395738269, + "tokens_seen": 721420288 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006314463390170512, + "loss": 1.4735, + "theoretical_loss": 3.79636982916516, + "tokens_seen": 721485824 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006314302908726179, + "loss": 1.5044, + "theoretical_loss": 3.7963352666108214, + "tokens_seen": 721551360 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006314142427281846, + "loss": 1.5976, + "theoretical_loss": 3.796300708074421, + "tokens_seen": 721616896 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006313981945837513, + "loss": 1.5767, + "theoretical_loss": 3.796266153555127, + "tokens_seen": 721682432 + }, + { + "epoch": 0.22, + "learning_rate": 0.000631382146439318, + "loss": 1.5774, + "theoretical_loss": 3.7962316030521066, + "tokens_seen": 721747968 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006313660982948847, + "loss": 1.462, + "theoretical_loss": 3.79619705656453, + "tokens_seen": 721813504 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006313500501504514, + "loss": 1.482, + "theoretical_loss": 3.7961625140915647, + "tokens_seen": 721879040 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006313340020060181, + "loss": 1.4397, + "theoretical_loss": 3.7961279756323805, + "tokens_seen": 721944576 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006313179538615848, + "loss": 1.5039, + "theoretical_loss": 3.7960934411861462, + "tokens_seen": 722010112 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006313019057171515, + "loss": 1.5556, + "theoretical_loss": 3.7960589107520324, + "tokens_seen": 722075648 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006312858575727182, + "loss": 1.4682, + "theoretical_loss": 3.7960243843292076, + "tokens_seen": 722141184 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006312698094282849, + "loss": 1.5494, + "theoretical_loss": 3.795989861916843, + "tokens_seen": 722206720 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006312537612838516, + "loss": 1.467, + "theoretical_loss": 3.7959553435141076, + "tokens_seen": 722272256 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006312377131394183, + "loss": 1.5168, + "theoretical_loss": 3.795920829120173, + "tokens_seen": 722337792 + }, + { + "epoch": 0.22, + "learning_rate": 0.000631221664994985, + "loss": 1.3952, + "theoretical_loss": 3.79588631873421, + "tokens_seen": 722403328 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006312056168505517, + "loss": 1.4058, + "theoretical_loss": 3.7958518123553886, + "tokens_seen": 722468864 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006311895687061184, + "loss": 1.4835, + "theoretical_loss": 3.7958173099828816, + "tokens_seen": 722534400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006311735205616851, + "loss": 1.5298, + "theoretical_loss": 3.795782811615859, + "tokens_seen": 722599936 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006311574724172518, + "loss": 1.4935, + "theoretical_loss": 3.7957483172534934, + "tokens_seen": 722665472 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006311414242728185, + "loss": 1.4658, + "theoretical_loss": 3.795713826894957, + "tokens_seen": 722731008 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006311253761283852, + "loss": 1.6395, + "theoretical_loss": 3.7956793405394214, + "tokens_seen": 722796544 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006311093279839519, + "loss": 1.5811, + "theoretical_loss": 3.79564485818606, + "tokens_seen": 722862080 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006310932798395186, + "loss": 1.5302, + "theoretical_loss": 3.7956103798340446, + "tokens_seen": 722927616 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006310772316950853, + "loss": 1.5108, + "theoretical_loss": 3.795575905482549, + "tokens_seen": 722993152 + }, + { + "epoch": 0.22, + "learning_rate": 0.000631061183550652, + "loss": 1.4824, + "theoretical_loss": 3.7955414351307457, + "tokens_seen": 723058688 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006310451354062187, + "loss": 1.5179, + "theoretical_loss": 3.7955069687778087, + "tokens_seen": 723124224 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006310290872617854, + "loss": 1.584, + "theoretical_loss": 3.795472506422912, + "tokens_seen": 723189760 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006310130391173521, + "loss": 1.5402, + "theoretical_loss": 3.795438048065229, + "tokens_seen": 723255296 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006309969909729188, + "loss": 1.5552, + "theoretical_loss": 3.7954035937039348, + "tokens_seen": 723320832 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006309809428284855, + "loss": 1.541, + "theoretical_loss": 3.7953691433382026, + "tokens_seen": 723386368 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006309648946840522, + "loss": 1.5554, + "theoretical_loss": 3.795334696967208, + "tokens_seen": 723451904 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006309488465396189, + "loss": 1.5046, + "theoretical_loss": 3.795300254590126, + "tokens_seen": 723517440 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006309327983951856, + "loss": 1.4751, + "theoretical_loss": 3.7952658162061317, + "tokens_seen": 723582976 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006309167502507523, + "loss": 1.3736, + "theoretical_loss": 3.7952313818144, + "tokens_seen": 723648512 + }, + { + "epoch": 0.22, + "learning_rate": 0.000630900702106319, + "loss": 1.5635, + "theoretical_loss": 3.7951969514141073, + "tokens_seen": 723714048 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006308846539618857, + "loss": 1.4734, + "theoretical_loss": 3.795162525004429, + "tokens_seen": 723779584 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006308686058174524, + "loss": 1.4575, + "theoretical_loss": 3.795128102584542, + "tokens_seen": 723845120 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006308525576730191, + "loss": 1.4684, + "theoretical_loss": 3.7950936841536222, + "tokens_seen": 723910656 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006308365095285858, + "loss": 1.5107, + "theoretical_loss": 3.7950592697108467, + "tokens_seen": 723976192 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006308204613841525, + "loss": 1.5117, + "theoretical_loss": 3.795024859255392, + "tokens_seen": 724041728 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006308044132397192, + "loss": 1.4903, + "theoretical_loss": 3.7949904527864353, + "tokens_seen": 724107264 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.9941313862800598, + "objective/train/avg_token_score": 0.005868606735020876, + "objective/train/docs_used": 268786, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5166075229644775, + "objective/train/original_loss": 1.5272202491760254, + "objective/train/theoretical_loss": 3.7949560503031545, + "objective/train/tokens_used": 744632800, + "objective/train/unlikelihood_loss": 0.012871145270764828, + "theoretical_loss": 3.7949560503031545, + "tokens_seen": 724172800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006307883650952859, + "loss": 1.5496, + "theoretical_loss": 3.7949560503031545, + "tokens_seen": 724172800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006307723169508526, + "loss": 1.5475, + "theoretical_loss": 3.794921651804727, + "tokens_seen": 724238336 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006307562688064193, + "loss": 1.7031, + "theoretical_loss": 3.7948872572903305, + "tokens_seen": 724303872 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006307402206619859, + "loss": 1.4338, + "theoretical_loss": 3.7948528667591432, + "tokens_seen": 724369408 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006307241725175527, + "loss": 1.4635, + "theoretical_loss": 3.7948184802103437, + "tokens_seen": 724434944 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006307081243731193, + "loss": 1.5147, + "theoretical_loss": 3.7947840976431104, + "tokens_seen": 724500480 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006306920762286861, + "loss": 1.4227, + "theoretical_loss": 3.7947497190566226, + "tokens_seen": 724566016 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006306760280842527, + "loss": 1.4694, + "theoretical_loss": 3.7947153444500588, + "tokens_seen": 724631552 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006306599799398195, + "loss": 1.5368, + "theoretical_loss": 3.794680973822599, + "tokens_seen": 724697088 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006306439317953861, + "loss": 1.404, + "theoretical_loss": 3.7946466071734224, + "tokens_seen": 724762624 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006306278836509529, + "loss": 1.5309, + "theoretical_loss": 3.794612244501709, + "tokens_seen": 724828160 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006306118355065195, + "loss": 1.4748, + "theoretical_loss": 3.794577885806639, + "tokens_seen": 724893696 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006305957873620863, + "loss": 1.4623, + "theoretical_loss": 3.7945435310873927, + "tokens_seen": 724959232 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006305797392176529, + "loss": 1.557, + "theoretical_loss": 3.7945091803431503, + "tokens_seen": 725024768 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006305636910732197, + "loss": 1.4992, + "theoretical_loss": 3.7944748335730933, + "tokens_seen": 725090304 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006305476429287863, + "loss": 1.5393, + "theoretical_loss": 3.794440490776402, + "tokens_seen": 725155840 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006305315947843531, + "loss": 1.4952, + "theoretical_loss": 3.7944061519522583, + "tokens_seen": 725221376 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006305155466399197, + "loss": 1.411, + "theoretical_loss": 3.7943718170998437, + "tokens_seen": 725286912 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006304994984954865, + "loss": 1.4277, + "theoretical_loss": 3.79433748621834, + "tokens_seen": 725352448 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006304834503510531, + "loss": 1.5399, + "theoretical_loss": 3.7943031593069287, + "tokens_seen": 725417984 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006304674022066199, + "loss": 1.4838, + "theoretical_loss": 3.794268836364793, + "tokens_seen": 725483520 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006304513540621866, + "loss": 1.4651, + "theoretical_loss": 3.7942345173911147, + "tokens_seen": 725549056 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006304353059177533, + "loss": 1.4355, + "theoretical_loss": 3.7942002023850767, + "tokens_seen": 725614592 + }, + { + "epoch": 0.22, + "learning_rate": 0.00063041925777332, + "loss": 1.3981, + "theoretical_loss": 3.7941658913458625, + "tokens_seen": 725680128 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006304032096288867, + "loss": 1.485, + "theoretical_loss": 3.794131584272655, + "tokens_seen": 725745664 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006303871614844534, + "loss": 1.3984, + "theoretical_loss": 3.794097281164637, + "tokens_seen": 725811200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006303711133400201, + "loss": 1.5036, + "theoretical_loss": 3.7940629820209937, + "tokens_seen": 725876736 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006303550651955868, + "loss": 1.4936, + "theoretical_loss": 3.794028686840908, + "tokens_seen": 725942272 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006303390170511535, + "loss": 1.4877, + "theoretical_loss": 3.7939943956235647, + "tokens_seen": 726007808 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006303229689067202, + "loss": 1.4547, + "theoretical_loss": 3.7939601083681476, + "tokens_seen": 726073344 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006303069207622869, + "loss": 1.5448, + "theoretical_loss": 3.793925825073842, + "tokens_seen": 726138880 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006302908726178536, + "loss": 1.4795, + "theoretical_loss": 3.7938915457398323, + "tokens_seen": 726204416 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006302748244734203, + "loss": 1.5104, + "theoretical_loss": 3.793857270365304, + "tokens_seen": 726269952 + }, + { + "epoch": 0.22, + "learning_rate": 0.000630258776328987, + "loss": 1.5085, + "theoretical_loss": 3.7938229989494427, + "tokens_seen": 726335488 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006302427281845537, + "loss": 1.4368, + "theoretical_loss": 3.793788731491434, + "tokens_seen": 726401024 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006302266800401204, + "loss": 1.5273, + "theoretical_loss": 3.7937544679904636, + "tokens_seen": 726466560 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006302106318956871, + "loss": 1.5552, + "theoretical_loss": 3.793720208445718, + "tokens_seen": 726532096 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006301945837512538, + "loss": 1.4176, + "theoretical_loss": 3.793685952856383, + "tokens_seen": 726597632 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006301785356068205, + "loss": 1.3434, + "theoretical_loss": 3.7936517012216457, + "tokens_seen": 726663168 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006301624874623872, + "loss": 1.5482, + "theoretical_loss": 3.7936174535406932, + "tokens_seen": 726728704 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006301464393179539, + "loss": 1.4509, + "theoretical_loss": 3.793583209812712, + "tokens_seen": 726794240 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006301303911735206, + "loss": 1.4339, + "theoretical_loss": 3.7935489700368894, + "tokens_seen": 726859776 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006301143430290873, + "loss": 1.4977, + "theoretical_loss": 3.7935147342124136, + "tokens_seen": 726925312 + }, + { + "epoch": 0.22, + "learning_rate": 0.000630098294884654, + "loss": 1.4262, + "theoretical_loss": 3.7934805023384723, + "tokens_seen": 726990848 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006300822467402207, + "loss": 1.5963, + "theoretical_loss": 3.7934462744142534, + "tokens_seen": 727056384 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006300661985957874, + "loss": 1.635, + "theoretical_loss": 3.7934120504389455, + "tokens_seen": 727121920 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006300501504513541, + "loss": 1.5048, + "theoretical_loss": 3.7933778304117363, + "tokens_seen": 727187456 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006300341023069208, + "loss": 1.5498, + "theoretical_loss": 3.7933436143318153, + "tokens_seen": 727252992 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006300180541624875, + "loss": 1.5052, + "theoretical_loss": 3.7933094021983713, + "tokens_seen": 727318528 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006300020060180542, + "loss": 1.4613, + "theoretical_loss": 3.793275194010594, + "tokens_seen": 727384064 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.9955691695213318, + "objective/train/avg_token_score": 0.004430805332958698, + "objective/train/docs_used": 269904, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5208779573440552, + "objective/train/original_loss": 1.5283366441726685, + "objective/train/theoretical_loss": 3.7932409897676727, + "objective/train/tokens_used": 747909600, + "objective/train/unlikelihood_loss": 0.009863597340881824, + "theoretical_loss": 3.7932409897676727, + "tokens_seen": 727449600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006299859578736209, + "loss": 1.5316, + "theoretical_loss": 3.7932409897676727, + "tokens_seen": 727449600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006299699097291876, + "loss": 1.3903, + "theoretical_loss": 3.793206789468797, + "tokens_seen": 727515136 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006299538615847543, + "loss": 1.5456, + "theoretical_loss": 3.793172593113157, + "tokens_seen": 727580672 + }, + { + "epoch": 0.22, + "learning_rate": 0.000629937813440321, + "loss": 1.4711, + "theoretical_loss": 3.7931384006999425, + "tokens_seen": 727646208 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006299217652958877, + "loss": 1.452, + "theoretical_loss": 3.7931042122283447, + "tokens_seen": 727711744 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006299057171514544, + "loss": 1.5468, + "theoretical_loss": 3.7930700276975537, + "tokens_seen": 727777280 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006298896690070211, + "loss": 1.4567, + "theoretical_loss": 3.793035847106761, + "tokens_seen": 727842816 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006298736208625878, + "loss": 1.5449, + "theoretical_loss": 3.7930016704551575, + "tokens_seen": 727908352 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006298575727181545, + "loss": 1.4075, + "theoretical_loss": 3.7929674977419343, + "tokens_seen": 727973888 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006298415245737212, + "loss": 1.4175, + "theoretical_loss": 3.792933328966283, + "tokens_seen": 728039424 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006298254764292879, + "loss": 1.4383, + "theoretical_loss": 3.792899164127397, + "tokens_seen": 728104960 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006298094282848546, + "loss": 1.4457, + "theoretical_loss": 3.792865003224467, + "tokens_seen": 728170496 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006297933801404213, + "loss": 1.5253, + "theoretical_loss": 3.792830846256685, + "tokens_seen": 728236032 + }, + { + "epoch": 0.22, + "learning_rate": 0.000629777331995988, + "loss": 1.4523, + "theoretical_loss": 3.792796693223245, + "tokens_seen": 728301568 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006297612838515547, + "loss": 1.4397, + "theoretical_loss": 3.792762544123339, + "tokens_seen": 728367104 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006297452357071214, + "loss": 1.4319, + "theoretical_loss": 3.79272839895616, + "tokens_seen": 728432640 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006297291875626881, + "loss": 1.5138, + "theoretical_loss": 3.792694257720902, + "tokens_seen": 728498176 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006297131394182548, + "loss": 1.5501, + "theoretical_loss": 3.792660120416758, + "tokens_seen": 728563712 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006296970912738215, + "loss": 1.4839, + "theoretical_loss": 3.792625987042922, + "tokens_seen": 728629248 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006296810431293882, + "loss": 1.4501, + "theoretical_loss": 3.7925918575985884, + "tokens_seen": 728694784 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006296649949849549, + "loss": 1.469, + "theoretical_loss": 3.7925577320829507, + "tokens_seen": 728760320 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006296489468405216, + "loss": 1.4844, + "theoretical_loss": 3.792523610495204, + "tokens_seen": 728825856 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006296328986960883, + "loss": 1.4435, + "theoretical_loss": 3.7924894928345427, + "tokens_seen": 728891392 + }, + { + "epoch": 0.22, + "learning_rate": 0.000629616850551655, + "loss": 1.484, + "theoretical_loss": 3.7924553791001623, + "tokens_seen": 728956928 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006296008024072217, + "loss": 1.4566, + "theoretical_loss": 3.792421269291258, + "tokens_seen": 729022464 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006295847542627884, + "loss": 1.4879, + "theoretical_loss": 3.7923871634070245, + "tokens_seen": 729088000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006295687061183551, + "loss": 1.5528, + "theoretical_loss": 3.792353061446658, + "tokens_seen": 729153536 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006295526579739218, + "loss": 1.43, + "theoretical_loss": 3.792318963409355, + "tokens_seen": 729219072 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006295366098294885, + "loss": 1.5193, + "theoretical_loss": 3.7922848692943103, + "tokens_seen": 729284608 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006295205616850552, + "loss": 1.4303, + "theoretical_loss": 3.792250779100722, + "tokens_seen": 729350144 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006295045135406219, + "loss": 1.4569, + "theoretical_loss": 3.792216692827785, + "tokens_seen": 729415680 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006294884653961886, + "loss": 1.4089, + "theoretical_loss": 3.792182610474698, + "tokens_seen": 729481216 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006294724172517553, + "loss": 1.498, + "theoretical_loss": 3.792148532040657, + "tokens_seen": 729546752 + }, + { + "epoch": 0.22, + "learning_rate": 0.000629456369107322, + "loss": 1.5751, + "theoretical_loss": 3.7921144575248595, + "tokens_seen": 729612288 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006294403209628887, + "loss": 1.3467, + "theoretical_loss": 3.7920803869265027, + "tokens_seen": 729677824 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006294242728184554, + "loss": 1.4683, + "theoretical_loss": 3.792046320244786, + "tokens_seen": 729743360 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006294082246740221, + "loss": 1.4295, + "theoretical_loss": 3.792012257478906, + "tokens_seen": 729808896 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006293921765295888, + "loss": 1.3487, + "theoretical_loss": 3.791978198628061, + "tokens_seen": 729874432 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006293761283851555, + "loss": 1.4362, + "theoretical_loss": 3.7919441436914507, + "tokens_seen": 729939968 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006293600802407222, + "loss": 1.4155, + "theoretical_loss": 3.7919100926682727, + "tokens_seen": 730005504 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006293440320962888, + "loss": 1.5047, + "theoretical_loss": 3.7918760455577267, + "tokens_seen": 730071040 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006293279839518556, + "loss": 1.4317, + "theoretical_loss": 3.791842002359012, + "tokens_seen": 730136576 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006293119358074222, + "loss": 1.4147, + "theoretical_loss": 3.7918079630713275, + "tokens_seen": 730202112 + }, + { + "epoch": 0.22, + "learning_rate": 0.000629295887662989, + "loss": 1.5307, + "theoretical_loss": 3.7917739276938733, + "tokens_seen": 730267648 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006292798395185556, + "loss": 1.4245, + "theoretical_loss": 3.7917398962258493, + "tokens_seen": 730333184 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006292637913741224, + "loss": 1.5129, + "theoretical_loss": 3.791705868666456, + "tokens_seen": 730398720 + }, + { + "epoch": 0.22, + "learning_rate": 0.000629247743229689, + "loss": 1.3689, + "theoretical_loss": 3.791671845014893, + "tokens_seen": 730464256 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006292316950852558, + "loss": 1.3549, + "theoretical_loss": 3.7916378252703624, + "tokens_seen": 730529792 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006292156469408224, + "loss": 1.5945, + "theoretical_loss": 3.791603809432064, + "tokens_seen": 730595328 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006291995987963892, + "loss": 1.4588, + "theoretical_loss": 3.7915697974991986, + "tokens_seen": 730660864 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.994825005531311, + "objective/train/avg_token_score": 0.005174937192350626, + "objective/train/docs_used": 271086, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4457859992980957, + "objective/train/original_loss": 1.453922986984253, + "objective/train/theoretical_loss": 3.7915357894709687, + "objective/train/tokens_used": 751186400, + "objective/train/unlikelihood_loss": 0.011141125112771988, + "theoretical_loss": 3.7915357894709687, + "tokens_seen": 730726400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006291835506519558, + "loss": 1.4349, + "theoretical_loss": 3.7915357894709687, + "tokens_seen": 730726400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006291675025075226, + "loss": 1.3708, + "theoretical_loss": 3.791501785346575, + "tokens_seen": 730791936 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006291514543630892, + "loss": 1.4503, + "theoretical_loss": 3.79146778512522, + "tokens_seen": 730857472 + }, + { + "epoch": 0.22, + "learning_rate": 0.000629135406218656, + "loss": 1.5031, + "theoretical_loss": 3.7914337888061054, + "tokens_seen": 730923008 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006291193580742226, + "loss": 1.4374, + "theoretical_loss": 3.7913997963884336, + "tokens_seen": 730988544 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006291033099297894, + "loss": 1.4523, + "theoretical_loss": 3.791365807871407, + "tokens_seen": 731054080 + }, + { + "epoch": 0.22, + "learning_rate": 0.000629087261785356, + "loss": 1.566, + "theoretical_loss": 3.791331823254229, + "tokens_seen": 731119616 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006290712136409228, + "loss": 1.4972, + "theoretical_loss": 3.791297842536102, + "tokens_seen": 731185152 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006290551654964895, + "loss": 1.3938, + "theoretical_loss": 3.791263865716229, + "tokens_seen": 731250688 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006290391173520562, + "loss": 1.5116, + "theoretical_loss": 3.7912298927938144, + "tokens_seen": 731316224 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006290230692076229, + "loss": 1.4935, + "theoretical_loss": 3.7911959237680612, + "tokens_seen": 731381760 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006290070210631896, + "loss": 1.4181, + "theoretical_loss": 3.791161958638173, + "tokens_seen": 731447296 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006289909729187563, + "loss": 1.5199, + "theoretical_loss": 3.7911279974033554, + "tokens_seen": 731512832 + }, + { + "epoch": 0.22, + "learning_rate": 0.000628974924774323, + "loss": 1.4121, + "theoretical_loss": 3.7910940400628115, + "tokens_seen": 731578368 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006289588766298897, + "loss": 1.3679, + "theoretical_loss": 3.7910600866157464, + "tokens_seen": 731643904 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006289428284854564, + "loss": 1.4383, + "theoretical_loss": 3.7910261370613654, + "tokens_seen": 731709440 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006289267803410231, + "loss": 1.472, + "theoretical_loss": 3.790992191398873, + "tokens_seen": 731774976 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006289107321965898, + "loss": 1.5009, + "theoretical_loss": 3.790958249627475, + "tokens_seen": 731840512 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006288946840521565, + "loss": 1.5504, + "theoretical_loss": 3.7909243117463767, + "tokens_seen": 731906048 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006288786359077232, + "loss": 1.4025, + "theoretical_loss": 3.7908903777547835, + "tokens_seen": 731971584 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006288625877632899, + "loss": 1.4097, + "theoretical_loss": 3.7908564476519024, + "tokens_seen": 732037120 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006288465396188566, + "loss": 1.4888, + "theoretical_loss": 3.7908225214369393, + "tokens_seen": 732102656 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006288304914744233, + "loss": 1.5177, + "theoretical_loss": 3.7907885991091, + "tokens_seen": 732168192 + }, + { + "epoch": 0.22, + "learning_rate": 0.00062881444332999, + "loss": 1.4407, + "theoretical_loss": 3.790754680667593, + "tokens_seen": 732233728 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006287983951855567, + "loss": 1.4767, + "theoretical_loss": 3.7907207661116233, + "tokens_seen": 732299264 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006287823470411234, + "loss": 1.5243, + "theoretical_loss": 3.7906868554403994, + "tokens_seen": 732364800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006287662988966901, + "loss": 1.3747, + "theoretical_loss": 3.7906529486531286, + "tokens_seen": 732430336 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006287502507522568, + "loss": 1.4633, + "theoretical_loss": 3.7906190457490183, + "tokens_seen": 732495872 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006287342026078235, + "loss": 1.4176, + "theoretical_loss": 3.790585146727276, + "tokens_seen": 732561408 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006287181544633902, + "loss": 1.4471, + "theoretical_loss": 3.790551251587111, + "tokens_seen": 732626944 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006287021063189569, + "loss": 1.4879, + "theoretical_loss": 3.7905173603277307, + "tokens_seen": 732692480 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006286860581745236, + "loss": 1.405, + "theoretical_loss": 3.790483472948344, + "tokens_seen": 732758016 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006286700100300903, + "loss": 1.3702, + "theoretical_loss": 3.79044958944816, + "tokens_seen": 732823552 + }, + { + "epoch": 0.22, + "learning_rate": 0.000628653961885657, + "loss": 1.6136, + "theoretical_loss": 3.7904157098263873, + "tokens_seen": 732889088 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006286379137412237, + "loss": 1.5503, + "theoretical_loss": 3.7903818340822353, + "tokens_seen": 732954624 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006286218655967904, + "loss": 1.4906, + "theoretical_loss": 3.790347962214914, + "tokens_seen": 733020160 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006286058174523571, + "loss": 1.4114, + "theoretical_loss": 3.790314094223633, + "tokens_seen": 733085696 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006285897693079238, + "loss": 1.5053, + "theoretical_loss": 3.7902802301076024, + "tokens_seen": 733151232 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006285737211634905, + "loss": 1.4726, + "theoretical_loss": 3.790246369866032, + "tokens_seen": 733216768 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006285576730190572, + "loss": 1.448, + "theoretical_loss": 3.790212513498132, + "tokens_seen": 733282304 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006285416248746239, + "loss": 1.5496, + "theoretical_loss": 3.7901786610031145, + "tokens_seen": 733347840 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006285255767301906, + "loss": 1.4401, + "theoretical_loss": 3.7901448123801886, + "tokens_seen": 733413376 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006285095285857573, + "loss": 1.4994, + "theoretical_loss": 3.790110967628567, + "tokens_seen": 733478912 + }, + { + "epoch": 0.22, + "learning_rate": 0.000628493480441324, + "loss": 1.5427, + "theoretical_loss": 3.7900771267474607, + "tokens_seen": 733544448 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006284774322968907, + "loss": 1.4542, + "theoretical_loss": 3.7900432897360803, + "tokens_seen": 733609984 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006284613841524574, + "loss": 1.4413, + "theoretical_loss": 3.790009456593639, + "tokens_seen": 733675520 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006284453360080241, + "loss": 1.4042, + "theoretical_loss": 3.7899756273193486, + "tokens_seen": 733741056 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006284292878635908, + "loss": 1.499, + "theoretical_loss": 3.7899418019124207, + "tokens_seen": 733806592 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006284132397191575, + "loss": 1.4697, + "theoretical_loss": 3.7899079803720688, + "tokens_seen": 733872128 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006283971915747242, + "loss": 1.5337, + "theoretical_loss": 3.7898741626975054, + "tokens_seen": 733937664 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.9948803186416626, + "objective/train/avg_token_score": 0.005119704641401768, + "objective/train/docs_used": 272255, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5650900602340698, + "objective/train/original_loss": 1.574652910232544, + "objective/train/theoretical_loss": 3.7898403488879424, + "objective/train/tokens_used": 754463200, + "objective/train/unlikelihood_loss": 0.011106050573289394, + "theoretical_loss": 3.7898403488879424, + "tokens_seen": 734003200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006283811434302909, + "loss": 1.5981, + "theoretical_loss": 3.7898403488879424, + "tokens_seen": 734003200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006283650952858576, + "loss": 1.5361, + "theoretical_loss": 3.7898065389425946, + "tokens_seen": 734068736 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006283490471414243, + "loss": 1.301, + "theoretical_loss": 3.789772732860675, + "tokens_seen": 734134272 + }, + { + "epoch": 0.22, + "learning_rate": 0.000628332998996991, + "loss": 1.4924, + "theoretical_loss": 3.7897389306413967, + "tokens_seen": 734199808 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006283169508525577, + "loss": 1.5491, + "theoretical_loss": 3.7897051322839745, + "tokens_seen": 734265344 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006283009027081244, + "loss": 1.4597, + "theoretical_loss": 3.7896713377876217, + "tokens_seen": 734330880 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006282848545636911, + "loss": 1.4869, + "theoretical_loss": 3.7896375471515533, + "tokens_seen": 734396416 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006282688064192578, + "loss": 1.4873, + "theoretical_loss": 3.789603760374984, + "tokens_seen": 734461952 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006282527582748245, + "loss": 1.4653, + "theoretical_loss": 3.789569977457128, + "tokens_seen": 734527488 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006282367101303912, + "loss": 1.4897, + "theoretical_loss": 3.789536198397201, + "tokens_seen": 734593024 + }, + { + "epoch": 0.22, + "learning_rate": 0.000628220661985958, + "loss": 1.6463, + "theoretical_loss": 3.789502423194418, + "tokens_seen": 734658560 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006282046138415246, + "loss": 1.4889, + "theoretical_loss": 3.7894686518479945, + "tokens_seen": 734724096 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006281885656970914, + "loss": 1.4513, + "theoretical_loss": 3.789434884357147, + "tokens_seen": 734789632 + }, + { + "epoch": 0.22, + "learning_rate": 0.000628172517552658, + "loss": 1.5499, + "theoretical_loss": 3.7894011207210907, + "tokens_seen": 734855168 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006281564694082248, + "loss": 1.4152, + "theoretical_loss": 3.7893673609390417, + "tokens_seen": 734920704 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006281404212637914, + "loss": 1.4662, + "theoretical_loss": 3.789333605010217, + "tokens_seen": 734986240 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006281243731193582, + "loss": 1.3803, + "theoretical_loss": 3.7892998529338326, + "tokens_seen": 735051776 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006281083249749248, + "loss": 1.5648, + "theoretical_loss": 3.7892661047091067, + "tokens_seen": 735117312 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006280922768304916, + "loss": 1.451, + "theoretical_loss": 3.789232360335255, + "tokens_seen": 735182848 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006280762286860582, + "loss": 1.5009, + "theoretical_loss": 3.789198619811496, + "tokens_seen": 735248384 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006280601805416249, + "loss": 1.417, + "theoretical_loss": 3.7891648831370466, + "tokens_seen": 735313920 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006280441323971916, + "loss": 1.5569, + "theoretical_loss": 3.7891311503111247, + "tokens_seen": 735379456 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006280280842527583, + "loss": 1.3334, + "theoretical_loss": 3.789097421332949, + "tokens_seen": 735444992 + }, + { + "epoch": 0.22, + "learning_rate": 0.000628012036108325, + "loss": 1.4927, + "theoretical_loss": 3.7890636962017368, + "tokens_seen": 735510528 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006279959879638917, + "loss": 1.4931, + "theoretical_loss": 3.7890299749167076, + "tokens_seen": 735576064 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006279799398194584, + "loss": 1.4796, + "theoretical_loss": 3.7889962574770792, + "tokens_seen": 735641600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006279638916750251, + "loss": 1.4487, + "theoretical_loss": 3.7889625438820715, + "tokens_seen": 735707136 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006279478435305917, + "loss": 1.4004, + "theoretical_loss": 3.788928834130903, + "tokens_seen": 735772672 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006279317953861585, + "loss": 1.4561, + "theoretical_loss": 3.7888951282227934, + "tokens_seen": 735838208 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006279157472417251, + "loss": 1.5086, + "theoretical_loss": 3.788861426156962, + "tokens_seen": 735903744 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006278996990972919, + "loss": 1.4483, + "theoretical_loss": 3.7888277279326292, + "tokens_seen": 735969280 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006278836509528585, + "loss": 1.5907, + "theoretical_loss": 3.7887940335490153, + "tokens_seen": 736034816 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006278676028084253, + "loss": 1.4759, + "theoretical_loss": 3.7887603430053396, + "tokens_seen": 736100352 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006278515546639919, + "loss": 1.4611, + "theoretical_loss": 3.788726656300824, + "tokens_seen": 736165888 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006278355065195587, + "loss": 1.4341, + "theoretical_loss": 3.7886929734346886, + "tokens_seen": 736231424 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006278194583751253, + "loss": 1.5464, + "theoretical_loss": 3.788659294406154, + "tokens_seen": 736296960 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006278034102306921, + "loss": 1.5298, + "theoretical_loss": 3.788625619214442, + "tokens_seen": 736362496 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006277873620862587, + "loss": 1.4048, + "theoretical_loss": 3.7885919478587744, + "tokens_seen": 736428032 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006277713139418255, + "loss": 1.4289, + "theoretical_loss": 3.788558280338372, + "tokens_seen": 736493568 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006277552657973921, + "loss": 1.4994, + "theoretical_loss": 3.7885246166524578, + "tokens_seen": 736559104 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006277392176529589, + "loss": 1.3846, + "theoretical_loss": 3.7884909568002536, + "tokens_seen": 736624640 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006277231695085255, + "loss": 1.4697, + "theoretical_loss": 3.7884573007809808, + "tokens_seen": 736690176 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006277071213640923, + "loss": 1.5145, + "theoretical_loss": 3.788423648593863, + "tokens_seen": 736755712 + }, + { + "epoch": 0.22, + "learning_rate": 0.000627691073219659, + "loss": 1.4273, + "theoretical_loss": 3.788390000238123, + "tokens_seen": 736821248 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006276750250752257, + "loss": 1.5085, + "theoretical_loss": 3.788356355712984, + "tokens_seen": 736886784 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006276589769307924, + "loss": 1.5106, + "theoretical_loss": 3.788322715017668, + "tokens_seen": 736952320 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006276429287863591, + "loss": 1.4829, + "theoretical_loss": 3.7882890781514007, + "tokens_seen": 737017856 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006276268806419258, + "loss": 1.5681, + "theoretical_loss": 3.7882554451134043, + "tokens_seen": 737083392 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006276108324974925, + "loss": 1.3783, + "theoretical_loss": 3.7882218159029026, + "tokens_seen": 737148928 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006275947843530592, + "loss": 1.4824, + "theoretical_loss": 3.7881881905191213, + "tokens_seen": 737214464 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.9932360053062439, + "objective/train/avg_token_score": 0.006763988174498081, + "objective/train/docs_used": 273341, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5203301906585693, + "objective/train/original_loss": 1.5319938659667969, + "objective/train/theoretical_loss": 3.788154568961283, + "objective/train/tokens_used": 757740000, + "objective/train/unlikelihood_loss": 0.01553349383175373, + "theoretical_loss": 3.788154568961283, + "tokens_seen": 737280000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006275787362086259, + "loss": 1.5064, + "theoretical_loss": 3.788154568961283, + "tokens_seen": 737280000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006275626880641926, + "loss": 1.526, + "theoretical_loss": 3.788120951228614, + "tokens_seen": 737345536 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006275466399197593, + "loss": 1.4946, + "theoretical_loss": 3.788087337320338, + "tokens_seen": 737411072 + }, + { + "epoch": 0.22, + "learning_rate": 0.000627530591775326, + "loss": 1.4115, + "theoretical_loss": 3.7880537272356802, + "tokens_seen": 737476608 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006275145436308927, + "loss": 1.5305, + "theoretical_loss": 3.788020120973867, + "tokens_seen": 737542144 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006274984954864594, + "loss": 1.4944, + "theoretical_loss": 3.7879865185341224, + "tokens_seen": 737607680 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006274824473420261, + "loss": 1.4456, + "theoretical_loss": 3.787952919915673, + "tokens_seen": 737673216 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006274663991975928, + "loss": 1.4621, + "theoretical_loss": 3.7879193251177457, + "tokens_seen": 737738752 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006274503510531595, + "loss": 1.5206, + "theoretical_loss": 3.7878857341395653, + "tokens_seen": 737804288 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006274343029087262, + "loss": 1.4436, + "theoretical_loss": 3.7878521469803585, + "tokens_seen": 737869824 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006274182547642929, + "loss": 1.5328, + "theoretical_loss": 3.7878185636393527, + "tokens_seen": 737935360 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006274022066198596, + "loss": 1.4932, + "theoretical_loss": 3.787784984115774, + "tokens_seen": 738000896 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006273861584754263, + "loss": 1.4102, + "theoretical_loss": 3.78775140840885, + "tokens_seen": 738066432 + }, + { + "epoch": 0.22, + "learning_rate": 0.000627370110330993, + "loss": 1.4832, + "theoretical_loss": 3.7877178365178086, + "tokens_seen": 738131968 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006273540621865597, + "loss": 1.4518, + "theoretical_loss": 3.787684268441876, + "tokens_seen": 738197504 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006273380140421264, + "loss": 1.467, + "theoretical_loss": 3.787650704180281, + "tokens_seen": 738263040 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006273219658976931, + "loss": 1.555, + "theoretical_loss": 3.787617143732251, + "tokens_seen": 738328576 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006273059177532598, + "loss": 1.4959, + "theoretical_loss": 3.787583587097015, + "tokens_seen": 738394112 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006272898696088265, + "loss": 1.396, + "theoretical_loss": 3.7875500342738015, + "tokens_seen": 738459648 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006272738214643932, + "loss": 1.4561, + "theoretical_loss": 3.787516485261839, + "tokens_seen": 738525184 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006272577733199599, + "loss": 1.4565, + "theoretical_loss": 3.7874829400603556, + "tokens_seen": 738590720 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006272417251755266, + "loss": 1.3506, + "theoretical_loss": 3.7874493986685813, + "tokens_seen": 738656256 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006272256770310933, + "loss": 1.4926, + "theoretical_loss": 3.787415861085745, + "tokens_seen": 738721792 + }, + { + "epoch": 0.22, + "learning_rate": 0.00062720962888666, + "loss": 1.3733, + "theoretical_loss": 3.787382327311077, + "tokens_seen": 738787328 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006271935807422267, + "loss": 1.4561, + "theoretical_loss": 3.7873487973438067, + "tokens_seen": 738852864 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006271775325977934, + "loss": 1.5375, + "theoretical_loss": 3.7873152711831644, + "tokens_seen": 738918400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006271614844533601, + "loss": 1.438, + "theoretical_loss": 3.7872817488283803, + "tokens_seen": 738983936 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006271454363089268, + "loss": 1.4221, + "theoretical_loss": 3.7872482302786845, + "tokens_seen": 739049472 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006271293881644935, + "loss": 1.4394, + "theoretical_loss": 3.787214715533308, + "tokens_seen": 739115008 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006271133400200602, + "loss": 1.5442, + "theoretical_loss": 3.787181204591482, + "tokens_seen": 739180544 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006270972918756269, + "loss": 1.3824, + "theoretical_loss": 3.787147697452437, + "tokens_seen": 739246080 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006270812437311936, + "loss": 1.6488, + "theoretical_loss": 3.7871141941154054, + "tokens_seen": 739311616 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006270651955867603, + "loss": 1.4626, + "theoretical_loss": 3.7870806945796174, + "tokens_seen": 739377152 + }, + { + "epoch": 0.22, + "learning_rate": 0.000627049147442327, + "loss": 1.3481, + "theoretical_loss": 3.7870471988443066, + "tokens_seen": 739442688 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006270330992978937, + "loss": 1.3804, + "theoretical_loss": 3.7870137069087035, + "tokens_seen": 739508224 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006270170511534605, + "loss": 1.4861, + "theoretical_loss": 3.786980218772041, + "tokens_seen": 739573760 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006270010030090271, + "loss": 1.6306, + "theoretical_loss": 3.7869467344335526, + "tokens_seen": 739639296 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006269849548645939, + "loss": 1.4397, + "theoretical_loss": 3.7869132538924695, + "tokens_seen": 739704832 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006269689067201605, + "loss": 1.5008, + "theoretical_loss": 3.786879777148025, + "tokens_seen": 739770368 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006269528585757273, + "loss": 1.4224, + "theoretical_loss": 3.786846304199453, + "tokens_seen": 739835904 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006269368104312939, + "loss": 1.496, + "theoretical_loss": 3.7868128350459864, + "tokens_seen": 739901440 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006269207622868607, + "loss": 1.3911, + "theoretical_loss": 3.7867793696868586, + "tokens_seen": 739966976 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006269047141424273, + "loss": 1.5017, + "theoretical_loss": 3.7867459081213037, + "tokens_seen": 740032512 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006268886659979941, + "loss": 1.5305, + "theoretical_loss": 3.7867124503485563, + "tokens_seen": 740098048 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006268726178535607, + "loss": 1.4141, + "theoretical_loss": 3.7866789963678498, + "tokens_seen": 740163584 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006268565697091275, + "loss": 1.5387, + "theoretical_loss": 3.786645546178419, + "tokens_seen": 740229120 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006268405215646941, + "loss": 1.4691, + "theoretical_loss": 3.7866120997794983, + "tokens_seen": 740294656 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006268244734202609, + "loss": 1.6327, + "theoretical_loss": 3.786578657170324, + "tokens_seen": 740360192 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006268084252758275, + "loss": 1.46, + "theoretical_loss": 3.7865452183501302, + "tokens_seen": 740425728 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006267923771313943, + "loss": 1.5544, + "theoretical_loss": 3.7865117833181525, + "tokens_seen": 740491264 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.9925503134727478, + "objective/train/avg_token_score": 0.007449670694768429, + "objective/train/docs_used": 274540, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6093357801437378, + "objective/train/original_loss": 1.623950481414795, + "objective/train/theoretical_loss": 3.786478352073626, + "objective/train/tokens_used": 761016800, + "objective/train/unlikelihood_loss": 0.013323385268449783, + "theoretical_loss": 3.786478352073626, + "tokens_seen": 740556800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006267763289869609, + "loss": 1.5739, + "theoretical_loss": 3.786478352073626, + "tokens_seen": 740556800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006267602808425277, + "loss": 1.4388, + "theoretical_loss": 3.786444924615788, + "tokens_seen": 740622336 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006267442326980943, + "loss": 1.5034, + "theoretical_loss": 3.7864115009438724, + "tokens_seen": 740687872 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006267281845536611, + "loss": 1.4203, + "theoretical_loss": 3.7863780810571175, + "tokens_seen": 740753408 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006267121364092277, + "loss": 1.4156, + "theoretical_loss": 3.7863446649547594, + "tokens_seen": 740818944 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006266960882647945, + "loss": 1.5012, + "theoretical_loss": 3.786311252636034, + "tokens_seen": 740884480 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006266800401203611, + "loss": 1.4069, + "theoretical_loss": 3.7862778441001783, + "tokens_seen": 740950016 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006266639919759278, + "loss": 1.3329, + "theoretical_loss": 3.7862444393464307, + "tokens_seen": 741015552 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006266479438314945, + "loss": 1.6563, + "theoretical_loss": 3.7862110383740273, + "tokens_seen": 741081088 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006266318956870612, + "loss": 1.3579, + "theoretical_loss": 3.7861776411822063, + "tokens_seen": 741146624 + }, + { + "epoch": 0.22, + "learning_rate": 0.000626615847542628, + "loss": 1.3742, + "theoretical_loss": 3.7861442477702063, + "tokens_seen": 741212160 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006265997993981946, + "loss": 1.4983, + "theoretical_loss": 3.7861108581372633, + "tokens_seen": 741277696 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006265837512537613, + "loss": 1.4275, + "theoretical_loss": 3.786077472282618, + "tokens_seen": 741343232 + }, + { + "epoch": 0.22, + "learning_rate": 0.000626567703109328, + "loss": 1.505, + "theoretical_loss": 3.786044090205507, + "tokens_seen": 741408768 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006265516549648948, + "loss": 1.579, + "theoretical_loss": 3.7860107119051696, + "tokens_seen": 741474304 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006265356068204614, + "loss": 1.4928, + "theoretical_loss": 3.785977337380846, + "tokens_seen": 741539840 + }, + { + "epoch": 0.22, + "learning_rate": 0.000626519558676028, + "loss": 1.4907, + "theoretical_loss": 3.7859439666317733, + "tokens_seen": 741605376 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006265035105315948, + "loss": 1.4846, + "theoretical_loss": 3.7859105996571922, + "tokens_seen": 741670912 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006264874623871614, + "loss": 1.4841, + "theoretical_loss": 3.7858772364563418, + "tokens_seen": 741736448 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006264714142427282, + "loss": 1.4095, + "theoretical_loss": 3.785843877028462, + "tokens_seen": 741801984 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006264553660982948, + "loss": 1.539, + "theoretical_loss": 3.7858105213727935, + "tokens_seen": 741867520 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006264393179538616, + "loss": 1.4598, + "theoretical_loss": 3.7857771694885756, + "tokens_seen": 741933056 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006264232698094282, + "loss": 1.4862, + "theoretical_loss": 3.785743821375049, + "tokens_seen": 741998592 + }, + { + "epoch": 0.22, + "learning_rate": 0.000626407221664995, + "loss": 1.4065, + "theoretical_loss": 3.785710477031455, + "tokens_seen": 742064128 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006263911735205616, + "loss": 1.4951, + "theoretical_loss": 3.785677136457034, + "tokens_seen": 742129664 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006263751253761284, + "loss": 1.4258, + "theoretical_loss": 3.7856437996510275, + "tokens_seen": 742195200 + }, + { + "epoch": 0.22, + "learning_rate": 0.000626359077231695, + "loss": 1.436, + "theoretical_loss": 3.7856104666126766, + "tokens_seen": 742260736 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006263430290872618, + "loss": 1.5384, + "theoretical_loss": 3.785577137341223, + "tokens_seen": 742326272 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006263269809428285, + "loss": 1.4487, + "theoretical_loss": 3.7855438118359084, + "tokens_seen": 742391808 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006263109327983952, + "loss": 1.482, + "theoretical_loss": 3.785510490095975, + "tokens_seen": 742457344 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006262948846539619, + "loss": 1.5418, + "theoretical_loss": 3.7854771721206646, + "tokens_seen": 742522880 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006262788365095286, + "loss": 1.4608, + "theoretical_loss": 3.78544385790922, + "tokens_seen": 742588416 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006262627883650953, + "loss": 1.4989, + "theoretical_loss": 3.7854105474608843, + "tokens_seen": 742653952 + }, + { + "epoch": 0.23, + "learning_rate": 0.000626246740220662, + "loss": 1.5777, + "theoretical_loss": 3.785377240774899, + "tokens_seen": 742719488 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006262306920762287, + "loss": 1.2791, + "theoretical_loss": 3.7853439378505094, + "tokens_seen": 742785024 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006262146439317954, + "loss": 1.3197, + "theoretical_loss": 3.7853106386869575, + "tokens_seen": 742850560 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006261985957873621, + "loss": 1.519, + "theoretical_loss": 3.7852773432834868, + "tokens_seen": 742916096 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006261825476429288, + "loss": 1.5045, + "theoretical_loss": 3.7852440516393413, + "tokens_seen": 742981632 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006261664994984955, + "loss": 1.5128, + "theoretical_loss": 3.785210763753765, + "tokens_seen": 743047168 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006261504513540622, + "loss": 1.5008, + "theoretical_loss": 3.785177479626002, + "tokens_seen": 743112704 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006261344032096289, + "loss": 1.4914, + "theoretical_loss": 3.785144199255297, + "tokens_seen": 743178240 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006261183550651956, + "loss": 1.3524, + "theoretical_loss": 3.7851109226408943, + "tokens_seen": 743243776 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006261023069207623, + "loss": 1.5098, + "theoretical_loss": 3.785077649782039, + "tokens_seen": 743309312 + }, + { + "epoch": 0.23, + "learning_rate": 0.000626086258776329, + "loss": 1.4729, + "theoretical_loss": 3.7850443806779763, + "tokens_seen": 743374848 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006260702106318957, + "loss": 1.5717, + "theoretical_loss": 3.7850111153279515, + "tokens_seen": 743440384 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006260541624874624, + "loss": 1.447, + "theoretical_loss": 3.78497785373121, + "tokens_seen": 743505920 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006260381143430291, + "loss": 1.4552, + "theoretical_loss": 3.784944595886998, + "tokens_seen": 743571456 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006260220661985958, + "loss": 1.3438, + "theoretical_loss": 3.7849113417945603, + "tokens_seen": 743636992 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006260060180541625, + "loss": 1.5336, + "theoretical_loss": 3.7848780914531437, + "tokens_seen": 743702528 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006259899699097292, + "loss": 1.6366, + "theoretical_loss": 3.784844844861995, + "tokens_seen": 743768064 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.9921451210975647, + "objective/train/avg_token_score": 0.007854845374822617, + "objective/train/docs_used": 275779, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.577544093132019, + "objective/train/original_loss": 1.5891109704971313, + "objective/train/theoretical_loss": 3.784811602020361, + "objective/train/tokens_used": 764293600, + "objective/train/unlikelihood_loss": 0.019479064270853996, + "theoretical_loss": 3.784811602020361, + "tokens_seen": 743833600 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006259739217652959, + "loss": 1.5005, + "theoretical_loss": 3.784811602020361, + "tokens_seen": 743833600 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006259578736208626, + "loss": 1.4566, + "theoretical_loss": 3.7847783629274874, + "tokens_seen": 743899136 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006259418254764293, + "loss": 1.5513, + "theoretical_loss": 3.7847451275826227, + "tokens_seen": 743964672 + }, + { + "epoch": 0.23, + "learning_rate": 0.000625925777331996, + "loss": 1.5255, + "theoretical_loss": 3.784711895985013, + "tokens_seen": 744030208 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006259097291875627, + "loss": 1.5616, + "theoretical_loss": 3.784678668133906, + "tokens_seen": 744095744 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006258936810431294, + "loss": 1.4265, + "theoretical_loss": 3.7846454440285493, + "tokens_seen": 744161280 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006258776328986961, + "loss": 1.4452, + "theoretical_loss": 3.7846122236681916, + "tokens_seen": 744226816 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006258615847542628, + "loss": 1.4222, + "theoretical_loss": 3.7845790070520806, + "tokens_seen": 744292352 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006258455366098295, + "loss": 1.4105, + "theoretical_loss": 3.784545794179464, + "tokens_seen": 744357888 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006258294884653962, + "loss": 1.4312, + "theoretical_loss": 3.7845125850495918, + "tokens_seen": 744423424 + }, + { + "epoch": 0.23, + "learning_rate": 0.000625813440320963, + "loss": 1.4832, + "theoretical_loss": 3.7844793796617116, + "tokens_seen": 744488960 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006257973921765296, + "loss": 1.4227, + "theoretical_loss": 3.784446178015073, + "tokens_seen": 744554496 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006257813440320964, + "loss": 1.4719, + "theoretical_loss": 3.7844129801089244, + "tokens_seen": 744620032 + }, + { + "epoch": 0.23, + "learning_rate": 0.000625765295887663, + "loss": 1.4929, + "theoretical_loss": 3.784379785942517, + "tokens_seen": 744685568 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006257492477432298, + "loss": 1.4681, + "theoretical_loss": 3.784346595515098, + "tokens_seen": 744751104 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006257331995987964, + "loss": 1.5002, + "theoretical_loss": 3.784313408825919, + "tokens_seen": 744816640 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006257171514543632, + "loss": 1.465, + "theoretical_loss": 3.78428022587423, + "tokens_seen": 744882176 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006257011033099298, + "loss": 1.3761, + "theoretical_loss": 3.7842470466592806, + "tokens_seen": 744947712 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006256850551654966, + "loss": 1.4476, + "theoretical_loss": 3.784213871180322, + "tokens_seen": 745013248 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006256690070210632, + "loss": 1.4811, + "theoretical_loss": 3.7841806994366047, + "tokens_seen": 745078784 + }, + { + "epoch": 0.23, + "learning_rate": 0.00062565295887663, + "loss": 1.4352, + "theoretical_loss": 3.7841475314273794, + "tokens_seen": 745144320 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006256369107321966, + "loss": 1.3106, + "theoretical_loss": 3.7841143671518975, + "tokens_seen": 745209856 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006256208625877634, + "loss": 1.6524, + "theoretical_loss": 3.7840812066094105, + "tokens_seen": 745275392 + }, + { + "epoch": 0.23, + "learning_rate": 0.00062560481444333, + "loss": 1.4745, + "theoretical_loss": 3.78404804979917, + "tokens_seen": 745340928 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006255887662988968, + "loss": 1.4636, + "theoretical_loss": 3.7840148967204277, + "tokens_seen": 745406464 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006255727181544634, + "loss": 1.4406, + "theoretical_loss": 3.7839817473724353, + "tokens_seen": 745472000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006255566700100302, + "loss": 1.5217, + "theoretical_loss": 3.783948601754445, + "tokens_seen": 745537536 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006255406218655968, + "loss": 1.526, + "theoretical_loss": 3.7839154598657103, + "tokens_seen": 745603072 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006255245737211636, + "loss": 1.4239, + "theoretical_loss": 3.7838823217054833, + "tokens_seen": 745668608 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006255085255767302, + "loss": 1.5104, + "theoretical_loss": 3.7838491872730167, + "tokens_seen": 745734144 + }, + { + "epoch": 0.23, + "learning_rate": 0.000625492477432297, + "loss": 1.4839, + "theoretical_loss": 3.783816056567564, + "tokens_seen": 745799680 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006254764292878636, + "loss": 1.4844, + "theoretical_loss": 3.783782929588378, + "tokens_seen": 745865216 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006254603811434304, + "loss": 1.4266, + "theoretical_loss": 3.783749806334712, + "tokens_seen": 745930752 + }, + { + "epoch": 0.23, + "learning_rate": 0.000625444332998997, + "loss": 1.5247, + "theoretical_loss": 3.783716686805821, + "tokens_seen": 745996288 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006254282848545638, + "loss": 1.4302, + "theoretical_loss": 3.7836835710009584, + "tokens_seen": 746061824 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006254122367101304, + "loss": 1.5098, + "theoretical_loss": 3.783650458919378, + "tokens_seen": 746127360 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006253961885656972, + "loss": 1.4132, + "theoretical_loss": 3.783617350560334, + "tokens_seen": 746192896 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006253801404212638, + "loss": 1.4577, + "theoretical_loss": 3.7835842459230817, + "tokens_seen": 746258432 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006253640922768306, + "loss": 1.3886, + "theoretical_loss": 3.783551145006876, + "tokens_seen": 746323968 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006253480441323972, + "loss": 1.4362, + "theoretical_loss": 3.783518047810971, + "tokens_seen": 746389504 + }, + { + "epoch": 0.23, + "learning_rate": 0.000625331995987964, + "loss": 1.5406, + "theoretical_loss": 3.7834849543346234, + "tokens_seen": 746455040 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006253159478435306, + "loss": 1.4761, + "theoretical_loss": 3.7834518645770876, + "tokens_seen": 746520576 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006252998996990974, + "loss": 1.4793, + "theoretical_loss": 3.7834187785376194, + "tokens_seen": 746586112 + }, + { + "epoch": 0.23, + "learning_rate": 0.000625283851554664, + "loss": 1.5022, + "theoretical_loss": 3.7833856962154755, + "tokens_seen": 746651648 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006252678034102307, + "loss": 1.4941, + "theoretical_loss": 3.7833526176099106, + "tokens_seen": 746717184 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006252517552657974, + "loss": 1.3594, + "theoretical_loss": 3.7833195427201822, + "tokens_seen": 746782720 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006252357071213641, + "loss": 1.4498, + "theoretical_loss": 3.783286471545547, + "tokens_seen": 746848256 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006252196589769308, + "loss": 1.5244, + "theoretical_loss": 3.7832534040852606, + "tokens_seen": 746913792 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006252036108324975, + "loss": 1.4273, + "theoretical_loss": 3.7832203403385805, + "tokens_seen": 746979328 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006251875626880643, + "loss": 1.4981, + "theoretical_loss": 3.783187280304764, + "tokens_seen": 747044864 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.989722490310669, + "objective/train/avg_token_score": 0.01027749478816986, + "objective/train/docs_used": 277055, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.474670171737671, + "objective/train/original_loss": 1.4911327362060547, + "objective/train/theoretical_loss": 3.783154223983069, + "objective/train/tokens_used": 767570400, + "objective/train/unlikelihood_loss": 0.026652587577700615, + "theoretical_loss": 3.783154223983069, + "tokens_seen": 747110400 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006251715145436309, + "loss": 1.5557, + "theoretical_loss": 3.783154223983069, + "tokens_seen": 747110400 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006251554663991977, + "loss": 1.3979, + "theoretical_loss": 3.783121171372753, + "tokens_seen": 747175936 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006251394182547643, + "loss": 1.4373, + "theoretical_loss": 3.783088122473073, + "tokens_seen": 747241472 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006251233701103309, + "loss": 1.528, + "theoretical_loss": 3.7830550772832874, + "tokens_seen": 747307008 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006251073219658977, + "loss": 1.5224, + "theoretical_loss": 3.7830220358026545, + "tokens_seen": 747372544 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006250912738214643, + "loss": 1.402, + "theoretical_loss": 3.782988998030433, + "tokens_seen": 747438080 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006250752256770311, + "loss": 1.4642, + "theoretical_loss": 3.7829559639658816, + "tokens_seen": 747503616 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006250591775325977, + "loss": 1.3956, + "theoretical_loss": 3.7829229336082584, + "tokens_seen": 747569152 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006250431293881645, + "loss": 1.4297, + "theoretical_loss": 3.7828899069568243, + "tokens_seen": 747634688 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006250270812437311, + "loss": 1.46, + "theoretical_loss": 3.7828568840108363, + "tokens_seen": 747700224 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006250110330992979, + "loss": 1.4143, + "theoretical_loss": 3.782823864769556, + "tokens_seen": 747765760 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006249949849548645, + "loss": 1.4048, + "theoretical_loss": 3.7827908492322413, + "tokens_seen": 747831296 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006249789368104313, + "loss": 1.4524, + "theoretical_loss": 3.7827578373981536, + "tokens_seen": 747896832 + }, + { + "epoch": 0.23, + "learning_rate": 0.000624962888665998, + "loss": 1.5006, + "theoretical_loss": 3.782724829266553, + "tokens_seen": 747962368 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006249468405215647, + "loss": 1.3611, + "theoretical_loss": 3.782691824836699, + "tokens_seen": 748027904 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006249307923771314, + "loss": 1.5574, + "theoretical_loss": 3.7826588241078527, + "tokens_seen": 748093440 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006249147442326981, + "loss": 1.4056, + "theoretical_loss": 3.7826258270792747, + "tokens_seen": 748158976 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006248986960882648, + "loss": 1.4514, + "theoretical_loss": 3.7825928337502264, + "tokens_seen": 748224512 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006248826479438315, + "loss": 1.6401, + "theoretical_loss": 3.7825598441199695, + "tokens_seen": 748290048 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006248665997993982, + "loss": 1.4633, + "theoretical_loss": 3.7825268581877642, + "tokens_seen": 748355584 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006248505516549649, + "loss": 1.4744, + "theoretical_loss": 3.7824938759528726, + "tokens_seen": 748421120 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006248345035105316, + "loss": 1.4934, + "theoretical_loss": 3.7824608974145573, + "tokens_seen": 748486656 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006248184553660983, + "loss": 1.4624, + "theoretical_loss": 3.782427922572079, + "tokens_seen": 748552192 + }, + { + "epoch": 0.23, + "learning_rate": 0.000624802407221665, + "loss": 1.492, + "theoretical_loss": 3.7823949514247017, + "tokens_seen": 748617728 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006247863590772317, + "loss": 1.4379, + "theoretical_loss": 3.782361983971686, + "tokens_seen": 748683264 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006247703109327984, + "loss": 1.521, + "theoretical_loss": 3.7823290202122966, + "tokens_seen": 748748800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006247542627883651, + "loss": 1.3403, + "theoretical_loss": 3.7822960601457947, + "tokens_seen": 748814336 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006247382146439318, + "loss": 1.4042, + "theoretical_loss": 3.782263103771445, + "tokens_seen": 748879872 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006247221664994985, + "loss": 1.5789, + "theoretical_loss": 3.78223015108851, + "tokens_seen": 748945408 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006247061183550652, + "loss": 1.4547, + "theoretical_loss": 3.7821972020962527, + "tokens_seen": 749010944 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006246900702106319, + "loss": 1.4888, + "theoretical_loss": 3.782164256793938, + "tokens_seen": 749076480 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006246740220661986, + "loss": 1.4572, + "theoretical_loss": 3.782131315180829, + "tokens_seen": 749142016 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006246579739217653, + "loss": 1.5011, + "theoretical_loss": 3.782098377256191, + "tokens_seen": 749207552 + }, + { + "epoch": 0.23, + "learning_rate": 0.000624641925777332, + "loss": 1.3903, + "theoretical_loss": 3.7820654430192873, + "tokens_seen": 749273088 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006246258776328987, + "loss": 1.4359, + "theoretical_loss": 3.782032512469383, + "tokens_seen": 749338624 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006246098294884655, + "loss": 1.518, + "theoretical_loss": 3.781999585605743, + "tokens_seen": 749404160 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006245937813440321, + "loss": 1.5575, + "theoretical_loss": 3.781966662427632, + "tokens_seen": 749469696 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006245777331995989, + "loss": 1.5133, + "theoretical_loss": 3.7819337429343154, + "tokens_seen": 749535232 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006245616850551655, + "loss": 1.465, + "theoretical_loss": 3.7819008271250594, + "tokens_seen": 749600768 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006245456369107323, + "loss": 1.5475, + "theoretical_loss": 3.7818679149991286, + "tokens_seen": 749666304 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006245295887662989, + "loss": 1.5406, + "theoretical_loss": 3.7818350065557893, + "tokens_seen": 749731840 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006245135406218657, + "loss": 1.5664, + "theoretical_loss": 3.7818021017943075, + "tokens_seen": 749797376 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006244974924774323, + "loss": 1.3977, + "theoretical_loss": 3.78176920071395, + "tokens_seen": 749862912 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006244814443329991, + "loss": 1.4385, + "theoretical_loss": 3.7817363033139824, + "tokens_seen": 749928448 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006244653961885657, + "loss": 1.4653, + "theoretical_loss": 3.781703409593672, + "tokens_seen": 749993984 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006244493480441325, + "loss": 1.4784, + "theoretical_loss": 3.7816705195522866, + "tokens_seen": 750059520 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006244332998996991, + "loss": 1.4902, + "theoretical_loss": 3.7816376331890913, + "tokens_seen": 750125056 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006244172517552659, + "loss": 1.5507, + "theoretical_loss": 3.7816047505033556, + "tokens_seen": 750190592 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006244012036108325, + "loss": 1.5342, + "theoretical_loss": 3.7815718714943456, + "tokens_seen": 750256128 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006243851554663993, + "loss": 1.4137, + "theoretical_loss": 3.7815389961613297, + "tokens_seen": 750321664 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.9931403398513794, + "objective/train/avg_token_score": 0.006859661545604467, + "objective/train/docs_used": 278273, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4160070419311523, + "objective/train/original_loss": 1.427170991897583, + "objective/train/theoretical_loss": 3.7815061245035757, + "objective/train/tokens_used": 770847200, + "objective/train/unlikelihood_loss": 0.014989304356276989, + "theoretical_loss": 3.7815061245035757, + "tokens_seen": 750387200 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006243691073219659, + "loss": 1.4491, + "theoretical_loss": 3.7815061245035757, + "tokens_seen": 750387200 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006243530591775327, + "loss": 1.4712, + "theoretical_loss": 3.7814732565203517, + "tokens_seen": 750452736 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006243370110330993, + "loss": 1.4903, + "theoretical_loss": 3.7814403922109263, + "tokens_seen": 750518272 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006243209628886661, + "loss": 1.3276, + "theoretical_loss": 3.781407531574568, + "tokens_seen": 750583808 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006243049147442327, + "loss": 1.4338, + "theoretical_loss": 3.7813746746105465, + "tokens_seen": 750649344 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006242888665997995, + "loss": 1.4066, + "theoretical_loss": 3.7813418213181293, + "tokens_seen": 750714880 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006242728184553661, + "loss": 1.3571, + "theoretical_loss": 3.7813089716965864, + "tokens_seen": 750780416 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006242567703109329, + "loss": 1.3921, + "theoretical_loss": 3.7812761257451872, + "tokens_seen": 750845952 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006242407221664995, + "loss": 1.5093, + "theoretical_loss": 3.7812432834632017, + "tokens_seen": 750911488 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006242246740220663, + "loss": 1.433, + "theoretical_loss": 3.7812104448498998, + "tokens_seen": 750977024 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006242086258776329, + "loss": 1.3821, + "theoretical_loss": 3.7811776099045504, + "tokens_seen": 751042560 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006241925777331997, + "loss": 1.5152, + "theoretical_loss": 3.781144778626425, + "tokens_seen": 751108096 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006241765295887663, + "loss": 1.4925, + "theoretical_loss": 3.781111951014794, + "tokens_seen": 751173632 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006241604814443331, + "loss": 1.5189, + "theoretical_loss": 3.781079127068928, + "tokens_seen": 751239168 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006241444332998997, + "loss": 1.4953, + "theoretical_loss": 3.781046306788097, + "tokens_seen": 751304704 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006241283851554665, + "loss": 1.5385, + "theoretical_loss": 3.7810134901715733, + "tokens_seen": 751370240 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006241123370110331, + "loss": 1.5257, + "theoretical_loss": 3.7809806772186283, + "tokens_seen": 751435776 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006240962888665999, + "loss": 1.5091, + "theoretical_loss": 3.780947867928532, + "tokens_seen": 751501312 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006240802407221665, + "loss": 1.468, + "theoretical_loss": 3.7809150623005583, + "tokens_seen": 751566848 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006240641925777333, + "loss": 1.5028, + "theoretical_loss": 3.7808822603339776, + "tokens_seen": 751632384 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006240481444332999, + "loss": 1.5176, + "theoretical_loss": 3.780849462028063, + "tokens_seen": 751697920 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006240320962888667, + "loss": 1.4452, + "theoretical_loss": 3.780816667382086, + "tokens_seen": 751763456 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006240160481444333, + "loss": 1.5047, + "theoretical_loss": 3.7807838763953194, + "tokens_seen": 751828992 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006240000000000001, + "loss": 1.3799, + "theoretical_loss": 3.780751089067037, + "tokens_seen": 751894528 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006239839518555667, + "loss": 1.4798, + "theoretical_loss": 3.78071830539651, + "tokens_seen": 751960064 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006239679037111335, + "loss": 1.5024, + "theoretical_loss": 3.7806855253830136, + "tokens_seen": 752025600 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006239518555667001, + "loss": 1.5164, + "theoretical_loss": 3.78065274902582, + "tokens_seen": 752091136 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006239358074222669, + "loss": 1.4721, + "theoretical_loss": 3.7806199763242025, + "tokens_seen": 752156672 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006239197592778335, + "loss": 1.4463, + "theoretical_loss": 3.780587207277436, + "tokens_seen": 752222208 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006239037111334003, + "loss": 1.4576, + "theoretical_loss": 3.7805544418847936, + "tokens_seen": 752287744 + }, + { + "epoch": 0.23, + "learning_rate": 0.000623887662988967, + "loss": 1.5264, + "theoretical_loss": 3.7805216801455503, + "tokens_seen": 752353280 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006238716148445336, + "loss": 1.365, + "theoretical_loss": 3.7804889220589804, + "tokens_seen": 752418816 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006238555667001003, + "loss": 1.5884, + "theoretical_loss": 3.780456167624358, + "tokens_seen": 752484352 + }, + { + "epoch": 0.23, + "learning_rate": 0.000623839518555667, + "loss": 1.4834, + "theoretical_loss": 3.780423416840958, + "tokens_seen": 752549888 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006238234704112337, + "loss": 1.629, + "theoretical_loss": 3.7803906697080567, + "tokens_seen": 752615424 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006238074222668004, + "loss": 1.4345, + "theoretical_loss": 3.780357926224928, + "tokens_seen": 752680960 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006237913741223672, + "loss": 1.5139, + "theoretical_loss": 3.780325186390848, + "tokens_seen": 752746496 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006237753259779338, + "loss": 1.5903, + "theoretical_loss": 3.780292450205092, + "tokens_seen": 752812032 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006237592778335006, + "loss": 1.4718, + "theoretical_loss": 3.7802597176669366, + "tokens_seen": 752877568 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006237432296890672, + "loss": 1.3748, + "theoretical_loss": 3.7802269887756577, + "tokens_seen": 752943104 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006237271815446338, + "loss": 1.3759, + "theoretical_loss": 3.7801942635305306, + "tokens_seen": 753008640 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006237111334002006, + "loss": 1.5263, + "theoretical_loss": 3.780161541930833, + "tokens_seen": 753074176 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006236950852557672, + "loss": 1.4515, + "theoretical_loss": 3.7801288239758417, + "tokens_seen": 753139712 + }, + { + "epoch": 0.23, + "learning_rate": 0.000623679037111334, + "loss": 1.5066, + "theoretical_loss": 3.7800961096648327, + "tokens_seen": 753205248 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006236629889669006, + "loss": 1.4679, + "theoretical_loss": 3.7800633989970835, + "tokens_seen": 753270784 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006236469408224674, + "loss": 1.4268, + "theoretical_loss": 3.780030691971872, + "tokens_seen": 753336320 + }, + { + "epoch": 0.23, + "learning_rate": 0.000623630892678034, + "loss": 1.5255, + "theoretical_loss": 3.7799979885884754, + "tokens_seen": 753401856 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006236148445336008, + "loss": 1.5501, + "theoretical_loss": 3.7799652888461712, + "tokens_seen": 753467392 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006235987963891675, + "loss": 1.5267, + "theoretical_loss": 3.7799325927442373, + "tokens_seen": 753532928 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006235827482447342, + "loss": 1.4716, + "theoretical_loss": 3.7798999002819524, + "tokens_seen": 753598464 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.9877617955207825, + "objective/train/avg_token_score": 0.012238132767379284, + "objective/train/docs_used": 279492, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5139352083206177, + "objective/train/original_loss": 1.5322635173797607, + "objective/train/theoretical_loss": 3.779867211458595, + "objective/train/tokens_used": 774124000, + "objective/train/unlikelihood_loss": 0.027050426229834557, + "theoretical_loss": 3.779867211458595, + "tokens_seen": 753664000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006235667001003009, + "loss": 1.4614, + "theoretical_loss": 3.779867211458595, + "tokens_seen": 753664000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006235506519558676, + "loss": 1.4868, + "theoretical_loss": 3.7798345262734427, + "tokens_seen": 753729536 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006235346038114343, + "loss": 1.4864, + "theoretical_loss": 3.779801844725775, + "tokens_seen": 753795072 + }, + { + "epoch": 0.23, + "learning_rate": 0.000623518555667001, + "loss": 1.456, + "theoretical_loss": 3.779769166814871, + "tokens_seen": 753860608 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006235025075225677, + "loss": 1.4814, + "theoretical_loss": 3.7797364925400094, + "tokens_seen": 753926144 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006234864593781344, + "loss": 1.4829, + "theoretical_loss": 3.77970382190047, + "tokens_seen": 753991680 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006234704112337011, + "loss": 1.4661, + "theoretical_loss": 3.7796711548955324, + "tokens_seen": 754057216 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006234543630892678, + "loss": 1.5105, + "theoretical_loss": 3.7796384915244765, + "tokens_seen": 754122752 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006234383149448345, + "loss": 1.4405, + "theoretical_loss": 3.7796058317865815, + "tokens_seen": 754188288 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006234222668004012, + "loss": 1.4644, + "theoretical_loss": 3.7795731756811284, + "tokens_seen": 754253824 + }, + { + "epoch": 0.23, + "learning_rate": 0.000623406218655968, + "loss": 1.5667, + "theoretical_loss": 3.779540523207398, + "tokens_seen": 754319360 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006233901705115346, + "loss": 1.5147, + "theoretical_loss": 3.77950787436467, + "tokens_seen": 754384896 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006233741223671014, + "loss": 1.4136, + "theoretical_loss": 3.779475229152226, + "tokens_seen": 754450432 + }, + { + "epoch": 0.23, + "learning_rate": 0.000623358074222668, + "loss": 1.5329, + "theoretical_loss": 3.7794425875693465, + "tokens_seen": 754515968 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006233420260782348, + "loss": 1.4801, + "theoretical_loss": 3.779409949615313, + "tokens_seen": 754581504 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006233259779338014, + "loss": 1.4998, + "theoretical_loss": 3.7793773152894072, + "tokens_seen": 754647040 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006233099297893682, + "loss": 1.5205, + "theoretical_loss": 3.77934468459091, + "tokens_seen": 754712576 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006232938816449348, + "loss": 1.4159, + "theoretical_loss": 3.779312057519104, + "tokens_seen": 754778112 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006232778335005016, + "loss": 1.4853, + "theoretical_loss": 3.7792794340732714, + "tokens_seen": 754843648 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006232617853560682, + "loss": 1.5365, + "theoretical_loss": 3.7792468142526934, + "tokens_seen": 754909184 + }, + { + "epoch": 0.23, + "learning_rate": 0.000623245737211635, + "loss": 1.4184, + "theoretical_loss": 3.7792141980566534, + "tokens_seen": 754974720 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006232296890672016, + "loss": 1.4045, + "theoretical_loss": 3.7791815854844337, + "tokens_seen": 755040256 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006232136409227684, + "loss": 1.4936, + "theoretical_loss": 3.7791489765353177, + "tokens_seen": 755105792 + }, + { + "epoch": 0.23, + "learning_rate": 0.000623197592778335, + "loss": 1.4773, + "theoretical_loss": 3.7791163712085885, + "tokens_seen": 755171328 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006231815446339018, + "loss": 1.4125, + "theoretical_loss": 3.7790837695035284, + "tokens_seen": 755236864 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006231654964894684, + "loss": 1.4318, + "theoretical_loss": 3.7790511714194217, + "tokens_seen": 755302400 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006231494483450352, + "loss": 1.532, + "theoretical_loss": 3.779018576955552, + "tokens_seen": 755367936 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006231334002006018, + "loss": 1.4568, + "theoretical_loss": 3.778985986111203, + "tokens_seen": 755433472 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006231173520561686, + "loss": 1.5811, + "theoretical_loss": 3.778953398885659, + "tokens_seen": 755499008 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006231013039117352, + "loss": 1.525, + "theoretical_loss": 3.7789208152782043, + "tokens_seen": 755564544 + }, + { + "epoch": 0.23, + "learning_rate": 0.000623085255767302, + "loss": 1.4904, + "theoretical_loss": 3.7788882352881235, + "tokens_seen": 755630080 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006230692076228686, + "loss": 1.4189, + "theoretical_loss": 3.7788556589147015, + "tokens_seen": 755695616 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006230531594784354, + "loss": 1.5167, + "theoretical_loss": 3.7788230861572227, + "tokens_seen": 755761152 + }, + { + "epoch": 0.23, + "learning_rate": 0.000623037111334002, + "loss": 1.4354, + "theoretical_loss": 3.778790517014972, + "tokens_seen": 755826688 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006230210631895688, + "loss": 1.4904, + "theoretical_loss": 3.7787579514872354, + "tokens_seen": 755892224 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006230050150451354, + "loss": 1.4204, + "theoretical_loss": 3.7787253895732986, + "tokens_seen": 755957760 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006229889669007022, + "loss": 1.4305, + "theoretical_loss": 3.778692831272447, + "tokens_seen": 756023296 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006229729187562688, + "loss": 1.5075, + "theoretical_loss": 3.778660276583966, + "tokens_seen": 756088832 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006229568706118356, + "loss": 1.5872, + "theoretical_loss": 3.7786277255071425, + "tokens_seen": 756154368 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006229408224674022, + "loss": 1.5157, + "theoretical_loss": 3.7785951780412628, + "tokens_seen": 756219904 + }, + { + "epoch": 0.23, + "learning_rate": 0.000622924774322969, + "loss": 1.4584, + "theoretical_loss": 3.7785626341856133, + "tokens_seen": 756285440 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006229087261785356, + "loss": 1.384, + "theoretical_loss": 3.77853009393948, + "tokens_seen": 756350976 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006228926780341024, + "loss": 1.5551, + "theoretical_loss": 3.7784975573021518, + "tokens_seen": 756416512 + }, + { + "epoch": 0.23, + "learning_rate": 0.000622876629889669, + "loss": 1.377, + "theoretical_loss": 3.7784650242729136, + "tokens_seen": 756482048 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006228605817452358, + "loss": 1.487, + "theoretical_loss": 3.7784324948510544, + "tokens_seen": 756547584 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006228445336008024, + "loss": 1.4421, + "theoretical_loss": 3.7783999690358607, + "tokens_seen": 756613120 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006228284854563692, + "loss": 1.4075, + "theoretical_loss": 3.778367446826621, + "tokens_seen": 756678656 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006228124373119358, + "loss": 1.4993, + "theoretical_loss": 3.778334928222623, + "tokens_seen": 756744192 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006227963891675026, + "loss": 1.3834, + "theoretical_loss": 3.7783024132231544, + "tokens_seen": 756809728 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006227803410230692, + "loss": 1.5241, + "theoretical_loss": 3.7782699018275046, + "tokens_seen": 756875264 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.9940966367721558, + "objective/train/avg_token_score": 0.0059033590368926525, + "objective/train/docs_used": 280659, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5905523300170898, + "objective/train/original_loss": 1.6004141569137573, + "objective/train/theoretical_loss": 3.778237394034961, + "objective/train/tokens_used": 777400800, + "objective/train/unlikelihood_loss": 0.013756711035966873, + "theoretical_loss": 3.778237394034961, + "tokens_seen": 756940800 + }, + { + "epoch": 0.23, + "learning_rate": 0.000622764292878636, + "loss": 1.51, + "theoretical_loss": 3.778237394034961, + "tokens_seen": 756940800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006227482447342026, + "loss": 1.4641, + "theoretical_loss": 3.7782048898448135, + "tokens_seen": 757006336 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006227321965897694, + "loss": 1.5376, + "theoretical_loss": 3.7781723892563503, + "tokens_seen": 757071872 + }, + { + "epoch": 0.23, + "learning_rate": 0.000622716148445336, + "loss": 1.3988, + "theoretical_loss": 3.778139892268861, + "tokens_seen": 757137408 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006227001003009028, + "loss": 1.5514, + "theoretical_loss": 3.7781073988816347, + "tokens_seen": 757202944 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006226840521564694, + "loss": 1.4354, + "theoretical_loss": 3.7780749090939616, + "tokens_seen": 757268480 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006226680040120362, + "loss": 1.5395, + "theoretical_loss": 3.7780424229051306, + "tokens_seen": 757334016 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006226519558676028, + "loss": 1.4605, + "theoretical_loss": 3.778009940314432, + "tokens_seen": 757399552 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006226359077231696, + "loss": 1.4467, + "theoretical_loss": 3.7779774613211563, + "tokens_seen": 757465088 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006226198595787362, + "loss": 1.5092, + "theoretical_loss": 3.7779449859245933, + "tokens_seen": 757530624 + }, + { + "epoch": 0.23, + "learning_rate": 0.000622603811434303, + "loss": 1.4642, + "theoretical_loss": 3.777912514124034, + "tokens_seen": 757596160 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006225877632898696, + "loss": 1.4787, + "theoretical_loss": 3.7778800459187694, + "tokens_seen": 757661696 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006225717151454364, + "loss": 1.497, + "theoretical_loss": 3.77784758130809, + "tokens_seen": 757727232 + }, + { + "epoch": 0.23, + "learning_rate": 0.000622555667001003, + "loss": 1.4477, + "theoretical_loss": 3.7778151202912875, + "tokens_seen": 757792768 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006225396188565698, + "loss": 1.5087, + "theoretical_loss": 3.777782662867653, + "tokens_seen": 757858304 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006225235707121364, + "loss": 1.495, + "theoretical_loss": 3.777750209036478, + "tokens_seen": 757923840 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006225075225677032, + "loss": 1.5009, + "theoretical_loss": 3.7777177587970545, + "tokens_seen": 757989376 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006224914744232698, + "loss": 1.5242, + "theoretical_loss": 3.777685312148674, + "tokens_seen": 758054912 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006224754262788365, + "loss": 1.5043, + "theoretical_loss": 3.77765286909063, + "tokens_seen": 758120448 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006224593781344032, + "loss": 1.3409, + "theoretical_loss": 3.777620429622213, + "tokens_seen": 758185984 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006224433299899699, + "loss": 1.4559, + "theoretical_loss": 3.7775879937427175, + "tokens_seen": 758251520 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006224272818455367, + "loss": 1.4731, + "theoretical_loss": 3.777555561451435, + "tokens_seen": 758317056 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006224112337011033, + "loss": 1.5639, + "theoretical_loss": 3.777523132747659, + "tokens_seen": 758382592 + }, + { + "epoch": 0.23, + "learning_rate": 0.00062239518555667, + "loss": 1.5153, + "theoretical_loss": 3.7774907076306823, + "tokens_seen": 758448128 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006223791374122367, + "loss": 1.4423, + "theoretical_loss": 3.777458286099799, + "tokens_seen": 758513664 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006223630892678035, + "loss": 1.5189, + "theoretical_loss": 3.7774258681543027, + "tokens_seen": 758579200 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006223470411233701, + "loss": 1.5991, + "theoretical_loss": 3.7773934537934863, + "tokens_seen": 758644736 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006223309929789367, + "loss": 1.5773, + "theoretical_loss": 3.7773610430166444, + "tokens_seen": 758710272 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006223149448345035, + "loss": 1.6411, + "theoretical_loss": 3.7773286358230713, + "tokens_seen": 758775808 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006222988966900701, + "loss": 1.4808, + "theoretical_loss": 3.777296232212061, + "tokens_seen": 758841344 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006222828485456369, + "loss": 1.4785, + "theoretical_loss": 3.777263832182909, + "tokens_seen": 758906880 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006222668004012035, + "loss": 1.3962, + "theoretical_loss": 3.7772314357349086, + "tokens_seen": 758972416 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006222507522567703, + "loss": 1.5278, + "theoretical_loss": 3.7771990428673563, + "tokens_seen": 759037952 + }, + { + "epoch": 0.23, + "learning_rate": 0.000622234704112337, + "loss": 1.5213, + "theoretical_loss": 3.7771666535795463, + "tokens_seen": 759103488 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006222186559679037, + "loss": 1.5772, + "theoretical_loss": 3.777134267870774, + "tokens_seen": 759169024 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006222026078234705, + "loss": 1.3815, + "theoretical_loss": 3.7771018857403362, + "tokens_seen": 759234560 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006221865596790371, + "loss": 1.5569, + "theoretical_loss": 3.7770695071875275, + "tokens_seen": 759300096 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006221705115346039, + "loss": 1.5084, + "theoretical_loss": 3.7770371322116443, + "tokens_seen": 759365632 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006221544633901705, + "loss": 1.5247, + "theoretical_loss": 3.7770047608119826, + "tokens_seen": 759431168 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006221384152457373, + "loss": 1.5569, + "theoretical_loss": 3.7769723929878394, + "tokens_seen": 759496704 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006221223671013039, + "loss": 1.4319, + "theoretical_loss": 3.77694002873851, + "tokens_seen": 759562240 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006221063189568707, + "loss": 1.4533, + "theoretical_loss": 3.7769076680632927, + "tokens_seen": 759627776 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006220902708124373, + "loss": 1.4769, + "theoretical_loss": 3.7768753109614837, + "tokens_seen": 759693312 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006220742226680041, + "loss": 1.4012, + "theoretical_loss": 3.7768429574323803, + "tokens_seen": 759758848 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006220581745235707, + "loss": 1.3655, + "theoretical_loss": 3.77681060747528, + "tokens_seen": 759824384 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006220421263791375, + "loss": 1.4804, + "theoretical_loss": 3.7767782610894804, + "tokens_seen": 759889920 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006220260782347041, + "loss": 1.5183, + "theoretical_loss": 3.776745918274279, + "tokens_seen": 759955456 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006220100300902709, + "loss": 1.5205, + "theoretical_loss": 3.776713579028974, + "tokens_seen": 760020992 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006219939819458375, + "loss": 1.5137, + "theoretical_loss": 3.776681243352864, + "tokens_seen": 760086528 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006219779338014043, + "loss": 1.625, + "theoretical_loss": 3.7766489112452466, + "tokens_seen": 760152064 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.9914008378982544, + "objective/train/avg_token_score": 0.008599154651165009, + "objective/train/docs_used": 281753, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5280004739761353, + "objective/train/original_loss": 1.5435279607772827, + "objective/train/theoretical_loss": 3.776616582705421, + "objective/train/tokens_used": 780677600, + "objective/train/unlikelihood_loss": 0.01848743110895157, + "theoretical_loss": 3.776616582705421, + "tokens_seen": 760217600 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006219618856569709, + "loss": 1.47, + "theoretical_loss": 3.776616582705421, + "tokens_seen": 760217600 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006219458375125377, + "loss": 1.4353, + "theoretical_loss": 3.7765842577326856, + "tokens_seen": 760283136 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006219297893681043, + "loss": 1.6459, + "theoretical_loss": 3.7765519363263396, + "tokens_seen": 760348672 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006219137412236711, + "loss": 1.4455, + "theoretical_loss": 3.7765196184856826, + "tokens_seen": 760414208 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006218976930792377, + "loss": 1.574, + "theoretical_loss": 3.776487304210012, + "tokens_seen": 760479744 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006218816449348045, + "loss": 1.4195, + "theoretical_loss": 3.7764549934986302, + "tokens_seen": 760545280 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006218655967903711, + "loss": 1.5408, + "theoretical_loss": 3.776422686350836, + "tokens_seen": 760610816 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006218495486459379, + "loss": 1.4502, + "theoretical_loss": 3.7763903827659275, + "tokens_seen": 760676352 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006218335005015045, + "loss": 1.5595, + "theoretical_loss": 3.776358082743208, + "tokens_seen": 760741888 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006218174523570713, + "loss": 1.4021, + "theoretical_loss": 3.776325786281975, + "tokens_seen": 760807424 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006218014042126379, + "loss": 1.4874, + "theoretical_loss": 3.776293493381531, + "tokens_seen": 760872960 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006217853560682047, + "loss": 1.5518, + "theoretical_loss": 3.776261204041176, + "tokens_seen": 760938496 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006217693079237713, + "loss": 1.387, + "theoretical_loss": 3.7762289182602107, + "tokens_seen": 761004032 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006217532597793381, + "loss": 1.4159, + "theoretical_loss": 3.7761966360379366, + "tokens_seen": 761069568 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006217372116349047, + "loss": 1.4549, + "theoretical_loss": 3.776164357373655, + "tokens_seen": 761135104 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006217211634904715, + "loss": 1.5355, + "theoretical_loss": 3.7761320822666677, + "tokens_seen": 761200640 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006217051153460381, + "loss": 1.55, + "theoretical_loss": 3.7760998107162767, + "tokens_seen": 761266176 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006216890672016049, + "loss": 1.423, + "theoretical_loss": 3.7760675427217825, + "tokens_seen": 761331712 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006216730190571715, + "loss": 1.4499, + "theoretical_loss": 3.776035278282489, + "tokens_seen": 761397248 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006216569709127383, + "loss": 1.4148, + "theoretical_loss": 3.7760030173976977, + "tokens_seen": 761462784 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006216409227683049, + "loss": 1.4519, + "theoretical_loss": 3.775970760066711, + "tokens_seen": 761528320 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006216248746238717, + "loss": 1.3434, + "theoretical_loss": 3.7759385062888318, + "tokens_seen": 761593856 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006216088264794383, + "loss": 1.4119, + "theoretical_loss": 3.7759062560633634, + "tokens_seen": 761659392 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006215927783350051, + "loss": 1.5048, + "theoretical_loss": 3.775874009389608, + "tokens_seen": 761724928 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006215767301905717, + "loss": 1.5316, + "theoretical_loss": 3.77584176626687, + "tokens_seen": 761790464 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006215606820461385, + "loss": 1.495, + "theoretical_loss": 3.7758095266944522, + "tokens_seen": 761856000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006215446339017051, + "loss": 1.4571, + "theoretical_loss": 3.775777290671659, + "tokens_seen": 761921536 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006215285857572719, + "loss": 1.4642, + "theoretical_loss": 3.775745058197794, + "tokens_seen": 761987072 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006215125376128385, + "loss": 1.5657, + "theoretical_loss": 3.7757128292721607, + "tokens_seen": 762052608 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006214964894684053, + "loss": 1.595, + "theoretical_loss": 3.775680603894064, + "tokens_seen": 762118144 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006214804413239719, + "loss": 1.4604, + "theoretical_loss": 3.7756483820628084, + "tokens_seen": 762183680 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006214643931795387, + "loss": 1.4642, + "theoretical_loss": 3.7756161637776984, + "tokens_seen": 762249216 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006214483450351053, + "loss": 1.4959, + "theoretical_loss": 3.775583949038039, + "tokens_seen": 762314752 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006214322968906721, + "loss": 1.4797, + "theoretical_loss": 3.775551737843135, + "tokens_seen": 762380288 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006214162487462387, + "loss": 1.5334, + "theoretical_loss": 3.775519530192293, + "tokens_seen": 762445824 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006214002006018055, + "loss": 1.4758, + "theoretical_loss": 3.775487326084816, + "tokens_seen": 762511360 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006213841524573721, + "loss": 1.4368, + "theoretical_loss": 3.775455125520012, + "tokens_seen": 762576896 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006213681043129389, + "loss": 1.4279, + "theoretical_loss": 3.7754229284971856, + "tokens_seen": 762642432 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006213520561685055, + "loss": 1.5928, + "theoretical_loss": 3.775390735015644, + "tokens_seen": 762707968 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006213360080240723, + "loss": 1.4527, + "theoretical_loss": 3.7753585450746927, + "tokens_seen": 762773504 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006213199598796389, + "loss": 1.4809, + "theoretical_loss": 3.775326358673637, + "tokens_seen": 762839040 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006213039117352057, + "loss": 1.5006, + "theoretical_loss": 3.775294175811786, + "tokens_seen": 762904576 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006212878635907723, + "loss": 1.5651, + "theoretical_loss": 3.775261996488445, + "tokens_seen": 762970112 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006212718154463391, + "loss": 1.3661, + "theoretical_loss": 3.775229820702921, + "tokens_seen": 763035648 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006212557673019057, + "loss": 1.4802, + "theoretical_loss": 3.7751976484545224, + "tokens_seen": 763101184 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006212397191574725, + "loss": 1.5038, + "theoretical_loss": 3.7751654797425553, + "tokens_seen": 763166720 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006212236710130391, + "loss": 1.5359, + "theoretical_loss": 3.775133314566328, + "tokens_seen": 763232256 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006212076228686059, + "loss": 1.4336, + "theoretical_loss": 3.7751011529251475, + "tokens_seen": 763297792 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006211915747241725, + "loss": 1.5043, + "theoretical_loss": 3.7750689948183234, + "tokens_seen": 763363328 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006211755265797393, + "loss": 1.6028, + "theoretical_loss": 3.7750368402451624, + "tokens_seen": 763428864 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.9942235946655273, + "objective/train/avg_token_score": 0.005776367615908384, + "objective/train/docs_used": 282838, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.3910167217254639, + "objective/train/original_loss": 1.3996963500976562, + "objective/train/theoretical_loss": 3.775004689204974, + "objective/train/tokens_used": 783954400, + "objective/train/unlikelihood_loss": 0.014890413731336594, + "theoretical_loss": 3.775004689204974, + "tokens_seen": 763494400 + }, + { + "epoch": 0.23, + "learning_rate": 0.000621159478435306, + "loss": 1.4922, + "theoretical_loss": 3.775004689204974, + "tokens_seen": 763494400 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006211434302908727, + "loss": 1.5808, + "theoretical_loss": 3.7749725416970668, + "tokens_seen": 763559936 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006211273821464393, + "loss": 1.5019, + "theoretical_loss": 3.7749403977207483, + "tokens_seen": 763625472 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006211113340020061, + "loss": 1.454, + "theoretical_loss": 3.774908257275329, + "tokens_seen": 763691008 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006210952858575727, + "loss": 1.5106, + "theoretical_loss": 3.7748761203601164, + "tokens_seen": 763756544 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006210792377131395, + "loss": 1.5039, + "theoretical_loss": 3.774843986974422, + "tokens_seen": 763822080 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006210631895687062, + "loss": 1.4824, + "theoretical_loss": 3.774811857117554, + "tokens_seen": 763887616 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006210471414242728, + "loss": 1.4542, + "theoretical_loss": 3.7747797307888225, + "tokens_seen": 763953152 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006210310932798396, + "loss": 1.5149, + "theoretical_loss": 3.7747476079875373, + "tokens_seen": 764018688 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006210150451354062, + "loss": 1.4748, + "theoretical_loss": 3.7747154887130088, + "tokens_seen": 764084224 + }, + { + "epoch": 0.23, + "learning_rate": 0.000620998996990973, + "loss": 1.5286, + "theoretical_loss": 3.7746833729645473, + "tokens_seen": 764149760 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006209829488465396, + "loss": 1.5132, + "theoretical_loss": 3.774651260741463, + "tokens_seen": 764215296 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006209669007021064, + "loss": 1.5839, + "theoretical_loss": 3.7746191520430674, + "tokens_seen": 764280832 + }, + { + "epoch": 0.23, + "learning_rate": 0.000620950852557673, + "loss": 1.3138, + "theoretical_loss": 3.774587046868671, + "tokens_seen": 764346368 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006209348044132398, + "loss": 1.464, + "theoretical_loss": 3.7745549452175844, + "tokens_seen": 764411904 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006209187562688064, + "loss": 1.4526, + "theoretical_loss": 3.77452284708912, + "tokens_seen": 764477440 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006209027081243732, + "loss": 1.4393, + "theoretical_loss": 3.7744907524825884, + "tokens_seen": 764542976 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006208866599799398, + "loss": 1.5151, + "theoretical_loss": 3.774458661397302, + "tokens_seen": 764608512 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006208706118355066, + "loss": 1.5423, + "theoretical_loss": 3.774426573832572, + "tokens_seen": 764674048 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006208545636910732, + "loss": 1.5632, + "theoretical_loss": 3.7743944897877113, + "tokens_seen": 764739584 + }, + { + "epoch": 0.23, + "learning_rate": 0.00062083851554664, + "loss": 1.491, + "theoretical_loss": 3.7743624092620314, + "tokens_seen": 764805120 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006208224674022066, + "loss": 1.4513, + "theoretical_loss": 3.774330332254846, + "tokens_seen": 764870656 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006208064192577734, + "loss": 1.4243, + "theoretical_loss": 3.774298258765466, + "tokens_seen": 764936192 + }, + { + "epoch": 0.23, + "learning_rate": 0.00062079037111334, + "loss": 1.5068, + "theoretical_loss": 3.774266188793206, + "tokens_seen": 765001728 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006207743229689068, + "loss": 1.5388, + "theoretical_loss": 3.7742341223373774, + "tokens_seen": 765067264 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006207582748244734, + "loss": 1.4578, + "theoretical_loss": 3.774202059397295, + "tokens_seen": 765132800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006207422266800402, + "loss": 1.4572, + "theoretical_loss": 3.774169999972271, + "tokens_seen": 765198336 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006207261785356068, + "loss": 1.4017, + "theoretical_loss": 3.77413794406162, + "tokens_seen": 765263872 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006207101303911736, + "loss": 1.4188, + "theoretical_loss": 3.7741058916646555, + "tokens_seen": 765329408 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006206940822467402, + "loss": 1.497, + "theoretical_loss": 3.7740738427806915, + "tokens_seen": 765394944 + }, + { + "epoch": 0.23, + "learning_rate": 0.000620678034102307, + "loss": 1.4766, + "theoretical_loss": 3.7740417974090423, + "tokens_seen": 765460480 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006206619859578736, + "loss": 1.5294, + "theoretical_loss": 3.774009755549022, + "tokens_seen": 765526016 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006206459378134404, + "loss": 1.585, + "theoretical_loss": 3.773977717199946, + "tokens_seen": 765591552 + }, + { + "epoch": 0.23, + "learning_rate": 0.000620629889669007, + "loss": 1.4988, + "theoretical_loss": 3.773945682361128, + "tokens_seen": 765657088 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006206138415245738, + "loss": 1.5553, + "theoretical_loss": 3.7739136510318834, + "tokens_seen": 765722624 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006205977933801404, + "loss": 1.3457, + "theoretical_loss": 3.7738816232115275, + "tokens_seen": 765788160 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006205817452357072, + "loss": 1.6278, + "theoretical_loss": 3.773849598899376, + "tokens_seen": 765853696 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006205656970912738, + "loss": 1.5576, + "theoretical_loss": 3.773817578094744, + "tokens_seen": 765919232 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006205496489468406, + "loss": 1.5624, + "theoretical_loss": 3.7737855607969477, + "tokens_seen": 765984768 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006205336008024072, + "loss": 1.4686, + "theoretical_loss": 3.7737535470053025, + "tokens_seen": 766050304 + }, + { + "epoch": 0.23, + "learning_rate": 0.000620517552657974, + "loss": 1.4741, + "theoretical_loss": 3.7737215367191252, + "tokens_seen": 766115840 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006205015045135406, + "loss": 1.5085, + "theoretical_loss": 3.773689529937731, + "tokens_seen": 766181376 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006204854563691074, + "loss": 1.4334, + "theoretical_loss": 3.773657526660438, + "tokens_seen": 766246912 + }, + { + "epoch": 0.23, + "learning_rate": 0.000620469408224674, + "loss": 1.4435, + "theoretical_loss": 3.773625526886562, + "tokens_seen": 766312448 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006204533600802408, + "loss": 1.5762, + "theoretical_loss": 3.7735935306154196, + "tokens_seen": 766377984 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006204373119358074, + "loss": 1.454, + "theoretical_loss": 3.7735615378463283, + "tokens_seen": 766443520 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006204212637913742, + "loss": 1.6265, + "theoretical_loss": 3.7735295485786056, + "tokens_seen": 766509056 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006204052156469408, + "loss": 1.4781, + "theoretical_loss": 3.773497562811569, + "tokens_seen": 766574592 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006203891675025076, + "loss": 1.4967, + "theoretical_loss": 3.773465580544536, + "tokens_seen": 766640128 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006203731193580742, + "loss": 1.5492, + "theoretical_loss": 3.7734336017768237, + "tokens_seen": 766705664 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.9903896450996399, + "objective/train/avg_token_score": 0.009610452689230442, + "objective/train/docs_used": 283988, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4810055494308472, + "objective/train/original_loss": 1.4948363304138184, + "objective/train/theoretical_loss": 3.773401626507752, + "objective/train/tokens_used": 787231200, + "objective/train/unlikelihood_loss": 0.024404456838965416, + "theoretical_loss": 3.773401626507752, + "tokens_seen": 766771200 + }, + { + "epoch": 0.23, + "learning_rate": 0.000620357071213641, + "loss": 1.4389, + "theoretical_loss": 3.773401626507752, + "tokens_seen": 766771200 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006203410230692076, + "loss": 1.5126, + "theoretical_loss": 3.773369654736637, + "tokens_seen": 766836736 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006203249749247744, + "loss": 1.4132, + "theoretical_loss": 3.773337686462799, + "tokens_seen": 766902272 + }, + { + "epoch": 0.23, + "learning_rate": 0.000620308926780341, + "loss": 1.5198, + "theoretical_loss": 3.7733057216855554, + "tokens_seen": 766967808 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006202928786359078, + "loss": 1.5264, + "theoretical_loss": 3.7732737604042255, + "tokens_seen": 767033344 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006202768304914744, + "loss": 1.4349, + "theoretical_loss": 3.773241802618129, + "tokens_seen": 767098880 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006202607823470412, + "loss": 1.4406, + "theoretical_loss": 3.7732098483265837, + "tokens_seen": 767164416 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006202447342026078, + "loss": 1.517, + "theoretical_loss": 3.7731778975289094, + "tokens_seen": 767229952 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006202286860581746, + "loss": 1.4844, + "theoretical_loss": 3.7731459502244262, + "tokens_seen": 767295488 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006202126379137412, + "loss": 1.4966, + "theoretical_loss": 3.773114006412454, + "tokens_seen": 767361024 + }, + { + "epoch": 0.23, + "learning_rate": 0.000620196589769308, + "loss": 1.4077, + "theoretical_loss": 3.773082066092312, + "tokens_seen": 767426560 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006201805416248746, + "loss": 1.5641, + "theoretical_loss": 3.773050129263321, + "tokens_seen": 767492096 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006201644934804414, + "loss": 1.4881, + "theoretical_loss": 3.7730181959248013, + "tokens_seen": 767557632 + }, + { + "epoch": 0.23, + "learning_rate": 0.000620148445336008, + "loss": 1.4627, + "theoretical_loss": 3.7729862660760727, + "tokens_seen": 767623168 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006201323971915748, + "loss": 1.4805, + "theoretical_loss": 3.7729543397164567, + "tokens_seen": 767688704 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006201163490471414, + "loss": 1.4857, + "theoretical_loss": 3.772922416845274, + "tokens_seen": 767754240 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006201003009027082, + "loss": 1.4748, + "theoretical_loss": 3.7728904974618462, + "tokens_seen": 767819776 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006200842527582748, + "loss": 1.4818, + "theoretical_loss": 3.7728585815654934, + "tokens_seen": 767885312 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006200682046138416, + "loss": 1.4623, + "theoretical_loss": 3.7728266691555383, + "tokens_seen": 767950848 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006200521564694082, + "loss": 1.456, + "theoretical_loss": 3.7727947602313017, + "tokens_seen": 768016384 + }, + { + "epoch": 0.23, + "learning_rate": 0.000620036108324975, + "loss": 1.4392, + "theoretical_loss": 3.772762854792106, + "tokens_seen": 768081920 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006200200601805416, + "loss": 1.4359, + "theoretical_loss": 3.772730952837273, + "tokens_seen": 768147456 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006200040120361084, + "loss": 1.4395, + "theoretical_loss": 3.7726990543661256, + "tokens_seen": 768212992 + }, + { + "epoch": 0.23, + "learning_rate": 0.000619987963891675, + "loss": 1.484, + "theoretical_loss": 3.7726671593779852, + "tokens_seen": 768278528 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006199719157472418, + "loss": 1.5456, + "theoretical_loss": 3.7726352678721753, + "tokens_seen": 768344064 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006199558676028084, + "loss": 1.5427, + "theoretical_loss": 3.772603379848018, + "tokens_seen": 768409600 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006199398194583752, + "loss": 1.4515, + "theoretical_loss": 3.772571495304837, + "tokens_seen": 768475136 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006199237713139418, + "loss": 1.4507, + "theoretical_loss": 3.7725396142419547, + "tokens_seen": 768540672 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006199077231695086, + "loss": 1.4425, + "theoretical_loss": 3.772507736658695, + "tokens_seen": 768606208 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006198916750250752, + "loss": 1.5016, + "theoretical_loss": 3.7724758625543817, + "tokens_seen": 768671744 + }, + { + "epoch": 0.23, + "learning_rate": 0.000619875626880642, + "loss": 1.5668, + "theoretical_loss": 3.7724439919283386, + "tokens_seen": 768737280 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006198595787362086, + "loss": 1.6025, + "theoretical_loss": 3.7724121247798887, + "tokens_seen": 768802816 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006198435305917754, + "loss": 1.5159, + "theoretical_loss": 3.7723802611083572, + "tokens_seen": 768868352 + }, + { + "epoch": 0.23, + "learning_rate": 0.000619827482447342, + "loss": 1.4013, + "theoretical_loss": 3.7723484009130677, + "tokens_seen": 768933888 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006198114343029088, + "loss": 1.6299, + "theoretical_loss": 3.772316544193345, + "tokens_seen": 768999424 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006197953861584754, + "loss": 1.5112, + "theoretical_loss": 3.772284690948514, + "tokens_seen": 769064960 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006197793380140422, + "loss": 1.4389, + "theoretical_loss": 3.7722528411778997, + "tokens_seen": 769130496 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006197632898696088, + "loss": 1.5096, + "theoretical_loss": 3.7722209948808265, + "tokens_seen": 769196032 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006197472417251756, + "loss": 1.5266, + "theoretical_loss": 3.77218915205662, + "tokens_seen": 769261568 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006197311935807422, + "loss": 1.5311, + "theoretical_loss": 3.772157312704606, + "tokens_seen": 769327104 + }, + { + "epoch": 0.23, + "learning_rate": 0.000619715145436309, + "loss": 1.4712, + "theoretical_loss": 3.7721254768241104, + "tokens_seen": 769392640 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006196990972918757, + "loss": 1.4549, + "theoretical_loss": 3.772093644414458, + "tokens_seen": 769458176 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006196830491474424, + "loss": 1.4128, + "theoretical_loss": 3.7720618154749754, + "tokens_seen": 769523712 + }, + { + "epoch": 0.23, + "learning_rate": 0.000619667001003009, + "loss": 1.4676, + "theoretical_loss": 3.772029990004989, + "tokens_seen": 769589248 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006196509528585757, + "loss": 1.5363, + "theoretical_loss": 3.771998168003825, + "tokens_seen": 769654784 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006196349047141425, + "loss": 1.3838, + "theoretical_loss": 3.77196634947081, + "tokens_seen": 769720320 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006196188565697091, + "loss": 1.5015, + "theoretical_loss": 3.7719345344052706, + "tokens_seen": 769785856 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006196028084252759, + "loss": 1.4935, + "theoretical_loss": 3.771902722806534, + "tokens_seen": 769851392 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006195867602808425, + "loss": 1.3821, + "theoretical_loss": 3.7718709146739275, + "tokens_seen": 769916928 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006195707121364093, + "loss": 1.4961, + "theoretical_loss": 3.771839110006778, + "tokens_seen": 769982464 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.9924308061599731, + "objective/train/avg_token_score": 0.007569195237010717, + "objective/train/docs_used": 285055, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5147517919540405, + "objective/train/original_loss": 1.5266177654266357, + "objective/train/theoretical_loss": 3.7718073088044135, + "objective/train/tokens_used": 790508000, + "objective/train/unlikelihood_loss": 0.018325457349419594, + "theoretical_loss": 3.7718073088044135, + "tokens_seen": 770048000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006195546639919759, + "loss": 1.5065, + "theoretical_loss": 3.7718073088044135, + "tokens_seen": 770048000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006195386158475427, + "loss": 1.4928, + "theoretical_loss": 3.7717755110661617, + "tokens_seen": 770113536 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006195225677031093, + "loss": 1.4313, + "theoretical_loss": 3.7717437167913497, + "tokens_seen": 770179072 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006195065195586761, + "loss": 1.4905, + "theoretical_loss": 3.771711925979307, + "tokens_seen": 770244608 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006194904714142427, + "loss": 1.532, + "theoretical_loss": 3.7716801386293604, + "tokens_seen": 770310144 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006194744232698095, + "loss": 1.3807, + "theoretical_loss": 3.7716483547408393, + "tokens_seen": 770375680 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006194583751253761, + "loss": 1.4768, + "theoretical_loss": 3.7716165743130725, + "tokens_seen": 770441216 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006194423269809429, + "loss": 1.3627, + "theoretical_loss": 3.7715847973453878, + "tokens_seen": 770506752 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006194262788365095, + "loss": 1.4482, + "theoretical_loss": 3.7715530238371153, + "tokens_seen": 770572288 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006194102306920763, + "loss": 1.4345, + "theoretical_loss": 3.7715212537875837, + "tokens_seen": 770637824 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006193941825476429, + "loss": 1.442, + "theoretical_loss": 3.7714894871961233, + "tokens_seen": 770703360 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006193781344032097, + "loss": 1.4739, + "theoretical_loss": 3.7714577240620617, + "tokens_seen": 770768896 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006193620862587763, + "loss": 1.4535, + "theoretical_loss": 3.771425964384731, + "tokens_seen": 770834432 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006193460381143431, + "loss": 1.5086, + "theoretical_loss": 3.7713942081634593, + "tokens_seen": 770899968 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006193299899699097, + "loss": 1.4445, + "theoretical_loss": 3.7713624553975778, + "tokens_seen": 770965504 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006193139418254765, + "loss": 1.5143, + "theoretical_loss": 3.7713307060864167, + "tokens_seen": 771031040 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006192978936810431, + "loss": 1.5177, + "theoretical_loss": 3.771298960229306, + "tokens_seen": 771096576 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006192818455366099, + "loss": 1.5254, + "theoretical_loss": 3.7712672178255775, + "tokens_seen": 771162112 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006192657973921765, + "loss": 1.5898, + "theoretical_loss": 3.7712354788745612, + "tokens_seen": 771227648 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006192497492477433, + "loss": 1.3617, + "theoretical_loss": 3.771203743375588, + "tokens_seen": 771293184 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006192337011033099, + "loss": 1.4843, + "theoretical_loss": 3.77117201132799, + "tokens_seen": 771358720 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006192176529588767, + "loss": 1.4742, + "theoretical_loss": 3.771140282731098, + "tokens_seen": 771424256 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006192016048144433, + "loss": 1.4712, + "theoretical_loss": 3.7711085575842436, + "tokens_seen": 771489792 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006191855566700101, + "loss": 1.4664, + "theoretical_loss": 3.7710768358867597, + "tokens_seen": 771555328 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006191695085255767, + "loss": 1.5273, + "theoretical_loss": 3.771045117637977, + "tokens_seen": 771620864 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006191534603811435, + "loss": 1.4721, + "theoretical_loss": 3.7710134028372284, + "tokens_seen": 771686400 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006191374122367101, + "loss": 1.6209, + "theoretical_loss": 3.770981691483846, + "tokens_seen": 771751936 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006191213640922769, + "loss": 1.4729, + "theoretical_loss": 3.7709499835771627, + "tokens_seen": 771817472 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006191053159478435, + "loss": 1.4356, + "theoretical_loss": 3.7709182791165112, + "tokens_seen": 771883008 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006190892678034103, + "loss": 1.4616, + "theoretical_loss": 3.770886578101224, + "tokens_seen": 771948544 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006190732196589769, + "loss": 1.5104, + "theoretical_loss": 3.770854880530635, + "tokens_seen": 772014080 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006190571715145437, + "loss": 1.4446, + "theoretical_loss": 3.770823186404077, + "tokens_seen": 772079616 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006190411233701103, + "loss": 1.4958, + "theoretical_loss": 3.770791495720883, + "tokens_seen": 772145152 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006190250752256771, + "loss": 1.4472, + "theoretical_loss": 3.770759808480388, + "tokens_seen": 772210688 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006190090270812437, + "loss": 1.3216, + "theoretical_loss": 3.770728124681925, + "tokens_seen": 772276224 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006189929789368105, + "loss": 1.6038, + "theoretical_loss": 3.7706964443248276, + "tokens_seen": 772341760 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006189769307923771, + "loss": 1.5053, + "theoretical_loss": 3.7706647674084315, + "tokens_seen": 772407296 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006189608826479439, + "loss": 1.3833, + "theoretical_loss": 3.77063309393207, + "tokens_seen": 772472832 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006189448345035105, + "loss": 1.5009, + "theoretical_loss": 3.7706014238950782, + "tokens_seen": 772538368 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006189287863590773, + "loss": 1.3719, + "theoretical_loss": 3.7705697572967902, + "tokens_seen": 772603904 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006189127382146439, + "loss": 1.535, + "theoretical_loss": 3.7705380941365423, + "tokens_seen": 772669440 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006188966900702107, + "loss": 1.3804, + "theoretical_loss": 3.770506434413668, + "tokens_seen": 772734976 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006188806419257773, + "loss": 1.5496, + "theoretical_loss": 3.770474778127504, + "tokens_seen": 772800512 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006188645937813441, + "loss": 1.4903, + "theoretical_loss": 3.7704431252773856, + "tokens_seen": 772866048 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006188485456369107, + "loss": 1.5877, + "theoretical_loss": 3.770411475862648, + "tokens_seen": 772931584 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006188324974924775, + "loss": 1.4858, + "theoretical_loss": 3.7703798298826277, + "tokens_seen": 772997120 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006188164493480441, + "loss": 1.671, + "theoretical_loss": 3.77034818733666, + "tokens_seen": 773062656 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006188004012036109, + "loss": 1.5545, + "theoretical_loss": 3.7703165482240824, + "tokens_seen": 773128192 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006187843530591775, + "loss": 1.4602, + "theoretical_loss": 3.77028491254423, + "tokens_seen": 773193728 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006187683049147443, + "loss": 1.3798, + "theoretical_loss": 3.7702532802964406, + "tokens_seen": 773259264 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.9907318353652954, + "objective/train/avg_token_score": 0.009268154390156269, + "objective/train/docs_used": 286255, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.3891985416412354, + "objective/train/original_loss": 1.4046556949615479, + "objective/train/theoretical_loss": 3.7702216514800506, + "objective/train/tokens_used": 793784800, + "objective/train/unlikelihood_loss": 0.021510187536478043, + "theoretical_loss": 3.7702216514800506, + "tokens_seen": 773324800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006187522567703109, + "loss": 1.4829, + "theoretical_loss": 3.7702216514800506, + "tokens_seen": 773324800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006187362086258777, + "loss": 1.4818, + "theoretical_loss": 3.770190026094397, + "tokens_seen": 773390336 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006187201604814443, + "loss": 1.4251, + "theoretical_loss": 3.7701584041388165, + "tokens_seen": 773455872 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006187041123370111, + "loss": 1.526, + "theoretical_loss": 3.7701267856126472, + "tokens_seen": 773521408 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006186880641925777, + "loss": 1.4461, + "theoretical_loss": 3.7700951705152264, + "tokens_seen": 773586944 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006186720160481445, + "loss": 1.5615, + "theoretical_loss": 3.7700635588458926, + "tokens_seen": 773652480 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006186559679037111, + "loss": 1.5131, + "theoretical_loss": 3.7700319506039826, + "tokens_seen": 773718016 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006186399197592779, + "loss": 1.552, + "theoretical_loss": 3.7700003457888345, + "tokens_seen": 773783552 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006186238716148445, + "loss": 1.5023, + "theoretical_loss": 3.7699687443997876, + "tokens_seen": 773849088 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006186078234704113, + "loss": 1.5422, + "theoretical_loss": 3.7699371464361797, + "tokens_seen": 773914624 + }, + { + "epoch": 0.23, + "learning_rate": 0.000618591775325978, + "loss": 1.4977, + "theoretical_loss": 3.7699055518973497, + "tokens_seen": 773980160 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006185757271815447, + "loss": 1.4378, + "theoretical_loss": 3.769873960782636, + "tokens_seen": 774045696 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006185596790371114, + "loss": 1.4271, + "theoretical_loss": 3.769842373091379, + "tokens_seen": 774111232 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006185436308926781, + "loss": 1.6267, + "theoretical_loss": 3.769810788822916, + "tokens_seen": 774176768 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006185275827482449, + "loss": 1.4693, + "theoretical_loss": 3.769779207976588, + "tokens_seen": 774242304 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006185115346038115, + "loss": 1.6012, + "theoretical_loss": 3.7697476305517337, + "tokens_seen": 774307840 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006184954864593783, + "loss": 1.4673, + "theoretical_loss": 3.7697160565476935, + "tokens_seen": 774373376 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006184794383149449, + "loss": 1.423, + "theoretical_loss": 3.769684485963807, + "tokens_seen": 774438912 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006184633901705117, + "loss": 1.5638, + "theoretical_loss": 3.7696529187994137, + "tokens_seen": 774504448 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006184473420260783, + "loss": 1.5209, + "theoretical_loss": 3.769621355053855, + "tokens_seen": 774569984 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006184312938816451, + "loss": 1.5808, + "theoretical_loss": 3.769589794726471, + "tokens_seen": 774635520 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006184152457372117, + "loss": 1.4716, + "theoretical_loss": 3.7695582378166024, + "tokens_seen": 774701056 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006183991975927783, + "loss": 1.5837, + "theoretical_loss": 3.7695266843235897, + "tokens_seen": 774766592 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006183831494483451, + "loss": 1.5274, + "theoretical_loss": 3.7694951342467746, + "tokens_seen": 774832128 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006183671013039117, + "loss": 1.5091, + "theoretical_loss": 3.7694635875854985, + "tokens_seen": 774897664 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006183510531594785, + "loss": 1.5187, + "theoretical_loss": 3.7694320443391023, + "tokens_seen": 774963200 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006183350050150452, + "loss": 1.432, + "theoretical_loss": 3.769400504506927, + "tokens_seen": 775028736 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006183189568706119, + "loss": 1.5862, + "theoretical_loss": 3.7693689680883153, + "tokens_seen": 775094272 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006183029087261786, + "loss": 1.3936, + "theoretical_loss": 3.769337435082609, + "tokens_seen": 775159808 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006182868605817453, + "loss": 1.5218, + "theoretical_loss": 3.769305905489151, + "tokens_seen": 775225344 + }, + { + "epoch": 0.23, + "learning_rate": 0.000618270812437312, + "loss": 1.6436, + "theoretical_loss": 3.769274379307282, + "tokens_seen": 775290880 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006182547642928786, + "loss": 1.4569, + "theoretical_loss": 3.7692428565363456, + "tokens_seen": 775356416 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006182387161484454, + "loss": 1.5283, + "theoretical_loss": 3.769211337175684, + "tokens_seen": 775421952 + }, + { + "epoch": 0.23, + "learning_rate": 0.000618222668004012, + "loss": 1.4693, + "theoretical_loss": 3.7691798212246406, + "tokens_seen": 775487488 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006182066198595788, + "loss": 1.4382, + "theoretical_loss": 3.769148308682558, + "tokens_seen": 775553024 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006181905717151454, + "loss": 1.5621, + "theoretical_loss": 3.76911679954878, + "tokens_seen": 775618560 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006181745235707122, + "loss": 1.4785, + "theoretical_loss": 3.7690852938226493, + "tokens_seen": 775684096 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006181584754262788, + "loss": 1.4758, + "theoretical_loss": 3.7690537915035103, + "tokens_seen": 775749632 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006181424272818456, + "loss": 1.6652, + "theoretical_loss": 3.769022292590706, + "tokens_seen": 775815168 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006181263791374122, + "loss": 1.5445, + "theoretical_loss": 3.768990797083581, + "tokens_seen": 775880704 + }, + { + "epoch": 0.24, + "learning_rate": 0.000618110330992979, + "loss": 1.4402, + "theoretical_loss": 3.7689593049814794, + "tokens_seen": 775946240 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006180942828485456, + "loss": 1.397, + "theoretical_loss": 3.768927816283745, + "tokens_seen": 776011776 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006180782347041124, + "loss": 1.5142, + "theoretical_loss": 3.768896330989723, + "tokens_seen": 776077312 + }, + { + "epoch": 0.24, + "learning_rate": 0.000618062186559679, + "loss": 1.5863, + "theoretical_loss": 3.7688648490987577, + "tokens_seen": 776142848 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006180461384152458, + "loss": 1.5126, + "theoretical_loss": 3.768833370610194, + "tokens_seen": 776208384 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006180300902708124, + "loss": 1.4059, + "theoretical_loss": 3.768801895523377, + "tokens_seen": 776273920 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006180140421263792, + "loss": 1.4567, + "theoretical_loss": 3.768770423837652, + "tokens_seen": 776339456 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006179979939819458, + "loss": 1.6051, + "theoretical_loss": 3.7687389555523643, + "tokens_seen": 776404992 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006179819458375126, + "loss": 1.4607, + "theoretical_loss": 3.7687074906668596, + "tokens_seen": 776470528 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006179658976930792, + "loss": 1.4857, + "theoretical_loss": 3.768676029180484, + "tokens_seen": 776536064 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.9946318864822388, + "objective/train/avg_token_score": 0.005368185229599476, + "objective/train/docs_used": 287303, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5221563577651978, + "objective/train/original_loss": 1.5319581031799316, + "objective/train/theoretical_loss": 3.768644571092583, + "objective/train/tokens_used": 797061600, + "objective/train/unlikelihood_loss": 0.01062019169330597, + "theoretical_loss": 3.768644571092583, + "tokens_seen": 776601600 + }, + { + "epoch": 0.24, + "learning_rate": 0.000617949849548646, + "loss": 1.5313, + "theoretical_loss": 3.768644571092583, + "tokens_seen": 776601600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006179338014042126, + "loss": 1.7139, + "theoretical_loss": 3.768613116402503, + "tokens_seen": 776667136 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006179177532597794, + "loss": 1.4222, + "theoretical_loss": 3.76858166510959, + "tokens_seen": 776732672 + }, + { + "epoch": 0.24, + "learning_rate": 0.000617901705115346, + "loss": 1.5268, + "theoretical_loss": 3.768550217213191, + "tokens_seen": 776798208 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006178856569709128, + "loss": 1.5615, + "theoretical_loss": 3.7685187727126523, + "tokens_seen": 776863744 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006178696088264794, + "loss": 1.4205, + "theoretical_loss": 3.768487331607321, + "tokens_seen": 776929280 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006178535606820462, + "loss": 1.449, + "theoretical_loss": 3.7684558938965442, + "tokens_seen": 776994816 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006178375125376128, + "loss": 1.4615, + "theoretical_loss": 3.768424459579669, + "tokens_seen": 777060352 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006178214643931796, + "loss": 1.5861, + "theoretical_loss": 3.7683930286560425, + "tokens_seen": 777125888 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006178054162487462, + "loss": 1.5032, + "theoretical_loss": 3.7683616011250134, + "tokens_seen": 777191424 + }, + { + "epoch": 0.24, + "learning_rate": 0.000617789368104313, + "loss": 1.3963, + "theoretical_loss": 3.768330176985928, + "tokens_seen": 777256960 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006177733199598796, + "loss": 1.5043, + "theoretical_loss": 3.7682987562381354, + "tokens_seen": 777322496 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006177572718154464, + "loss": 1.4765, + "theoretical_loss": 3.768267338880983, + "tokens_seen": 777388032 + }, + { + "epoch": 0.24, + "learning_rate": 0.000617741223671013, + "loss": 1.514, + "theoretical_loss": 3.7682359249138195, + "tokens_seen": 777453568 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006177251755265798, + "loss": 1.4247, + "theoretical_loss": 3.7682045143359937, + "tokens_seen": 777519104 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006177091273821464, + "loss": 1.556, + "theoretical_loss": 3.7681731071468536, + "tokens_seen": 777584640 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006176930792377132, + "loss": 1.5537, + "theoretical_loss": 3.7681417033457487, + "tokens_seen": 777650176 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006176770310932798, + "loss": 1.5266, + "theoretical_loss": 3.768110302932027, + "tokens_seen": 777715712 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006176609829488466, + "loss": 1.4852, + "theoretical_loss": 3.768078905905039, + "tokens_seen": 777781248 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006176449348044132, + "loss": 1.4686, + "theoretical_loss": 3.7680475122641335, + "tokens_seen": 777846784 + }, + { + "epoch": 0.24, + "learning_rate": 0.00061762888665998, + "loss": 1.5593, + "theoretical_loss": 3.7680161220086603, + "tokens_seen": 777912320 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006176128385155466, + "loss": 1.4386, + "theoretical_loss": 3.7679847351379685, + "tokens_seen": 777977856 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006175967903711134, + "loss": 1.4017, + "theoretical_loss": 3.7679533516514088, + "tokens_seen": 778043392 + }, + { + "epoch": 0.24, + "learning_rate": 0.00061758074222668, + "loss": 1.5608, + "theoretical_loss": 3.767921971548331, + "tokens_seen": 778108928 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006175646940822468, + "loss": 1.4687, + "theoretical_loss": 3.767890594828085, + "tokens_seen": 778174464 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006175486459378134, + "loss": 1.3592, + "theoretical_loss": 3.7678592214900224, + "tokens_seen": 778240000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006175325977933802, + "loss": 1.4003, + "theoretical_loss": 3.7678278515334926, + "tokens_seen": 778305536 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006175165496489468, + "loss": 1.372, + "theoretical_loss": 3.7677964849578474, + "tokens_seen": 778371072 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006175005015045136, + "loss": 1.3294, + "theoretical_loss": 3.7677651217624373, + "tokens_seen": 778436608 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006174844533600802, + "loss": 1.4498, + "theoretical_loss": 3.767733761946613, + "tokens_seen": 778502144 + }, + { + "epoch": 0.24, + "learning_rate": 0.000617468405215647, + "loss": 1.4859, + "theoretical_loss": 3.767702405509727, + "tokens_seen": 778567680 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006174523570712136, + "loss": 1.4484, + "theoretical_loss": 3.7676710524511305, + "tokens_seen": 778633216 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006174363089267804, + "loss": 1.4553, + "theoretical_loss": 3.767639702770175, + "tokens_seen": 778698752 + }, + { + "epoch": 0.24, + "learning_rate": 0.000617420260782347, + "loss": 1.321, + "theoretical_loss": 3.767608356466212, + "tokens_seen": 778764288 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006174042126379138, + "loss": 1.3749, + "theoretical_loss": 3.7675770135385944, + "tokens_seen": 778829824 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006173881644934805, + "loss": 1.5177, + "theoretical_loss": 3.7675456739866746, + "tokens_seen": 778895360 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006173721163490472, + "loss": 1.5233, + "theoretical_loss": 3.7675143378098035, + "tokens_seen": 778960896 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006173560682046139, + "loss": 1.4458, + "theoretical_loss": 3.7674830050073354, + "tokens_seen": 779026432 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006173400200601806, + "loss": 1.4715, + "theoretical_loss": 3.7674516755786227, + "tokens_seen": 779091968 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006173239719157473, + "loss": 1.4372, + "theoretical_loss": 3.767420349523018, + "tokens_seen": 779157504 + }, + { + "epoch": 0.24, + "learning_rate": 0.000617307923771314, + "loss": 1.5001, + "theoretical_loss": 3.7673890268398744, + "tokens_seen": 779223040 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006172918756268807, + "loss": 1.558, + "theoretical_loss": 3.7673577075285456, + "tokens_seen": 779288576 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006172758274824474, + "loss": 1.4045, + "theoretical_loss": 3.767326391588385, + "tokens_seen": 779354112 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006172597793380141, + "loss": 1.4972, + "theoretical_loss": 3.767295079018746, + "tokens_seen": 779419648 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006172437311935808, + "loss": 1.4678, + "theoretical_loss": 3.7672637698189835, + "tokens_seen": 779485184 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006172276830491475, + "loss": 1.4769, + "theoretical_loss": 3.7672324639884502, + "tokens_seen": 779550720 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006172116349047142, + "loss": 1.5534, + "theoretical_loss": 3.7672011615265015, + "tokens_seen": 779616256 + }, + { + "epoch": 0.24, + "learning_rate": 0.000617195586760281, + "loss": 1.5306, + "theoretical_loss": 3.7671698624324907, + "tokens_seen": 779681792 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006171795386158476, + "loss": 1.493, + "theoretical_loss": 3.767138566705773, + "tokens_seen": 779747328 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006171634904714144, + "loss": 1.5393, + "theoretical_loss": 3.7671072743457032, + "tokens_seen": 779812864 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.9958833456039429, + "objective/train/avg_token_score": 0.0041166674345731735, + "objective/train/docs_used": 288522, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4622470140457153, + "objective/train/original_loss": 1.4691448211669922, + "objective/train/theoretical_loss": 3.767075985351636, + "objective/train/tokens_used": 800338400, + "objective/train/unlikelihood_loss": 0.007990993559360504, + "theoretical_loss": 3.767075985351636, + "tokens_seen": 779878400 + }, + { + "epoch": 0.24, + "learning_rate": 0.000617147442326981, + "loss": 1.5599, + "theoretical_loss": 3.767075985351636, + "tokens_seen": 779878400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006171313941825478, + "loss": 1.622, + "theoretical_loss": 3.767044699722927, + "tokens_seen": 779943936 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006171153460381144, + "loss": 1.5446, + "theoretical_loss": 3.767013417458931, + "tokens_seen": 780009472 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006170992978936812, + "loss": 1.5136, + "theoretical_loss": 3.766982138559004, + "tokens_seen": 780075008 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006170832497492478, + "loss": 1.4532, + "theoretical_loss": 3.7669508630225006, + "tokens_seen": 780140544 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006170672016048146, + "loss": 1.4814, + "theoretical_loss": 3.7669195908487776, + "tokens_seen": 780206080 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006170511534603812, + "loss": 1.4035, + "theoretical_loss": 3.766888322037191, + "tokens_seen": 780271616 + }, + { + "epoch": 0.24, + "learning_rate": 0.000617035105315948, + "loss": 1.5251, + "theoretical_loss": 3.7668570565870967, + "tokens_seen": 780337152 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006170190571715146, + "loss": 1.6388, + "theoretical_loss": 3.7668257944978505, + "tokens_seen": 780402688 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006170030090270812, + "loss": 1.5157, + "theoretical_loss": 3.7667945357688097, + "tokens_seen": 780468224 + }, + { + "epoch": 0.24, + "learning_rate": 0.000616986960882648, + "loss": 1.5016, + "theoretical_loss": 3.7667632803993314, + "tokens_seen": 780533760 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006169709127382146, + "loss": 1.4889, + "theoretical_loss": 3.7667320283887715, + "tokens_seen": 780599296 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006169548645937814, + "loss": 1.5736, + "theoretical_loss": 3.7667007797364875, + "tokens_seen": 780664832 + }, + { + "epoch": 0.24, + "learning_rate": 0.000616938816449348, + "loss": 1.4397, + "theoretical_loss": 3.766669534441837, + "tokens_seen": 780730368 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006169227683049148, + "loss": 1.4797, + "theoretical_loss": 3.766638292504177, + "tokens_seen": 780795904 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006169067201604815, + "loss": 1.5545, + "theoretical_loss": 3.766607053922865, + "tokens_seen": 780861440 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006168906720160482, + "loss": 1.4453, + "theoretical_loss": 3.7665758186972584, + "tokens_seen": 780926976 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006168746238716149, + "loss": 1.4794, + "theoretical_loss": 3.7665445868267162, + "tokens_seen": 780992512 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006168585757271815, + "loss": 1.4371, + "theoretical_loss": 3.766513358310596, + "tokens_seen": 781058048 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006168425275827483, + "loss": 1.5423, + "theoretical_loss": 3.7664821331482568, + "tokens_seen": 781123584 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006168264794383149, + "loss": 1.4337, + "theoretical_loss": 3.7664509113390556, + "tokens_seen": 781189120 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006168104312938817, + "loss": 1.4213, + "theoretical_loss": 3.7664196928823523, + "tokens_seen": 781254656 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006167943831494483, + "loss": 1.4523, + "theoretical_loss": 3.7663884777775047, + "tokens_seen": 781320192 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006167783350050151, + "loss": 1.5327, + "theoretical_loss": 3.7663572660238733, + "tokens_seen": 781385728 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006167622868605817, + "loss": 1.4708, + "theoretical_loss": 3.766326057620816, + "tokens_seen": 781451264 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006167462387161485, + "loss": 1.4851, + "theoretical_loss": 3.7662948525676927, + "tokens_seen": 781516800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006167301905717151, + "loss": 1.4061, + "theoretical_loss": 3.7662636508638627, + "tokens_seen": 781582336 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006167141424272819, + "loss": 1.4645, + "theoretical_loss": 3.7662324525086865, + "tokens_seen": 781647872 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006166980942828485, + "loss": 1.648, + "theoretical_loss": 3.766201257501523, + "tokens_seen": 781713408 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006166820461384153, + "loss": 1.4691, + "theoretical_loss": 3.7661700658417328, + "tokens_seen": 781778944 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006166659979939819, + "loss": 1.3685, + "theoretical_loss": 3.7661388775286753, + "tokens_seen": 781844480 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006166499498495487, + "loss": 1.5134, + "theoretical_loss": 3.7661076925617123, + "tokens_seen": 781910016 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006166339017051153, + "loss": 1.4049, + "theoretical_loss": 3.7660765109402035, + "tokens_seen": 781975552 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006166178535606821, + "loss": 1.4924, + "theoretical_loss": 3.76604533266351, + "tokens_seen": 782041088 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006166018054162487, + "loss": 1.4588, + "theoretical_loss": 3.7660141577309925, + "tokens_seen": 782106624 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006165857572718155, + "loss": 1.494, + "theoretical_loss": 3.7659829861420127, + "tokens_seen": 782172160 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006165697091273821, + "loss": 1.5836, + "theoretical_loss": 3.765951817895931, + "tokens_seen": 782237696 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006165536609829489, + "loss": 1.4597, + "theoretical_loss": 3.765920652992109, + "tokens_seen": 782303232 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006165376128385155, + "loss": 1.477, + "theoretical_loss": 3.7658894914299097, + "tokens_seen": 782368768 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006165215646940823, + "loss": 1.3795, + "theoretical_loss": 3.7658583332086932, + "tokens_seen": 782434304 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006165055165496489, + "loss": 1.4027, + "theoretical_loss": 3.7658271783278225, + "tokens_seen": 782499840 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006164894684052157, + "loss": 1.5289, + "theoretical_loss": 3.765796026786659, + "tokens_seen": 782565376 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006164734202607823, + "loss": 1.4122, + "theoretical_loss": 3.7657648785845663, + "tokens_seen": 782630912 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006164573721163491, + "loss": 1.476, + "theoretical_loss": 3.765733733720906, + "tokens_seen": 782696448 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006164413239719157, + "loss": 1.4622, + "theoretical_loss": 3.7657025921950407, + "tokens_seen": 782761984 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006164252758274825, + "loss": 1.3371, + "theoretical_loss": 3.765671454006334, + "tokens_seen": 782827520 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006164092276830491, + "loss": 1.5358, + "theoretical_loss": 3.7656403191541483, + "tokens_seen": 782893056 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006163931795386159, + "loss": 1.5555, + "theoretical_loss": 3.765609187637847, + "tokens_seen": 782958592 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006163771313941825, + "loss": 1.3808, + "theoretical_loss": 3.765578059456794, + "tokens_seen": 783024128 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006163610832497493, + "loss": 1.4426, + "theoretical_loss": 3.765546934610352, + "tokens_seen": 783089664 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.9923056364059448, + "objective/train/avg_token_score": 0.007694398984313011, + "objective/train/docs_used": 289672, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.596255898475647, + "objective/train/original_loss": 1.608188509941101, + "objective/train/theoretical_loss": 3.7655158130978856, + "objective/train/tokens_used": 803615200, + "objective/train/unlikelihood_loss": 0.01939418539404869, + "theoretical_loss": 3.7655158130978856, + "tokens_seen": 783155200 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006163450351053159, + "loss": 1.493, + "theoretical_loss": 3.7655158130978856, + "tokens_seen": 783155200 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006163289869608827, + "loss": 1.5164, + "theoretical_loss": 3.765484694918758, + "tokens_seen": 783220736 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006163129388164493, + "loss": 1.461, + "theoretical_loss": 3.7654535800723337, + "tokens_seen": 783286272 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006162968906720161, + "loss": 1.3837, + "theoretical_loss": 3.7654224685579765, + "tokens_seen": 783351808 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006162808425275827, + "loss": 1.5159, + "theoretical_loss": 3.765391360375052, + "tokens_seen": 783417344 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006162647943831495, + "loss": 1.5136, + "theoretical_loss": 3.765360255522924, + "tokens_seen": 783482880 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006162487462387161, + "loss": 1.4497, + "theoretical_loss": 3.7653291540009572, + "tokens_seen": 783548416 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006162326980942829, + "loss": 1.5447, + "theoretical_loss": 3.7652980558085165, + "tokens_seen": 783613952 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006162166499498495, + "loss": 1.5864, + "theoretical_loss": 3.7652669609449676, + "tokens_seen": 783679488 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006162006018054163, + "loss": 1.6083, + "theoretical_loss": 3.7652358694096755, + "tokens_seen": 783745024 + }, + { + "epoch": 0.24, + "learning_rate": 0.000616184553660983, + "loss": 1.4444, + "theoretical_loss": 3.765204781202006, + "tokens_seen": 783810560 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006161685055165497, + "loss": 1.4267, + "theoretical_loss": 3.765173696321324, + "tokens_seen": 783876096 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006161524573721164, + "loss": 1.4268, + "theoretical_loss": 3.7651426147669964, + "tokens_seen": 783941632 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006161364092276831, + "loss": 1.4759, + "theoretical_loss": 3.765111536538388, + "tokens_seen": 784007168 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006161203610832498, + "loss": 1.428, + "theoretical_loss": 3.7650804616348665, + "tokens_seen": 784072704 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006161043129388165, + "loss": 1.3206, + "theoretical_loss": 3.765049390055797, + "tokens_seen": 784138240 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006160882647943832, + "loss": 1.4865, + "theoretical_loss": 3.765018321800546, + "tokens_seen": 784203776 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006160722166499499, + "loss": 1.5301, + "theoretical_loss": 3.7649872568684812, + "tokens_seen": 784269312 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006160561685055166, + "loss": 1.5102, + "theoretical_loss": 3.764956195258969, + "tokens_seen": 784334848 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006160401203610833, + "loss": 1.3382, + "theoretical_loss": 3.7649251369713763, + "tokens_seen": 784400384 + }, + { + "epoch": 0.24, + "learning_rate": 0.00061602407221665, + "loss": 1.3947, + "theoretical_loss": 3.7648940820050707, + "tokens_seen": 784465920 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006160080240722167, + "loss": 1.3571, + "theoretical_loss": 3.764863030359419, + "tokens_seen": 784531456 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006159919759277834, + "loss": 1.3528, + "theoretical_loss": 3.7648319820337894, + "tokens_seen": 784596992 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006159759277833501, + "loss": 1.4707, + "theoretical_loss": 3.7648009370275495, + "tokens_seen": 784662528 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006159598796389168, + "loss": 1.4327, + "theoretical_loss": 3.7647698953400672, + "tokens_seen": 784728064 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006159438314944835, + "loss": 1.4721, + "theoretical_loss": 3.76473885697071, + "tokens_seen": 784793600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006159277833500502, + "loss": 1.4272, + "theoretical_loss": 3.7647078219188472, + "tokens_seen": 784859136 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006159117352056169, + "loss": 1.5272, + "theoretical_loss": 3.7646767901838465, + "tokens_seen": 784924672 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006158956870611836, + "loss": 1.477, + "theoretical_loss": 3.764645761765077, + "tokens_seen": 784990208 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006158796389167503, + "loss": 1.4179, + "theoretical_loss": 3.7646147366619074, + "tokens_seen": 785055744 + }, + { + "epoch": 0.24, + "learning_rate": 0.000615863590772317, + "loss": 1.5431, + "theoretical_loss": 3.764583714873706, + "tokens_seen": 785121280 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006158475426278837, + "loss": 1.4132, + "theoretical_loss": 3.764552696399843, + "tokens_seen": 785186816 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006158314944834504, + "loss": 1.4163, + "theoretical_loss": 3.764521681239687, + "tokens_seen": 785252352 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006158154463390171, + "loss": 1.4521, + "theoretical_loss": 3.7644906693926075, + "tokens_seen": 785317888 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006157993981945839, + "loss": 1.6258, + "theoretical_loss": 3.7644596608579746, + "tokens_seen": 785383424 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006157833500501505, + "loss": 1.4864, + "theoretical_loss": 3.764428655635158, + "tokens_seen": 785448960 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006157673019057173, + "loss": 1.4072, + "theoretical_loss": 3.764397653723527, + "tokens_seen": 785514496 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006157512537612839, + "loss": 1.4443, + "theoretical_loss": 3.7643666551224526, + "tokens_seen": 785580032 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006157352056168507, + "loss": 1.4364, + "theoretical_loss": 3.764335659831305, + "tokens_seen": 785645568 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006157191574724173, + "loss": 1.3977, + "theoretical_loss": 3.7643046678494545, + "tokens_seen": 785711104 + }, + { + "epoch": 0.24, + "learning_rate": 0.000615703109327984, + "loss": 1.4149, + "theoretical_loss": 3.764273679176272, + "tokens_seen": 785776640 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006156870611835507, + "loss": 1.5849, + "theoretical_loss": 3.764242693811128, + "tokens_seen": 785842176 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006156710130391175, + "loss": 1.487, + "theoretical_loss": 3.764211711753394, + "tokens_seen": 785907712 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006156549648946841, + "loss": 1.3528, + "theoretical_loss": 3.7641807330024406, + "tokens_seen": 785973248 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006156389167502509, + "loss": 1.3777, + "theoretical_loss": 3.7641497575576404, + "tokens_seen": 786038784 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006156228686058175, + "loss": 1.4428, + "theoretical_loss": 3.764118785418363, + "tokens_seen": 786104320 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006156068204613843, + "loss": 1.5594, + "theoretical_loss": 3.7640878165839817, + "tokens_seen": 786169856 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006155907723169509, + "loss": 1.4455, + "theoretical_loss": 3.7640568510538683, + "tokens_seen": 786235392 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006155747241725176, + "loss": 1.4171, + "theoretical_loss": 3.764025888827394, + "tokens_seen": 786300928 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006155586760280843, + "loss": 1.4374, + "theoretical_loss": 3.763994929903932, + "tokens_seen": 786366464 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.9925498962402344, + "objective/train/avg_token_score": 0.0074501121416687965, + "objective/train/docs_used": 290810, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.3897367715835571, + "objective/train/original_loss": 1.3997472524642944, + "objective/train/theoretical_loss": 3.763963974282854, + "objective/train/tokens_used": 806892000, + "objective/train/unlikelihood_loss": 0.020250925794243813, + "theoretical_loss": 3.763963974282854, + "tokens_seen": 786432000 + }, + { + "epoch": 0.24, + "learning_rate": 0.000615542627883651, + "loss": 1.4517, + "theoretical_loss": 3.763963974282854, + "tokens_seen": 786432000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006155265797392177, + "loss": 1.4213, + "theoretical_loss": 3.763933021963533, + "tokens_seen": 786497536 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006155105315947844, + "loss": 1.4526, + "theoretical_loss": 3.763902072945341, + "tokens_seen": 786563072 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006154944834503511, + "loss": 1.4373, + "theoretical_loss": 3.7638711272276524, + "tokens_seen": 786628608 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006154784353059178, + "loss": 1.4652, + "theoretical_loss": 3.7638401848098386, + "tokens_seen": 786694144 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006154623871614844, + "loss": 1.5761, + "theoretical_loss": 3.763809245691274, + "tokens_seen": 786759680 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006154463390170512, + "loss": 1.4132, + "theoretical_loss": 3.7637783098713316, + "tokens_seen": 786825216 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006154302908726178, + "loss": 1.4677, + "theoretical_loss": 3.7637473773493855, + "tokens_seen": 786890752 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006154142427281846, + "loss": 1.4385, + "theoretical_loss": 3.7637164481248084, + "tokens_seen": 786956288 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006153981945837512, + "loss": 1.3981, + "theoretical_loss": 3.763685522196975, + "tokens_seen": 787021824 + }, + { + "epoch": 0.24, + "learning_rate": 0.000615382146439318, + "loss": 1.5127, + "theoretical_loss": 3.7636545995652604, + "tokens_seen": 787087360 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006153660982948846, + "loss": 1.5686, + "theoretical_loss": 3.7636236802290366, + "tokens_seen": 787152896 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006153500501504514, + "loss": 1.5137, + "theoretical_loss": 3.7635927641876803, + "tokens_seen": 787218432 + }, + { + "epoch": 0.24, + "learning_rate": 0.000615334002006018, + "loss": 1.504, + "theoretical_loss": 3.7635618514405644, + "tokens_seen": 787283968 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006153179538615848, + "loss": 1.5387, + "theoretical_loss": 3.7635309419870646, + "tokens_seen": 787349504 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006153019057171514, + "loss": 1.3939, + "theoretical_loss": 3.763500035826556, + "tokens_seen": 787415040 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006152858575727182, + "loss": 1.3978, + "theoretical_loss": 3.7634691329584133, + "tokens_seen": 787480576 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006152698094282848, + "loss": 1.4508, + "theoretical_loss": 3.763438233382012, + "tokens_seen": 787546112 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006152537612838516, + "loss": 1.5389, + "theoretical_loss": 3.7634073370967274, + "tokens_seen": 787611648 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006152377131394182, + "loss": 1.3977, + "theoretical_loss": 3.7633764441019357, + "tokens_seen": 787677184 + }, + { + "epoch": 0.24, + "learning_rate": 0.000615221664994985, + "loss": 1.3476, + "theoretical_loss": 3.763345554397012, + "tokens_seen": 787742720 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006152056168505516, + "loss": 1.4684, + "theoretical_loss": 3.763314667981333, + "tokens_seen": 787808256 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006151895687061184, + "loss": 1.5745, + "theoretical_loss": 3.7632837848542744, + "tokens_seen": 787873792 + }, + { + "epoch": 0.24, + "learning_rate": 0.000615173520561685, + "loss": 1.2962, + "theoretical_loss": 3.7632529050152126, + "tokens_seen": 787939328 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006151574724172518, + "loss": 1.4452, + "theoretical_loss": 3.7632220284635247, + "tokens_seen": 788004864 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006151414242728184, + "loss": 1.4082, + "theoretical_loss": 3.763191155198586, + "tokens_seen": 788070400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006151253761283852, + "loss": 1.4588, + "theoretical_loss": 3.7631602852197745, + "tokens_seen": 788135936 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006151093279839518, + "loss": 1.5218, + "theoretical_loss": 3.763129418526467, + "tokens_seen": 788201472 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006150932798395186, + "loss": 1.379, + "theoretical_loss": 3.7630985551180407, + "tokens_seen": 788267008 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006150772316950852, + "loss": 1.4062, + "theoretical_loss": 3.7630676949938726, + "tokens_seen": 788332544 + }, + { + "epoch": 0.24, + "learning_rate": 0.000615061183550652, + "loss": 1.4135, + "theoretical_loss": 3.763036838153341, + "tokens_seen": 788398080 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006150451354062186, + "loss": 1.5291, + "theoretical_loss": 3.763005984595823, + "tokens_seen": 788463616 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006150290872617854, + "loss": 1.4973, + "theoretical_loss": 3.762975134320696, + "tokens_seen": 788529152 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006150130391173521, + "loss": 1.4783, + "theoretical_loss": 3.762944287327339, + "tokens_seen": 788594688 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006149969909729188, + "loss": 1.3952, + "theoretical_loss": 3.7629134436151297, + "tokens_seen": 788660224 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006149809428284855, + "loss": 1.5264, + "theoretical_loss": 3.7628826031834466, + "tokens_seen": 788725760 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006149648946840522, + "loss": 1.5071, + "theoretical_loss": 3.7628517660316687, + "tokens_seen": 788791296 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006149488465396189, + "loss": 1.504, + "theoretical_loss": 3.7628209321591735, + "tokens_seen": 788856832 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006149327983951856, + "loss": 1.5343, + "theoretical_loss": 3.762790101565341, + "tokens_seen": 788922368 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006149167502507523, + "loss": 1.4202, + "theoretical_loss": 3.76275927424955, + "tokens_seen": 788987904 + }, + { + "epoch": 0.24, + "learning_rate": 0.000614900702106319, + "loss": 1.4147, + "theoretical_loss": 3.7627284502111795, + "tokens_seen": 789053440 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006148846539618857, + "loss": 1.5341, + "theoretical_loss": 3.762697629449609, + "tokens_seen": 789118976 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006148686058174524, + "loss": 1.4308, + "theoretical_loss": 3.762666811964218, + "tokens_seen": 789184512 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006148525576730191, + "loss": 1.5032, + "theoretical_loss": 3.762635997754386, + "tokens_seen": 789250048 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006148365095285858, + "loss": 1.4029, + "theoretical_loss": 3.762605186819494, + "tokens_seen": 789315584 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006148204613841525, + "loss": 1.5128, + "theoretical_loss": 3.76257437915892, + "tokens_seen": 789381120 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006148044132397192, + "loss": 1.3846, + "theoretical_loss": 3.762543574772047, + "tokens_seen": 789446656 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006147883650952859, + "loss": 1.3952, + "theoretical_loss": 3.7625127736582527, + "tokens_seen": 789512192 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006147723169508526, + "loss": 1.4244, + "theoretical_loss": 3.7624819758169195, + "tokens_seen": 789577728 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006147562688064193, + "loss": 1.4461, + "theoretical_loss": 3.7624511812474273, + "tokens_seen": 789643264 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.993568480014801, + "objective/train/avg_token_score": 0.006431538611650467, + "objective/train/docs_used": 292000, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6527864933013916, + "objective/train/original_loss": 1.6644139289855957, + "objective/train/theoretical_loss": 3.762420389949157, + "objective/train/tokens_used": 810168800, + "objective/train/unlikelihood_loss": 0.01496758684515953, + "theoretical_loss": 3.762420389949157, + "tokens_seen": 789708800 + }, + { + "epoch": 0.24, + "learning_rate": 0.000614740220661986, + "loss": 1.5449, + "theoretical_loss": 3.762420389949157, + "tokens_seen": 789708800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006147241725175527, + "loss": 1.4985, + "theoretical_loss": 3.76238960192149, + "tokens_seen": 789774336 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006147081243731194, + "loss": 1.4424, + "theoretical_loss": 3.762358817163808, + "tokens_seen": 789839872 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006146920762286861, + "loss": 1.5041, + "theoretical_loss": 3.762328035675491, + "tokens_seen": 789905408 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006146760280842528, + "loss": 1.4438, + "theoretical_loss": 3.7622972574559226, + "tokens_seen": 789970944 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006146599799398195, + "loss": 1.4757, + "theoretical_loss": 3.762266482504483, + "tokens_seen": 790036480 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006146439317953862, + "loss": 1.4916, + "theoretical_loss": 3.762235710820554, + "tokens_seen": 790102016 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006146278836509529, + "loss": 1.4919, + "theoretical_loss": 3.7622049424035184, + "tokens_seen": 790167552 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006146118355065196, + "loss": 1.5464, + "theoretical_loss": 3.762174177252758, + "tokens_seen": 790233088 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006145957873620863, + "loss": 1.4558, + "theoretical_loss": 3.7621434153676563, + "tokens_seen": 790298624 + }, + { + "epoch": 0.24, + "learning_rate": 0.000614579739217653, + "loss": 1.6009, + "theoretical_loss": 3.7621126567475947, + "tokens_seen": 790364160 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006145636910732197, + "loss": 1.4769, + "theoretical_loss": 3.762081901391956, + "tokens_seen": 790429696 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006145476429287864, + "loss": 1.4329, + "theoretical_loss": 3.7620511493001243, + "tokens_seen": 790495232 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006145315947843531, + "loss": 1.4833, + "theoretical_loss": 3.7620204004714815, + "tokens_seen": 790560768 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006145155466399198, + "loss": 1.4509, + "theoretical_loss": 3.761989654905411, + "tokens_seen": 790626304 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006144994984954865, + "loss": 1.4842, + "theoretical_loss": 3.7619589126012967, + "tokens_seen": 790691840 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006144834503510532, + "loss": 1.62, + "theoretical_loss": 3.761928173558522, + "tokens_seen": 790757376 + }, + { + "epoch": 0.24, + "learning_rate": 0.00061446740220662, + "loss": 1.5056, + "theoretical_loss": 3.7618974377764705, + "tokens_seen": 790822912 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006144513540621866, + "loss": 1.4729, + "theoretical_loss": 3.7618667052545263, + "tokens_seen": 790888448 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006144353059177534, + "loss": 1.4763, + "theoretical_loss": 3.7618359759920734, + "tokens_seen": 790953984 + }, + { + "epoch": 0.24, + "learning_rate": 0.00061441925777332, + "loss": 1.5835, + "theoretical_loss": 3.7618052499884964, + "tokens_seen": 791019520 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006144032096288868, + "loss": 1.4357, + "theoretical_loss": 3.7617745272431797, + "tokens_seen": 791085056 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006143871614844534, + "loss": 1.471, + "theoretical_loss": 3.761743807755507, + "tokens_seen": 791150592 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006143711133400202, + "loss": 1.4751, + "theoretical_loss": 3.7617130915248644, + "tokens_seen": 791216128 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006143550651955868, + "loss": 1.5089, + "theoretical_loss": 3.7616823785506357, + "tokens_seen": 791281664 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006143390170511536, + "loss": 1.5797, + "theoretical_loss": 3.761651668832207, + "tokens_seen": 791347200 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006143229689067202, + "loss": 1.4566, + "theoretical_loss": 3.761620962368963, + "tokens_seen": 791412736 + }, + { + "epoch": 0.24, + "learning_rate": 0.000614306920762287, + "loss": 1.4448, + "theoretical_loss": 3.7615902591602888, + "tokens_seen": 791478272 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006142908726178536, + "loss": 1.4452, + "theoretical_loss": 3.761559559205571, + "tokens_seen": 791543808 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006142748244734204, + "loss": 1.4737, + "theoretical_loss": 3.7615288625041945, + "tokens_seen": 791609344 + }, + { + "epoch": 0.24, + "learning_rate": 0.000614258776328987, + "loss": 1.3794, + "theoretical_loss": 3.7614981690555456, + "tokens_seen": 791674880 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006142427281845538, + "loss": 1.4681, + "theoretical_loss": 3.7614674788590103, + "tokens_seen": 791740416 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006142266800401204, + "loss": 1.5615, + "theoretical_loss": 3.761436791913975, + "tokens_seen": 791805952 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006142106318956872, + "loss": 1.467, + "theoretical_loss": 3.761406108219826, + "tokens_seen": 791871488 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006141945837512538, + "loss": 1.3781, + "theoretical_loss": 3.76137542777595, + "tokens_seen": 791937024 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006141785356068205, + "loss": 1.4927, + "theoretical_loss": 3.761344750581734, + "tokens_seen": 792002560 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006141624874623872, + "loss": 1.3792, + "theoretical_loss": 3.7613140766365643, + "tokens_seen": 792068096 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006141464393179539, + "loss": 1.4195, + "theoretical_loss": 3.7612834059398286, + "tokens_seen": 792133632 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006141303911735206, + "loss": 1.5424, + "theoretical_loss": 3.7612527384909136, + "tokens_seen": 792199168 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006141143430290873, + "loss": 1.388, + "theoretical_loss": 3.7612220742892077, + "tokens_seen": 792264704 + }, + { + "epoch": 0.24, + "learning_rate": 0.000614098294884654, + "loss": 1.4781, + "theoretical_loss": 3.761191413334098, + "tokens_seen": 792330240 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006140822467402207, + "loss": 1.476, + "theoretical_loss": 3.7611607556249718, + "tokens_seen": 792395776 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006140661985957874, + "loss": 1.3959, + "theoretical_loss": 3.761130101161217, + "tokens_seen": 792461312 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006140501504513541, + "loss": 1.4877, + "theoretical_loss": 3.761099449942223, + "tokens_seen": 792526848 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006140341023069207, + "loss": 1.404, + "theoretical_loss": 3.7610688019673764, + "tokens_seen": 792592384 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006140180541624875, + "loss": 1.469, + "theoretical_loss": 3.761038157236067, + "tokens_seen": 792657920 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006140020060180541, + "loss": 1.3983, + "theoretical_loss": 3.7610075157476826, + "tokens_seen": 792723456 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006139859578736209, + "loss": 1.4853, + "theoretical_loss": 3.760976877501612, + "tokens_seen": 792788992 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006139699097291875, + "loss": 1.4992, + "theoretical_loss": 3.7609462424972446, + "tokens_seen": 792854528 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006139538615847543, + "loss": 1.4566, + "theoretical_loss": 3.760915610733969, + "tokens_seen": 792920064 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.9903817176818848, + "objective/train/avg_token_score": 0.009618268348276615, + "objective/train/docs_used": 293128, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.3985661268234253, + "objective/train/original_loss": 1.412149429321289, + "objective/train/theoretical_loss": 3.7608849822111754, + "objective/train/tokens_used": 813445600, + "objective/train/unlikelihood_loss": 0.024049030616879463, + "theoretical_loss": 3.7608849822111754, + "tokens_seen": 792985600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006139378134403209, + "loss": 1.5164, + "theoretical_loss": 3.7608849822111754, + "tokens_seen": 792985600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006139217652958877, + "loss": 1.4795, + "theoretical_loss": 3.7608543569282515, + "tokens_seen": 793051136 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006139057171514543, + "loss": 1.4402, + "theoretical_loss": 3.7608237348845885, + "tokens_seen": 793116672 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006138896690070211, + "loss": 1.5803, + "theoretical_loss": 3.760793116079576, + "tokens_seen": 793182208 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006138736208625877, + "loss": 1.4522, + "theoretical_loss": 3.7607625005126026, + "tokens_seen": 793247744 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006138575727181545, + "loss": 1.4999, + "theoretical_loss": 3.7607318881830594, + "tokens_seen": 793313280 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006138415245737211, + "loss": 1.4879, + "theoretical_loss": 3.7607012790903367, + "tokens_seen": 793378816 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006138254764292879, + "loss": 1.4334, + "theoretical_loss": 3.7606706732338253, + "tokens_seen": 793444352 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006138094282848546, + "loss": 1.5848, + "theoretical_loss": 3.7606400706129146, + "tokens_seen": 793509888 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006137933801404213, + "loss": 1.521, + "theoretical_loss": 3.760609471226996, + "tokens_seen": 793575424 + }, + { + "epoch": 0.24, + "learning_rate": 0.000613777331995988, + "loss": 1.5767, + "theoretical_loss": 3.7605788750754607, + "tokens_seen": 793640960 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006137612838515547, + "loss": 1.5027, + "theoretical_loss": 3.7605482821577, + "tokens_seen": 793706496 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006137452357071214, + "loss": 1.4637, + "theoretical_loss": 3.760517692473104, + "tokens_seen": 793772032 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006137291875626881, + "loss": 1.5122, + "theoretical_loss": 3.7604871060210643, + "tokens_seen": 793837568 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006137131394182548, + "loss": 1.5524, + "theoretical_loss": 3.7604565228009736, + "tokens_seen": 793903104 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006136970912738215, + "loss": 1.4409, + "theoretical_loss": 3.760425942812223, + "tokens_seen": 793968640 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006136810431293882, + "loss": 1.4041, + "theoretical_loss": 3.760395366054204, + "tokens_seen": 794034176 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006136649949849549, + "loss": 1.5188, + "theoretical_loss": 3.7603647925263095, + "tokens_seen": 794099712 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006136489468405216, + "loss": 1.3228, + "theoretical_loss": 3.7603342222279315, + "tokens_seen": 794165248 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006136328986960883, + "loss": 1.496, + "theoretical_loss": 3.760303655158461, + "tokens_seen": 794230784 + }, + { + "epoch": 0.24, + "learning_rate": 0.000613616850551655, + "loss": 1.4207, + "theoretical_loss": 3.760273091317293, + "tokens_seen": 794296320 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006136008024072217, + "loss": 1.5137, + "theoretical_loss": 3.7602425307038185, + "tokens_seen": 794361856 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006135847542627884, + "loss": 1.4251, + "theoretical_loss": 3.760211973317431, + "tokens_seen": 794427392 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006135687061183551, + "loss": 1.6086, + "theoretical_loss": 3.760181419157523, + "tokens_seen": 794492928 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006135526579739218, + "loss": 1.3947, + "theoretical_loss": 3.7601508682234885, + "tokens_seen": 794558464 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006135366098294885, + "loss": 1.5438, + "theoretical_loss": 3.7601203205147202, + "tokens_seen": 794624000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006135205616850552, + "loss": 1.4476, + "theoretical_loss": 3.760089776030613, + "tokens_seen": 794689536 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006135045135406219, + "loss": 1.4332, + "theoretical_loss": 3.7600592347705586, + "tokens_seen": 794755072 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006134884653961886, + "loss": 1.4901, + "theoretical_loss": 3.7600286967339525, + "tokens_seen": 794820608 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006134724172517553, + "loss": 1.4911, + "theoretical_loss": 3.759998161920187, + "tokens_seen": 794886144 + }, + { + "epoch": 0.24, + "learning_rate": 0.000613456369107322, + "loss": 1.487, + "theoretical_loss": 3.7599676303286587, + "tokens_seen": 794951680 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006134403209628887, + "loss": 1.5075, + "theoretical_loss": 3.75993710195876, + "tokens_seen": 795017216 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006134242728184554, + "loss": 1.4588, + "theoretical_loss": 3.759906576809886, + "tokens_seen": 795082752 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006134082246740221, + "loss": 1.436, + "theoretical_loss": 3.759876054881432, + "tokens_seen": 795148288 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006133921765295888, + "loss": 1.4557, + "theoretical_loss": 3.759845536172792, + "tokens_seen": 795213824 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006133761283851555, + "loss": 1.6178, + "theoretical_loss": 3.7598150206833614, + "tokens_seen": 795279360 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006133600802407222, + "loss": 1.5026, + "theoretical_loss": 3.759784508412536, + "tokens_seen": 795344896 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006133440320962889, + "loss": 1.4943, + "theoretical_loss": 3.7597539993597096, + "tokens_seen": 795410432 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006133279839518556, + "loss": 1.447, + "theoretical_loss": 3.7597234935242794, + "tokens_seen": 795475968 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006133119358074223, + "loss": 1.4612, + "theoretical_loss": 3.7596929909056396, + "tokens_seen": 795541504 + }, + { + "epoch": 0.24, + "learning_rate": 0.000613295887662989, + "loss": 1.4033, + "theoretical_loss": 3.759662491503187, + "tokens_seen": 795607040 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006132798395185557, + "loss": 1.5019, + "theoretical_loss": 3.759631995316318, + "tokens_seen": 795672576 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006132637913741224, + "loss": 1.5086, + "theoretical_loss": 3.7596015023444274, + "tokens_seen": 795738112 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006132477432296891, + "loss": 1.449, + "theoretical_loss": 3.759571012586913, + "tokens_seen": 795803648 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006132316950852558, + "loss": 1.3818, + "theoretical_loss": 3.75954052604317, + "tokens_seen": 795869184 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006132156469408225, + "loss": 1.4548, + "theoretical_loss": 3.759510042712596, + "tokens_seen": 795934720 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006131995987963892, + "loss": 1.6435, + "theoretical_loss": 3.7594795625945867, + "tokens_seen": 796000256 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006131835506519559, + "loss": 1.4995, + "theoretical_loss": 3.75944908568854, + "tokens_seen": 796065792 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006131675025075226, + "loss": 1.4367, + "theoretical_loss": 3.759418611993853, + "tokens_seen": 796131328 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006131514543630893, + "loss": 1.5152, + "theoretical_loss": 3.7593881415099233, + "tokens_seen": 796196864 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.9926673173904419, + "objective/train/avg_token_score": 0.007332693785429001, + "objective/train/docs_used": 294439, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5598188638687134, + "objective/train/original_loss": 1.5719444751739502, + "objective/train/theoretical_loss": 3.7593576742361474, + "objective/train/tokens_used": 816722400, + "objective/train/unlikelihood_loss": 0.01674150861799717, + "theoretical_loss": 3.7593576742361474, + "tokens_seen": 796262400 + }, + { + "epoch": 0.24, + "learning_rate": 0.000613135406218656, + "loss": 1.5702, + "theoretical_loss": 3.7593576742361474, + "tokens_seen": 796262400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006131193580742227, + "loss": 1.4393, + "theoretical_loss": 3.759327210171924, + "tokens_seen": 796327936 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006131033099297894, + "loss": 1.5236, + "theoretical_loss": 3.7592967493166496, + "tokens_seen": 796393472 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006130872617853561, + "loss": 1.5487, + "theoretical_loss": 3.7592662916697237, + "tokens_seen": 796459008 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006130712136409228, + "loss": 1.5005, + "theoretical_loss": 3.759235837230543, + "tokens_seen": 796524544 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006130551654964895, + "loss": 1.4792, + "theoretical_loss": 3.7592053859985066, + "tokens_seen": 796590080 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006130391173520563, + "loss": 1.4753, + "theoretical_loss": 3.759174937973013, + "tokens_seen": 796655616 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006130230692076229, + "loss": 1.3558, + "theoretical_loss": 3.7591444931534603, + "tokens_seen": 796721152 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006130070210631897, + "loss": 1.3892, + "theoretical_loss": 3.759114051539248, + "tokens_seen": 796786688 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006129909729187563, + "loss": 1.4449, + "theoretical_loss": 3.759083613129774, + "tokens_seen": 796852224 + }, + { + "epoch": 0.24, + "learning_rate": 0.000612974924774323, + "loss": 1.4915, + "theoretical_loss": 3.7590531779244385, + "tokens_seen": 796917760 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006129588766298897, + "loss": 1.4302, + "theoretical_loss": 3.75902274592264, + "tokens_seen": 796983296 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006129428284854565, + "loss": 1.3476, + "theoretical_loss": 3.758992317123778, + "tokens_seen": 797048832 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006129267803410231, + "loss": 1.4258, + "theoretical_loss": 3.7589618915272522, + "tokens_seen": 797114368 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006129107321965899, + "loss": 1.3979, + "theoretical_loss": 3.7589314691324622, + "tokens_seen": 797179904 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006128946840521565, + "loss": 1.4452, + "theoretical_loss": 3.7589010499388085, + "tokens_seen": 797245440 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006128786359077233, + "loss": 1.4083, + "theoretical_loss": 3.7588706339456905, + "tokens_seen": 797310976 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006128625877632899, + "loss": 1.467, + "theoretical_loss": 3.7588402211525085, + "tokens_seen": 797376512 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006128465396188567, + "loss": 1.4482, + "theoretical_loss": 3.7588098115586632, + "tokens_seen": 797442048 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006128304914744233, + "loss": 1.5404, + "theoretical_loss": 3.758779405163555, + "tokens_seen": 797507584 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006128144433299901, + "loss": 1.5454, + "theoretical_loss": 3.7587490019665846, + "tokens_seen": 797573120 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006127983951855567, + "loss": 1.562, + "theoretical_loss": 3.7587186019671526, + "tokens_seen": 797638656 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006127823470411234, + "loss": 1.4815, + "theoretical_loss": 3.7586882051646606, + "tokens_seen": 797704192 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006127662988966901, + "loss": 1.5428, + "theoretical_loss": 3.7586578115585096, + "tokens_seen": 797769728 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006127502507522568, + "loss": 1.3936, + "theoretical_loss": 3.758627421148101, + "tokens_seen": 797835264 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006127342026078235, + "loss": 1.427, + "theoretical_loss": 3.758597033932836, + "tokens_seen": 797900800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006127181544633902, + "loss": 1.5222, + "theoretical_loss": 3.758566649912116, + "tokens_seen": 797966336 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006127021063189569, + "loss": 1.4369, + "theoretical_loss": 3.7585362690853445, + "tokens_seen": 798031872 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006126860581745236, + "loss": 1.6137, + "theoretical_loss": 3.758505891451921, + "tokens_seen": 798097408 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006126700100300903, + "loss": 1.4772, + "theoretical_loss": 3.7584755170112496, + "tokens_seen": 798162944 + }, + { + "epoch": 0.24, + "learning_rate": 0.000612653961885657, + "loss": 1.5304, + "theoretical_loss": 3.758445145762732, + "tokens_seen": 798228480 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006126379137412236, + "loss": 1.4795, + "theoretical_loss": 3.758414777705771, + "tokens_seen": 798294016 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006126218655967904, + "loss": 1.4328, + "theoretical_loss": 3.7583844128397685, + "tokens_seen": 798359552 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006126058174523571, + "loss": 1.5054, + "theoretical_loss": 3.758354051164128, + "tokens_seen": 798425088 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006125897693079238, + "loss": 1.4162, + "theoretical_loss": 3.7583236926782515, + "tokens_seen": 798490624 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006125737211634905, + "loss": 1.5307, + "theoretical_loss": 3.7582933373815433, + "tokens_seen": 798556160 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006125576730190572, + "loss": 1.4211, + "theoretical_loss": 3.758262985273406, + "tokens_seen": 798621696 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006125416248746239, + "loss": 1.3895, + "theoretical_loss": 3.7582326363532443, + "tokens_seen": 798687232 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006125255767301906, + "loss": 1.5345, + "theoretical_loss": 3.75820229062046, + "tokens_seen": 798752768 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006125095285857573, + "loss": 1.5391, + "theoretical_loss": 3.758171948074457, + "tokens_seen": 798818304 + }, + { + "epoch": 0.24, + "learning_rate": 0.000612493480441324, + "loss": 1.4364, + "theoretical_loss": 3.7581416087146406, + "tokens_seen": 798883840 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006124774322968907, + "loss": 1.4252, + "theoretical_loss": 3.7581112725404147, + "tokens_seen": 798949376 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006124613841524574, + "loss": 1.4561, + "theoretical_loss": 3.7580809395511823, + "tokens_seen": 799014912 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006124453360080241, + "loss": 1.5124, + "theoretical_loss": 3.7580506097463484, + "tokens_seen": 799080448 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006124292878635908, + "loss": 1.4362, + "theoretical_loss": 3.758020283125318, + "tokens_seen": 799145984 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006124132397191575, + "loss": 1.477, + "theoretical_loss": 3.757989959687496, + "tokens_seen": 799211520 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006123971915747242, + "loss": 1.5467, + "theoretical_loss": 3.7579596394322863, + "tokens_seen": 799277056 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006123811434302909, + "loss": 1.4684, + "theoretical_loss": 3.757929322359095, + "tokens_seen": 799342592 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006123650952858576, + "loss": 1.4986, + "theoretical_loss": 3.7578990084673256, + "tokens_seen": 799408128 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006123490471414243, + "loss": 1.5182, + "theoretical_loss": 3.7578686977563858, + "tokens_seen": 799473664 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.9962740540504456, + "objective/train/avg_token_score": 0.003725928720086813, + "objective/train/docs_used": 295724, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.3889269828796387, + "objective/train/original_loss": 1.3946465253829956, + "objective/train/theoretical_loss": 3.75783839022568, + "objective/train/tokens_used": 819999200, + "objective/train/unlikelihood_loss": 0.009429533034563065, + "theoretical_loss": 3.75783839022568, + "tokens_seen": 799539200 + }, + { + "epoch": 0.24, + "learning_rate": 0.000612332998996991, + "loss": 1.4824, + "theoretical_loss": 3.75783839022568, + "tokens_seen": 799539200 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006123169508525577, + "loss": 1.5251, + "theoretical_loss": 3.757808085874613, + "tokens_seen": 799604736 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006123009027081244, + "loss": 1.4595, + "theoretical_loss": 3.7577777847025926, + "tokens_seen": 799670272 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006122848545636911, + "loss": 1.451, + "theoretical_loss": 3.757747486709023, + "tokens_seen": 799735808 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006122688064192578, + "loss": 1.386, + "theoretical_loss": 3.7577171918933114, + "tokens_seen": 799801344 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006122527582748245, + "loss": 1.4181, + "theoretical_loss": 3.757686900254864, + "tokens_seen": 799866880 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006122367101303912, + "loss": 1.5798, + "theoretical_loss": 3.757656611793087, + "tokens_seen": 799932416 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006122206619859579, + "loss": 1.4926, + "theoretical_loss": 3.7576263265073875, + "tokens_seen": 799997952 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006122046138415246, + "loss": 1.3862, + "theoretical_loss": 3.7575960443971717, + "tokens_seen": 800063488 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006121885656970913, + "loss": 1.4784, + "theoretical_loss": 3.7575657654618473, + "tokens_seen": 800129024 + }, + { + "epoch": 0.24, + "learning_rate": 0.000612172517552658, + "loss": 1.4203, + "theoretical_loss": 3.75753548970082, + "tokens_seen": 800194560 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006121564694082247, + "loss": 1.4834, + "theoretical_loss": 3.757505217113499, + "tokens_seen": 800260096 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006121404212637914, + "loss": 1.3888, + "theoretical_loss": 3.757474947699291, + "tokens_seen": 800325632 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006121243731193581, + "loss": 1.4696, + "theoretical_loss": 3.757444681457603, + "tokens_seen": 800391168 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006121083249749248, + "loss": 1.4484, + "theoretical_loss": 3.7574144183878433, + "tokens_seen": 800456704 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006120922768304915, + "loss": 1.4446, + "theoretical_loss": 3.75738415848942, + "tokens_seen": 800522240 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006120762286860582, + "loss": 1.4069, + "theoretical_loss": 3.7573539017617406, + "tokens_seen": 800587776 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006120601805416249, + "loss": 1.4865, + "theoretical_loss": 3.7573236482042134, + "tokens_seen": 800653312 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006120441323971916, + "loss": 1.4504, + "theoretical_loss": 3.7572933978162477, + "tokens_seen": 800718848 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006120280842527583, + "loss": 1.4686, + "theoretical_loss": 3.757263150597251, + "tokens_seen": 800784384 + }, + { + "epoch": 0.24, + "learning_rate": 0.000612012036108325, + "loss": 1.548, + "theoretical_loss": 3.7572329065466326, + "tokens_seen": 800849920 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006119959879638917, + "loss": 1.4084, + "theoretical_loss": 3.7572026656638013, + "tokens_seen": 800915456 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006119799398194584, + "loss": 1.3998, + "theoretical_loss": 3.757172427948166, + "tokens_seen": 800980992 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006119638916750251, + "loss": 1.4955, + "theoretical_loss": 3.7571421933991362, + "tokens_seen": 801046528 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006119478435305918, + "loss": 1.4654, + "theoretical_loss": 3.7571119620161206, + "tokens_seen": 801112064 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006119317953861585, + "loss": 1.4972, + "theoretical_loss": 3.7570817337985294, + "tokens_seen": 801177600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006119157472417252, + "loss": 1.4776, + "theoretical_loss": 3.757051508745772, + "tokens_seen": 801243136 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006118996990972919, + "loss": 1.5194, + "theoretical_loss": 3.757021286857258, + "tokens_seen": 801308672 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006118836509528586, + "loss": 1.6375, + "theoretical_loss": 3.7569910681323977, + "tokens_seen": 801374208 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006118676028084253, + "loss": 1.5071, + "theoretical_loss": 3.756960852570601, + "tokens_seen": 801439744 + }, + { + "epoch": 0.24, + "learning_rate": 0.000611851554663992, + "loss": 1.529, + "theoretical_loss": 3.7569306401712783, + "tokens_seen": 801505280 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006118355065195587, + "loss": 1.5676, + "theoretical_loss": 3.7569004309338405, + "tokens_seen": 801570816 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006118194583751254, + "loss": 1.549, + "theoretical_loss": 3.756870224857698, + "tokens_seen": 801636352 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006118034102306921, + "loss": 1.522, + "theoretical_loss": 3.756840021942261, + "tokens_seen": 801701888 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006117873620862588, + "loss": 1.4627, + "theoretical_loss": 3.756809822186941, + "tokens_seen": 801767424 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006117713139418255, + "loss": 1.5026, + "theoretical_loss": 3.7567796255911485, + "tokens_seen": 801832960 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006117552657973922, + "loss": 1.4803, + "theoretical_loss": 3.7567494321542956, + "tokens_seen": 801898496 + }, + { + "epoch": 0.24, + "learning_rate": 0.000611739217652959, + "loss": 1.4955, + "theoretical_loss": 3.756719241875793, + "tokens_seen": 801964032 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006117231695085256, + "loss": 1.5072, + "theoretical_loss": 3.7566890547550527, + "tokens_seen": 802029568 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006117071213640923, + "loss": 1.5361, + "theoretical_loss": 3.7566588707914867, + "tokens_seen": 802095104 + }, + { + "epoch": 0.24, + "learning_rate": 0.000611691073219659, + "loss": 1.4014, + "theoretical_loss": 3.756628689984506, + "tokens_seen": 802160640 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006116750250752258, + "loss": 1.3555, + "theoretical_loss": 3.756598512333523, + "tokens_seen": 802226176 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006116589769307924, + "loss": 1.4655, + "theoretical_loss": 3.75656833783795, + "tokens_seen": 802291712 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006116429287863592, + "loss": 1.4141, + "theoretical_loss": 3.7565381664972, + "tokens_seen": 802357248 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006116268806419258, + "loss": 1.4588, + "theoretical_loss": 3.7565079983106844, + "tokens_seen": 802422784 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006116108324974926, + "loss": 1.3606, + "theoretical_loss": 3.7564778332778164, + "tokens_seen": 802488320 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006115947843530592, + "loss": 1.4157, + "theoretical_loss": 3.756447671398009, + "tokens_seen": 802553856 + }, + { + "epoch": 0.24, + "learning_rate": 0.000611578736208626, + "loss": 1.4194, + "theoretical_loss": 3.7564175126706743, + "tokens_seen": 802619392 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006115626880641926, + "loss": 1.4411, + "theoretical_loss": 3.7563873570952264, + "tokens_seen": 802684928 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006115466399197594, + "loss": 1.3841, + "theoretical_loss": 3.7563572046710787, + "tokens_seen": 802750464 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.9896016120910645, + "objective/train/avg_token_score": 0.010398434475064278, + "objective/train/docs_used": 296852, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4287099838256836, + "objective/train/original_loss": 1.4432551860809326, + "objective/train/theoretical_loss": 3.7563270553976436, + "objective/train/tokens_used": 823276000, + "objective/train/unlikelihood_loss": 0.028722649440169334, + "theoretical_loss": 3.7563270553976436, + "tokens_seen": 802816000 + }, + { + "epoch": 0.24, + "learning_rate": 0.000611530591775326, + "loss": 1.377, + "theoretical_loss": 3.7563270553976436, + "tokens_seen": 802816000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006115145436308928, + "loss": 1.3787, + "theoretical_loss": 3.7562969092743357, + "tokens_seen": 802881536 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006114984954864594, + "loss": 1.4087, + "theoretical_loss": 3.7562667663005684, + "tokens_seen": 802947072 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006114824473420262, + "loss": 1.4915, + "theoretical_loss": 3.7562366264757556, + "tokens_seen": 803012608 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006114663991975928, + "loss": 1.4882, + "theoretical_loss": 3.756206489799311, + "tokens_seen": 803078144 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006114503510531596, + "loss": 1.3708, + "theoretical_loss": 3.7561763562706503, + "tokens_seen": 803143680 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006114343029087262, + "loss": 1.6112, + "theoretical_loss": 3.7561462258891862, + "tokens_seen": 803209216 + }, + { + "epoch": 0.24, + "learning_rate": 0.000611418254764293, + "loss": 1.4845, + "theoretical_loss": 3.756116098654334, + "tokens_seen": 803274752 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006114022066198596, + "loss": 1.5093, + "theoretical_loss": 3.7560859745655084, + "tokens_seen": 803340288 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006113861584754263, + "loss": 1.4746, + "theoretical_loss": 3.756055853622124, + "tokens_seen": 803405824 + }, + { + "epoch": 0.24, + "learning_rate": 0.000611370110330993, + "loss": 1.4674, + "theoretical_loss": 3.756025735823596, + "tokens_seen": 803471360 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006113540621865597, + "loss": 1.4117, + "theoretical_loss": 3.7559956211693404, + "tokens_seen": 803536896 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006113380140421264, + "loss": 1.5323, + "theoretical_loss": 3.7559655096587705, + "tokens_seen": 803602432 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006113219658976931, + "loss": 1.4579, + "theoretical_loss": 3.7559354012913038, + "tokens_seen": 803667968 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006113059177532598, + "loss": 1.4543, + "theoretical_loss": 3.755905296066355, + "tokens_seen": 803733504 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006112898696088265, + "loss": 1.4991, + "theoretical_loss": 3.75587519398334, + "tokens_seen": 803799040 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006112738214643932, + "loss": 1.4847, + "theoretical_loss": 3.755845095041675, + "tokens_seen": 803864576 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006112577733199599, + "loss": 1.5677, + "theoretical_loss": 3.755814999240776, + "tokens_seen": 803930112 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006112417251755266, + "loss": 1.5855, + "theoretical_loss": 3.7557849065800593, + "tokens_seen": 803995648 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006112256770310933, + "loss": 1.5564, + "theoretical_loss": 3.755754817058941, + "tokens_seen": 804061184 + }, + { + "epoch": 0.24, + "learning_rate": 0.00061120962888666, + "loss": 1.5658, + "theoretical_loss": 3.755724730676838, + "tokens_seen": 804126720 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006111935807422267, + "loss": 1.4321, + "theoretical_loss": 3.7556946474331667, + "tokens_seen": 804192256 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006111775325977934, + "loss": 1.4384, + "theoretical_loss": 3.7556645673273445, + "tokens_seen": 804257792 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006111614844533601, + "loss": 1.5383, + "theoretical_loss": 3.755634490358789, + "tokens_seen": 804323328 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006111454363089268, + "loss": 1.521, + "theoretical_loss": 3.7556044165269156, + "tokens_seen": 804388864 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006111293881644935, + "loss": 1.4849, + "theoretical_loss": 3.7555743458311435, + "tokens_seen": 804454400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006111133400200602, + "loss": 1.5813, + "theoretical_loss": 3.7555442782708885, + "tokens_seen": 804519936 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006110972918756269, + "loss": 1.4068, + "theoretical_loss": 3.75551421384557, + "tokens_seen": 804585472 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006110812437311936, + "loss": 1.3386, + "theoretical_loss": 3.755484152554605, + "tokens_seen": 804651008 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006110651955867603, + "loss": 1.3804, + "theoretical_loss": 3.755454094397411, + "tokens_seen": 804716544 + }, + { + "epoch": 0.24, + "learning_rate": 0.000611049147442327, + "loss": 1.4489, + "theoretical_loss": 3.7554240393734073, + "tokens_seen": 804782080 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006110330992978937, + "loss": 1.4848, + "theoretical_loss": 3.755393987482011, + "tokens_seen": 804847616 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006110170511534604, + "loss": 1.4785, + "theoretical_loss": 3.755363938722642, + "tokens_seen": 804913152 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006110010030090271, + "loss": 1.4717, + "theoretical_loss": 3.7553338930947167, + "tokens_seen": 804978688 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006109849548645938, + "loss": 1.4364, + "theoretical_loss": 3.7553038505976564, + "tokens_seen": 805044224 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006109689067201605, + "loss": 1.3782, + "theoretical_loss": 3.7552738112308783, + "tokens_seen": 805109760 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006109528585757272, + "loss": 1.5244, + "theoretical_loss": 3.7552437749938017, + "tokens_seen": 805175296 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006109368104312939, + "loss": 1.5563, + "theoretical_loss": 3.755213741885847, + "tokens_seen": 805240832 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006109207622868606, + "loss": 1.517, + "theoretical_loss": 3.755183711906432, + "tokens_seen": 805306368 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006109047141424273, + "loss": 1.3637, + "theoretical_loss": 3.755153685054977, + "tokens_seen": 805371904 + }, + { + "epoch": 0.24, + "learning_rate": 0.000610888665997994, + "loss": 1.4848, + "theoretical_loss": 3.755123661330902, + "tokens_seen": 805437440 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006108726178535607, + "loss": 1.5478, + "theoretical_loss": 3.755093640733626, + "tokens_seen": 805502976 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006108565697091274, + "loss": 1.4697, + "theoretical_loss": 3.75506362326257, + "tokens_seen": 805568512 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006108405215646941, + "loss": 1.5593, + "theoretical_loss": 3.755033608917153, + "tokens_seen": 805634048 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006108244734202608, + "loss": 1.4411, + "theoretical_loss": 3.7550035976967964, + "tokens_seen": 805699584 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006108084252758275, + "loss": 1.4267, + "theoretical_loss": 3.7549735896009206, + "tokens_seen": 805765120 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006107923771313942, + "loss": 1.5048, + "theoretical_loss": 3.754943584628945, + "tokens_seen": 805830656 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006107763289869609, + "loss": 1.5495, + "theoretical_loss": 3.7549135827802917, + "tokens_seen": 805896192 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006107602808425276, + "loss": 1.4294, + "theoretical_loss": 3.7548835840543813, + "tokens_seen": 805961728 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006107442326980943, + "loss": 1.3786, + "theoretical_loss": 3.7548535884506347, + "tokens_seen": 806027264 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.9964765310287476, + "objective/train/avg_token_score": 0.0035234466195106506, + "objective/train/docs_used": 297892, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4902340173721313, + "objective/train/original_loss": 1.4962244033813477, + "objective/train/theoretical_loss": 3.754823595968473, + "objective/train/tokens_used": 826552800, + "objective/train/unlikelihood_loss": 0.009822587482631207, + "theoretical_loss": 3.754823595968473, + "tokens_seen": 806092800 + }, + { + "epoch": 0.24, + "learning_rate": 0.000610728184553661, + "loss": 1.4043, + "theoretical_loss": 3.754823595968473, + "tokens_seen": 806092800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006107121364092277, + "loss": 1.5002, + "theoretical_loss": 3.7547936066073175, + "tokens_seen": 806158336 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006106960882647944, + "loss": 1.4838, + "theoretical_loss": 3.7547636203665906, + "tokens_seen": 806223872 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006106800401203611, + "loss": 1.4099, + "theoretical_loss": 3.7547336372457134, + "tokens_seen": 806289408 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006106639919759278, + "loss": 1.4083, + "theoretical_loss": 3.7547036572441073, + "tokens_seen": 806354944 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006106479438314945, + "loss": 1.5902, + "theoretical_loss": 3.7546736803611953, + "tokens_seen": 806420480 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006106318956870612, + "loss": 1.6037, + "theoretical_loss": 3.754643706596399, + "tokens_seen": 806486016 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006106158475426279, + "loss": 1.4938, + "theoretical_loss": 3.7546137359491407, + "tokens_seen": 806551552 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006105997993981946, + "loss": 1.3488, + "theoretical_loss": 3.7545837684188426, + "tokens_seen": 806617088 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006105837512537613, + "loss": 1.3627, + "theoretical_loss": 3.7545538040049284, + "tokens_seen": 806682624 + }, + { + "epoch": 0.24, + "learning_rate": 0.000610567703109328, + "loss": 1.4542, + "theoretical_loss": 3.7545238427068197, + "tokens_seen": 806748160 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006105516549648947, + "loss": 1.3888, + "theoretical_loss": 3.7544938845239404, + "tokens_seen": 806813696 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006105356068204614, + "loss": 1.4659, + "theoretical_loss": 3.7544639294557127, + "tokens_seen": 806879232 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006105195586760281, + "loss": 1.4692, + "theoretical_loss": 3.7544339775015603, + "tokens_seen": 806944768 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006105035105315948, + "loss": 1.497, + "theoretical_loss": 3.7544040286609066, + "tokens_seen": 807010304 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006104874623871615, + "loss": 1.351, + "theoretical_loss": 3.7543740829331744, + "tokens_seen": 807075840 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006104714142427282, + "loss": 1.4974, + "theoretical_loss": 3.754344140317789, + "tokens_seen": 807141376 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006104553660982949, + "loss": 1.4157, + "theoretical_loss": 3.754314200814173, + "tokens_seen": 807206912 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006104393179538616, + "loss": 1.4401, + "theoretical_loss": 3.754284264421751, + "tokens_seen": 807272448 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006104232698094283, + "loss": 1.4546, + "theoretical_loss": 3.754254331139946, + "tokens_seen": 807337984 + }, + { + "epoch": 0.24, + "learning_rate": 0.000610407221664995, + "loss": 1.3685, + "theoretical_loss": 3.7542244009681847, + "tokens_seen": 807403520 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006103911735205617, + "loss": 1.4433, + "theoretical_loss": 3.7541944739058892, + "tokens_seen": 807469056 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006103751253761284, + "loss": 1.4318, + "theoretical_loss": 3.754164549952485, + "tokens_seen": 807534592 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006103590772316951, + "loss": 1.3735, + "theoretical_loss": 3.754134629107397, + "tokens_seen": 807600128 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006103430290872618, + "loss": 1.4083, + "theoretical_loss": 3.7541047113700503, + "tokens_seen": 807665664 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006103269809428285, + "loss": 1.5221, + "theoretical_loss": 3.7540747967398698, + "tokens_seen": 807731200 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006103109327983953, + "loss": 1.3707, + "theoretical_loss": 3.75404488521628, + "tokens_seen": 807796736 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006102948846539619, + "loss": 1.411, + "theoretical_loss": 3.7540149767987074, + "tokens_seen": 807862272 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006102788365095287, + "loss": 1.4479, + "theoretical_loss": 3.753985071486577, + "tokens_seen": 807927808 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006102627883650953, + "loss": 1.4482, + "theoretical_loss": 3.7539551692793145, + "tokens_seen": 807993344 + }, + { + "epoch": 0.24, + "learning_rate": 0.000610246740220662, + "loss": 1.5233, + "theoretical_loss": 3.753925270176346, + "tokens_seen": 808058880 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006102306920762287, + "loss": 1.4862, + "theoretical_loss": 3.7538953741770973, + "tokens_seen": 808124416 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006102146439317955, + "loss": 1.5229, + "theoretical_loss": 3.753865481280995, + "tokens_seen": 808189952 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006101985957873622, + "loss": 1.4152, + "theoretical_loss": 3.753835591487464, + "tokens_seen": 808255488 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006101825476429289, + "loss": 1.4448, + "theoretical_loss": 3.7538057047959326, + "tokens_seen": 808321024 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006101664994984956, + "loss": 1.5184, + "theoretical_loss": 3.753775821205826, + "tokens_seen": 808386560 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006101504513540623, + "loss": 1.4664, + "theoretical_loss": 3.7537459407165716, + "tokens_seen": 808452096 + }, + { + "epoch": 0.25, + "learning_rate": 0.000610134403209629, + "loss": 1.3919, + "theoretical_loss": 3.7537160633275963, + "tokens_seen": 808517632 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006101183550651957, + "loss": 1.5106, + "theoretical_loss": 3.7536861890383273, + "tokens_seen": 808583168 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006101023069207623, + "loss": 1.5569, + "theoretical_loss": 3.7536563178481916, + "tokens_seen": 808648704 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006100862587763291, + "loss": 1.4222, + "theoretical_loss": 3.753626449756616, + "tokens_seen": 808714240 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006100702106318957, + "loss": 1.4551, + "theoretical_loss": 3.753596584763029, + "tokens_seen": 808779776 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006100541624874625, + "loss": 1.4042, + "theoretical_loss": 3.753566722866858, + "tokens_seen": 808845312 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006100381143430291, + "loss": 1.543, + "theoretical_loss": 3.753536864067531, + "tokens_seen": 808910848 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006100220661985959, + "loss": 1.6147, + "theoretical_loss": 3.7535070083644753, + "tokens_seen": 808976384 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006100060180541625, + "loss": 1.4441, + "theoretical_loss": 3.7534771557571194, + "tokens_seen": 809041920 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006099899699097292, + "loss": 1.3949, + "theoretical_loss": 3.7534473062448916, + "tokens_seen": 809107456 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006099739217652959, + "loss": 1.4448, + "theoretical_loss": 3.7534174598272205, + "tokens_seen": 809172992 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006099578736208626, + "loss": 1.475, + "theoretical_loss": 3.7533876165035345, + "tokens_seen": 809238528 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006099418254764293, + "loss": 1.5156, + "theoretical_loss": 3.7533577762732624, + "tokens_seen": 809304064 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.9942484498023987, + "objective/train/avg_token_score": 0.005751538556069136, + "objective/train/docs_used": 298985, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5027519464492798, + "objective/train/original_loss": 1.5124866962432861, + "objective/train/theoretical_loss": 3.753327939135833, + "objective/train/tokens_used": 829829600, + "objective/train/unlikelihood_loss": 0.01276472955942154, + "theoretical_loss": 3.753327939135833, + "tokens_seen": 809369600 + }, + { + "epoch": 0.25, + "learning_rate": 0.000609925777331996, + "loss": 1.5301, + "theoretical_loss": 3.753327939135833, + "tokens_seen": 809369600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006099097291875627, + "loss": 1.5181, + "theoretical_loss": 3.7532981050906757, + "tokens_seen": 809435136 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006098936810431294, + "loss": 1.5117, + "theoretical_loss": 3.753268274137219, + "tokens_seen": 809500672 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006098776328986961, + "loss": 1.4521, + "theoretical_loss": 3.753238446274893, + "tokens_seen": 809566208 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006098615847542628, + "loss": 1.4182, + "theoretical_loss": 3.7532086215031266, + "tokens_seen": 809631744 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006098455366098295, + "loss": 1.513, + "theoretical_loss": 3.7531787998213497, + "tokens_seen": 809697280 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006098294884653962, + "loss": 1.4485, + "theoretical_loss": 3.7531489812289927, + "tokens_seen": 809762816 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006098134403209629, + "loss": 1.3862, + "theoretical_loss": 3.753119165725484, + "tokens_seen": 809828352 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006097973921765296, + "loss": 1.3988, + "theoretical_loss": 3.753089353310255, + "tokens_seen": 809893888 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006097813440320963, + "loss": 1.4042, + "theoretical_loss": 3.7530595439827357, + "tokens_seen": 809959424 + }, + { + "epoch": 0.25, + "learning_rate": 0.000609765295887663, + "loss": 1.4202, + "theoretical_loss": 3.7530297377423567, + "tokens_seen": 810024960 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006097492477432297, + "loss": 1.3759, + "theoretical_loss": 3.752999934588548, + "tokens_seen": 810090496 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006097331995987964, + "loss": 1.3963, + "theoretical_loss": 3.7529701345207402, + "tokens_seen": 810156032 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006097171514543631, + "loss": 1.5113, + "theoretical_loss": 3.752940337538365, + "tokens_seen": 810221568 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006097011033099298, + "loss": 1.3984, + "theoretical_loss": 3.752910543640853, + "tokens_seen": 810287104 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006096850551654965, + "loss": 1.3766, + "theoretical_loss": 3.752880752827635, + "tokens_seen": 810352640 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006096690070210632, + "loss": 1.4905, + "theoretical_loss": 3.752850965098143, + "tokens_seen": 810418176 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006096529588766299, + "loss": 1.4251, + "theoretical_loss": 3.752821180451808, + "tokens_seen": 810483712 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006096369107321966, + "loss": 1.458, + "theoretical_loss": 3.7527913988880615, + "tokens_seen": 810549248 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006096208625877633, + "loss": 1.5617, + "theoretical_loss": 3.7527616204063357, + "tokens_seen": 810614784 + }, + { + "epoch": 0.25, + "learning_rate": 0.00060960481444333, + "loss": 1.4322, + "theoretical_loss": 3.7527318450060623, + "tokens_seen": 810680320 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006095887662988967, + "loss": 1.5886, + "theoretical_loss": 3.752702072686673, + "tokens_seen": 810745856 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006095727181544634, + "loss": 1.3811, + "theoretical_loss": 3.7526723034476004, + "tokens_seen": 810811392 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006095566700100301, + "loss": 1.4941, + "theoretical_loss": 3.752642537288277, + "tokens_seen": 810876928 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006095406218655968, + "loss": 1.4912, + "theoretical_loss": 3.752612774208135, + "tokens_seen": 810942464 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006095245737211635, + "loss": 1.394, + "theoretical_loss": 3.752583014206608, + "tokens_seen": 811008000 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006095085255767302, + "loss": 1.5094, + "theoretical_loss": 3.752553257283127, + "tokens_seen": 811073536 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006094924774322969, + "loss": 1.5094, + "theoretical_loss": 3.7525235034371267, + "tokens_seen": 811139072 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006094764292878636, + "loss": 1.4491, + "theoretical_loss": 3.7524937526680393, + "tokens_seen": 811204608 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006094603811434303, + "loss": 1.4161, + "theoretical_loss": 3.752464004975298, + "tokens_seen": 811270144 + }, + { + "epoch": 0.25, + "learning_rate": 0.000609444332998997, + "loss": 1.4576, + "theoretical_loss": 3.7524342603583367, + "tokens_seen": 811335680 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006094282848545637, + "loss": 1.3587, + "theoretical_loss": 3.752404518816588, + "tokens_seen": 811401216 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006094122367101304, + "loss": 1.4662, + "theoretical_loss": 3.752374780349488, + "tokens_seen": 811466752 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006093961885656971, + "loss": 1.4368, + "theoretical_loss": 3.752345044956468, + "tokens_seen": 811532288 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006093801404212638, + "loss": 1.3656, + "theoretical_loss": 3.7523153126369633, + "tokens_seen": 811597824 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006093640922768305, + "loss": 1.4681, + "theoretical_loss": 3.7522855833904076, + "tokens_seen": 811663360 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006093480441323972, + "loss": 1.5003, + "theoretical_loss": 3.7522558572162352, + "tokens_seen": 811728896 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006093319959879639, + "loss": 1.4415, + "theoretical_loss": 3.752226134113881, + "tokens_seen": 811794432 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006093159478435306, + "loss": 1.4652, + "theoretical_loss": 3.7521964140827793, + "tokens_seen": 811859968 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006092998996990973, + "loss": 1.3949, + "theoretical_loss": 3.7521666971223646, + "tokens_seen": 811925504 + }, + { + "epoch": 0.25, + "learning_rate": 0.000609283851554664, + "loss": 1.4446, + "theoretical_loss": 3.752136983232073, + "tokens_seen": 811991040 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006092678034102307, + "loss": 1.5291, + "theoretical_loss": 3.7521072724113376, + "tokens_seen": 812056576 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006092517552657974, + "loss": 1.4747, + "theoretical_loss": 3.7520775646595954, + "tokens_seen": 812122112 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006092357071213641, + "loss": 1.3919, + "theoretical_loss": 3.752047859976281, + "tokens_seen": 812187648 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006092196589769308, + "loss": 1.4107, + "theoretical_loss": 3.75201815836083, + "tokens_seen": 812253184 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006092036108324975, + "loss": 1.4239, + "theoretical_loss": 3.751988459812678, + "tokens_seen": 812318720 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006091875626880642, + "loss": 1.4871, + "theoretical_loss": 3.7519587643312606, + "tokens_seen": 812384256 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006091715145436309, + "loss": 1.5238, + "theoretical_loss": 3.751929071916015, + "tokens_seen": 812449792 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006091554663991976, + "loss": 1.4471, + "theoretical_loss": 3.751899382566376, + "tokens_seen": 812515328 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006091394182547643, + "loss": 1.3848, + "theoretical_loss": 3.7518696962817795, + "tokens_seen": 812580864 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.9939968585968018, + "objective/train/avg_token_score": 0.0060031842440366745, + "objective/train/docs_used": 300102, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4078611135482788, + "objective/train/original_loss": 1.416305661201477, + "objective/train/theoretical_loss": 3.7518400130616634, + "objective/train/tokens_used": 833106400, + "objective/train/unlikelihood_loss": 0.018444731831550598, + "theoretical_loss": 3.7518400130616634, + "tokens_seen": 812646400 + }, + { + "epoch": 0.25, + "learning_rate": 0.000609123370110331, + "loss": 1.4527, + "theoretical_loss": 3.7518400130616634, + "tokens_seen": 812646400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006091073219658977, + "loss": 1.5625, + "theoretical_loss": 3.751810332905463, + "tokens_seen": 812711936 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006090912738214644, + "loss": 1.4249, + "theoretical_loss": 3.751780655812616, + "tokens_seen": 812777472 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006090752256770311, + "loss": 1.4757, + "theoretical_loss": 3.7517509817825583, + "tokens_seen": 812843008 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006090591775325978, + "loss": 1.4419, + "theoretical_loss": 3.7517213108147276, + "tokens_seen": 812908544 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006090431293881645, + "loss": 1.4672, + "theoretical_loss": 3.7516916429085603, + "tokens_seen": 812974080 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006090270812437312, + "loss": 1.4981, + "theoretical_loss": 3.751661978063495, + "tokens_seen": 813039616 + }, + { + "epoch": 0.25, + "learning_rate": 0.000609011033099298, + "loss": 1.3546, + "theoretical_loss": 3.751632316278968, + "tokens_seen": 813105152 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006089949849548647, + "loss": 1.4688, + "theoretical_loss": 3.751602657554418, + "tokens_seen": 813170688 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006089789368104313, + "loss": 1.304, + "theoretical_loss": 3.7515730018892812, + "tokens_seen": 813236224 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006089628886659981, + "loss": 1.3909, + "theoretical_loss": 3.7515433492829966, + "tokens_seen": 813301760 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006089468405215648, + "loss": 1.4644, + "theoretical_loss": 3.751513699735002, + "tokens_seen": 813367296 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006089307923771315, + "loss": 1.3508, + "theoretical_loss": 3.7514840532447353, + "tokens_seen": 813432832 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006089147442326982, + "loss": 1.429, + "theoretical_loss": 3.7514544098116356, + "tokens_seen": 813498368 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006088986960882649, + "loss": 1.525, + "theoretical_loss": 3.7514247694351406, + "tokens_seen": 813563904 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006088826479438316, + "loss": 1.464, + "theoretical_loss": 3.7513951321146894, + "tokens_seen": 813629440 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006088665997993983, + "loss": 1.4755, + "theoretical_loss": 3.7513654978497213, + "tokens_seen": 813694976 + }, + { + "epoch": 0.25, + "learning_rate": 0.000608850551654965, + "loss": 1.392, + "theoretical_loss": 3.751335866639674, + "tokens_seen": 813760512 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006088345035105317, + "loss": 1.4459, + "theoretical_loss": 3.751306238483987, + "tokens_seen": 813826048 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006088184553660984, + "loss": 1.5116, + "theoretical_loss": 3.7512766133820996, + "tokens_seen": 813891584 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006088024072216651, + "loss": 1.4149, + "theoretical_loss": 3.751246991333452, + "tokens_seen": 813957120 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006087863590772318, + "loss": 1.4328, + "theoretical_loss": 3.751217372337482, + "tokens_seen": 814022656 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006087703109327985, + "loss": 1.495, + "theoretical_loss": 3.751187756393631, + "tokens_seen": 814088192 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006087542627883652, + "loss": 1.4616, + "theoretical_loss": 3.751158143501338, + "tokens_seen": 814153728 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006087382146439319, + "loss": 1.4467, + "theoretical_loss": 3.7511285336600433, + "tokens_seen": 814219264 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006087221664994986, + "loss": 1.4012, + "theoretical_loss": 3.7510989268691866, + "tokens_seen": 814284800 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006087061183550652, + "loss": 1.5863, + "theoretical_loss": 3.751069323128209, + "tokens_seen": 814350336 + }, + { + "epoch": 0.25, + "learning_rate": 0.000608690070210632, + "loss": 1.4773, + "theoretical_loss": 3.7510397224365493, + "tokens_seen": 814415872 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006086740220661986, + "loss": 1.5295, + "theoretical_loss": 3.7510101247936496, + "tokens_seen": 814481408 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006086579739217654, + "loss": 1.5016, + "theoretical_loss": 3.75098053019895, + "tokens_seen": 814546944 + }, + { + "epoch": 0.25, + "learning_rate": 0.000608641925777332, + "loss": 1.4323, + "theoretical_loss": 3.7509509386518918, + "tokens_seen": 814612480 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006086258776328988, + "loss": 1.3469, + "theoretical_loss": 3.7509213501519154, + "tokens_seen": 814678016 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006086098294884654, + "loss": 1.4068, + "theoretical_loss": 3.750891764698462, + "tokens_seen": 814743552 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006085937813440322, + "loss": 1.4328, + "theoretical_loss": 3.750862182290973, + "tokens_seen": 814809088 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006085777331995988, + "loss": 1.334, + "theoretical_loss": 3.7508326029288908, + "tokens_seen": 814874624 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006085616850551655, + "loss": 1.5078, + "theoretical_loss": 3.750803026611656, + "tokens_seen": 814940160 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006085456369107322, + "loss": 1.4352, + "theoretical_loss": 3.7507734533387103, + "tokens_seen": 815005696 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006085295887662989, + "loss": 1.5239, + "theoretical_loss": 3.7507438831094957, + "tokens_seen": 815071232 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006085135406218656, + "loss": 1.5413, + "theoretical_loss": 3.750714315923454, + "tokens_seen": 815136768 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006084974924774323, + "loss": 1.5049, + "theoretical_loss": 3.7506847517800286, + "tokens_seen": 815202304 + }, + { + "epoch": 0.25, + "learning_rate": 0.000608481444332999, + "loss": 1.4277, + "theoretical_loss": 3.7506551906786605, + "tokens_seen": 815267840 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006084653961885657, + "loss": 1.3945, + "theoretical_loss": 3.7506256326187923, + "tokens_seen": 815333376 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006084493480441324, + "loss": 1.5663, + "theoretical_loss": 3.7505960775998677, + "tokens_seen": 815398912 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006084332998996991, + "loss": 1.3856, + "theoretical_loss": 3.7505665256213288, + "tokens_seen": 815464448 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006084172517552658, + "loss": 1.5472, + "theoretical_loss": 3.7505369766826178, + "tokens_seen": 815529984 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006084012036108325, + "loss": 1.4352, + "theoretical_loss": 3.7505074307831787, + "tokens_seen": 815595520 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006083851554663992, + "loss": 1.3861, + "theoretical_loss": 3.7504778879224547, + "tokens_seen": 815661056 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006083691073219659, + "loss": 1.3597, + "theoretical_loss": 3.750448348099889, + "tokens_seen": 815726592 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006083530591775326, + "loss": 1.4693, + "theoretical_loss": 3.7504188113149244, + "tokens_seen": 815792128 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006083370110330993, + "loss": 1.3156, + "theoretical_loss": 3.750389277567006, + "tokens_seen": 815857664 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.9913868308067322, + "objective/train/avg_token_score": 0.008613178506493568, + "objective/train/docs_used": 301260, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.429894208908081, + "objective/train/original_loss": 1.4434889554977417, + "objective/train/theoretical_loss": 3.7503597468555765, + "objective/train/tokens_used": 836383200, + "objective/train/unlikelihood_loss": 0.019538074731826782, + "theoretical_loss": 3.7503597468555765, + "tokens_seen": 815923200 + }, + { + "epoch": 0.25, + "learning_rate": 0.000608320962888666, + "loss": 1.4507, + "theoretical_loss": 3.7503597468555765, + "tokens_seen": 815923200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006083049147442327, + "loss": 1.3412, + "theoretical_loss": 3.7503302191800802, + "tokens_seen": 815988736 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006082888665997994, + "loss": 1.4806, + "theoretical_loss": 3.750300694539961, + "tokens_seen": 816054272 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006082728184553661, + "loss": 1.3449, + "theoretical_loss": 3.750271172934663, + "tokens_seen": 816119808 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006082567703109328, + "loss": 1.4778, + "theoretical_loss": 3.750241654363631, + "tokens_seen": 816185344 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006082407221664995, + "loss": 1.4349, + "theoretical_loss": 3.75021213882631, + "tokens_seen": 816250880 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006082246740220662, + "loss": 1.5573, + "theoretical_loss": 3.7501826263221436, + "tokens_seen": 816316416 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006082086258776329, + "loss": 1.512, + "theoretical_loss": 3.7501531168505773, + "tokens_seen": 816381952 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006081925777331996, + "loss": 1.4774, + "theoretical_loss": 3.750123610411056, + "tokens_seen": 816447488 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006081765295887663, + "loss": 1.4282, + "theoretical_loss": 3.750094107003024, + "tokens_seen": 816513024 + }, + { + "epoch": 0.25, + "learning_rate": 0.000608160481444333, + "loss": 1.2991, + "theoretical_loss": 3.7500646066259278, + "tokens_seen": 816578560 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006081444332998997, + "loss": 1.5492, + "theoretical_loss": 3.750035109279212, + "tokens_seen": 816644096 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006081283851554664, + "loss": 1.442, + "theoretical_loss": 3.7500056149623227, + "tokens_seen": 816709632 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006081123370110331, + "loss": 1.2932, + "theoretical_loss": 3.7499761236747053, + "tokens_seen": 816775168 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006080962888665998, + "loss": 1.4079, + "theoretical_loss": 3.7499466354158058, + "tokens_seen": 816840704 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006080802407221665, + "loss": 1.5041, + "theoretical_loss": 3.7499171501850697, + "tokens_seen": 816906240 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006080641925777332, + "loss": 1.4716, + "theoretical_loss": 3.749887667981944, + "tokens_seen": 816971776 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006080481444332999, + "loss": 1.4025, + "theoretical_loss": 3.7498581888058746, + "tokens_seen": 817037312 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006080320962888666, + "loss": 1.5121, + "theoretical_loss": 3.7498287126563072, + "tokens_seen": 817102848 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006080160481444333, + "loss": 1.4198, + "theoretical_loss": 3.7497992395326896, + "tokens_seen": 817168384 + }, + { + "epoch": 0.25, + "learning_rate": 0.000608, + "loss": 1.4636, + "theoretical_loss": 3.749769769434468, + "tokens_seen": 817233920 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006079839518555667, + "loss": 1.5153, + "theoretical_loss": 3.7497403023610882, + "tokens_seen": 817299456 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006079679037111334, + "loss": 1.5052, + "theoretical_loss": 3.7497108383119997, + "tokens_seen": 817364992 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006079518555667001, + "loss": 1.4822, + "theoretical_loss": 3.749681377286647, + "tokens_seen": 817430528 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006079358074222668, + "loss": 1.3941, + "theoretical_loss": 3.749651919284479, + "tokens_seen": 817496064 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006079197592778335, + "loss": 1.4373, + "theoretical_loss": 3.7496224643049425, + "tokens_seen": 817561600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006079037111334002, + "loss": 1.3928, + "theoretical_loss": 3.7495930123474857, + "tokens_seen": 817627136 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006078876629889669, + "loss": 1.5308, + "theoretical_loss": 3.749563563411556, + "tokens_seen": 817692672 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006078716148445336, + "loss": 1.5124, + "theoretical_loss": 3.7495341174966015, + "tokens_seen": 817758208 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006078555667001003, + "loss": 1.462, + "theoretical_loss": 3.7495046746020693, + "tokens_seen": 817823744 + }, + { + "epoch": 0.25, + "learning_rate": 0.000607839518555667, + "loss": 1.4331, + "theoretical_loss": 3.7494752347274085, + "tokens_seen": 817889280 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006078234704112337, + "loss": 1.4828, + "theoretical_loss": 3.749445797872067, + "tokens_seen": 817954816 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006078074222668004, + "loss": 1.5222, + "theoretical_loss": 3.7494163640354943, + "tokens_seen": 818020352 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006077913741223672, + "loss": 1.4041, + "theoretical_loss": 3.7493869332171377, + "tokens_seen": 818085888 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006077753259779338, + "loss": 1.44, + "theoretical_loss": 3.7493575054164463, + "tokens_seen": 818151424 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006077592778335006, + "loss": 1.4216, + "theoretical_loss": 3.749328080632869, + "tokens_seen": 818216960 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006077432296890672, + "loss": 1.4179, + "theoretical_loss": 3.749298658865855, + "tokens_seen": 818282496 + }, + { + "epoch": 0.25, + "learning_rate": 0.000607727181544634, + "loss": 1.3842, + "theoretical_loss": 3.7492692401148537, + "tokens_seen": 818348032 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006077111334002006, + "loss": 1.4478, + "theoretical_loss": 3.7492398243793144, + "tokens_seen": 818413568 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006076950852557674, + "loss": 1.3145, + "theoretical_loss": 3.7492104116586855, + "tokens_seen": 818479104 + }, + { + "epoch": 0.25, + "learning_rate": 0.000607679037111334, + "loss": 1.5449, + "theoretical_loss": 3.749181001952418, + "tokens_seen": 818544640 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006076629889669008, + "loss": 1.4514, + "theoretical_loss": 3.7491515952599617, + "tokens_seen": 818610176 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006076469408224674, + "loss": 1.4943, + "theoretical_loss": 3.7491221915807653, + "tokens_seen": 818675712 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006076308926780342, + "loss": 1.4517, + "theoretical_loss": 3.74909279091428, + "tokens_seen": 818741248 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006076148445336008, + "loss": 1.4922, + "theoretical_loss": 3.749063393259955, + "tokens_seen": 818806784 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006075987963891676, + "loss": 1.489, + "theoretical_loss": 3.749033998617241, + "tokens_seen": 818872320 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006075827482447342, + "loss": 1.3629, + "theoretical_loss": 3.7490046069855896, + "tokens_seen": 818937856 + }, + { + "epoch": 0.25, + "learning_rate": 0.000607566700100301, + "loss": 1.2809, + "theoretical_loss": 3.74897521836445, + "tokens_seen": 819003392 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006075506519558677, + "loss": 1.2954, + "theoretical_loss": 3.7489458327532734, + "tokens_seen": 819068928 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006075346038114344, + "loss": 1.4639, + "theoretical_loss": 3.748916450151511, + "tokens_seen": 819134464 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.9900604486465454, + "objective/train/avg_token_score": 0.009939553216099739, + "objective/train/docs_used": 302476, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5003114938735962, + "objective/train/original_loss": 1.514665126800537, + "objective/train/theoretical_loss": 3.7488870705586135, + "objective/train/tokens_used": 839660000, + "objective/train/unlikelihood_loss": 0.03018357791006565, + "theoretical_loss": 3.7488870705586135, + "tokens_seen": 819200000 + }, + { + "epoch": 0.25, + "learning_rate": 0.000607518555667001, + "loss": 1.5029, + "theoretical_loss": 3.7488870705586135, + "tokens_seen": 819200000 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006075025075225678, + "loss": 1.31, + "theoretical_loss": 3.7488576939740326, + "tokens_seen": 819265536 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006074864593781345, + "loss": 1.4148, + "theoretical_loss": 3.7488283203972195, + "tokens_seen": 819331072 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006074704112337012, + "loss": 1.4599, + "theoretical_loss": 3.748798949827625, + "tokens_seen": 819396608 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006074543630892679, + "loss": 1.4138, + "theoretical_loss": 3.7487695822647016, + "tokens_seen": 819462144 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006074383149448346, + "loss": 1.519, + "theoretical_loss": 3.7487402177079003, + "tokens_seen": 819527680 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006074222668004013, + "loss": 1.4286, + "theoretical_loss": 3.7487108561566735, + "tokens_seen": 819593216 + }, + { + "epoch": 0.25, + "learning_rate": 0.000607406218655968, + "loss": 1.4693, + "theoretical_loss": 3.748681497610474, + "tokens_seen": 819658752 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006073901705115347, + "loss": 1.4788, + "theoretical_loss": 3.748652142068753, + "tokens_seen": 819724288 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006073741223671014, + "loss": 1.4129, + "theoretical_loss": 3.748622789530963, + "tokens_seen": 819789824 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006073580742226681, + "loss": 1.5514, + "theoretical_loss": 3.748593439996557, + "tokens_seen": 819855360 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006073420260782348, + "loss": 1.4756, + "theoretical_loss": 3.748564093464987, + "tokens_seen": 819920896 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006073259779338015, + "loss": 1.4087, + "theoretical_loss": 3.748534749935706, + "tokens_seen": 819986432 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006073099297893681, + "loss": 1.3702, + "theoretical_loss": 3.748505409408167, + "tokens_seen": 820051968 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006072938816449349, + "loss": 1.3487, + "theoretical_loss": 3.7484760718818233, + "tokens_seen": 820117504 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006072778335005015, + "loss": 1.4857, + "theoretical_loss": 3.748446737356128, + "tokens_seen": 820183040 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006072617853560683, + "loss": 1.4918, + "theoretical_loss": 3.7484174058305344, + "tokens_seen": 820248576 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006072457372116349, + "loss": 1.4647, + "theoretical_loss": 3.748388077304496, + "tokens_seen": 820314112 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006072296890672017, + "loss": 1.415, + "theoretical_loss": 3.7483587517774666, + "tokens_seen": 820379648 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006072136409227683, + "loss": 1.3896, + "theoretical_loss": 3.7483294292488996, + "tokens_seen": 820445184 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006071975927783351, + "loss": 1.4691, + "theoretical_loss": 3.748300109718249, + "tokens_seen": 820510720 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006071815446339017, + "loss": 1.398, + "theoretical_loss": 3.74827079318497, + "tokens_seen": 820576256 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006071654964894684, + "loss": 1.5942, + "theoretical_loss": 3.748241479648515, + "tokens_seen": 820641792 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006071494483450351, + "loss": 1.295, + "theoretical_loss": 3.7482121691083394, + "tokens_seen": 820707328 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006071334002006018, + "loss": 1.4794, + "theoretical_loss": 3.748182861563898, + "tokens_seen": 820772864 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006071173520561685, + "loss": 1.4217, + "theoretical_loss": 3.748153557014645, + "tokens_seen": 820838400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006071013039117352, + "loss": 1.3839, + "theoretical_loss": 3.748124255460035, + "tokens_seen": 820903936 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006070852557673019, + "loss": 1.5676, + "theoretical_loss": 3.748094956899523, + "tokens_seen": 820969472 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006070692076228686, + "loss": 1.5316, + "theoretical_loss": 3.7480656613325642, + "tokens_seen": 821035008 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006070531594784353, + "loss": 1.4737, + "theoretical_loss": 3.7480363687586147, + "tokens_seen": 821100544 + }, + { + "epoch": 0.25, + "learning_rate": 0.000607037111334002, + "loss": 1.4, + "theoretical_loss": 3.7480070791771283, + "tokens_seen": 821166080 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006070210631895687, + "loss": 1.5753, + "theoretical_loss": 3.7479777925875615, + "tokens_seen": 821231616 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006070050150451354, + "loss": 1.5018, + "theoretical_loss": 3.74794850898937, + "tokens_seen": 821297152 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006069889669007021, + "loss": 1.5152, + "theoretical_loss": 3.7479192283820084, + "tokens_seen": 821362688 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006069729187562688, + "loss": 1.4318, + "theoretical_loss": 3.747889950764934, + "tokens_seen": 821428224 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006069568706118355, + "loss": 1.5724, + "theoretical_loss": 3.747860676137603, + "tokens_seen": 821493760 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006069408224674022, + "loss": 1.4528, + "theoretical_loss": 3.74783140449947, + "tokens_seen": 821559296 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006069247743229689, + "loss": 1.4423, + "theoretical_loss": 3.747802135849993, + "tokens_seen": 821624832 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006069087261785356, + "loss": 1.4985, + "theoretical_loss": 3.747772870188628, + "tokens_seen": 821690368 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006068926780341023, + "loss": 1.4572, + "theoretical_loss": 3.747743607514831, + "tokens_seen": 821755904 + }, + { + "epoch": 0.25, + "learning_rate": 0.000606876629889669, + "loss": 1.4531, + "theoretical_loss": 3.74771434782806, + "tokens_seen": 821821440 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006068605817452357, + "loss": 1.348, + "theoretical_loss": 3.747685091127771, + "tokens_seen": 821886976 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006068445336008024, + "loss": 1.3594, + "theoretical_loss": 3.7476558374134217, + "tokens_seen": 821952512 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006068284854563691, + "loss": 1.6128, + "theoretical_loss": 3.7476265866844685, + "tokens_seen": 822018048 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006068124373119358, + "loss": 1.4023, + "theoretical_loss": 3.7475973389403694, + "tokens_seen": 822083584 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006067963891675025, + "loss": 1.3504, + "theoretical_loss": 3.747568094180582, + "tokens_seen": 822149120 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006067803410230692, + "loss": 1.4502, + "theoretical_loss": 3.747538852404563, + "tokens_seen": 822214656 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006067642928786359, + "loss": 1.3684, + "theoretical_loss": 3.747509613611771, + "tokens_seen": 822280192 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006067482447342026, + "loss": 1.4259, + "theoretical_loss": 3.747480377801664, + "tokens_seen": 822345728 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006067321965897693, + "loss": 1.4389, + "theoretical_loss": 3.7474511449737, + "tokens_seen": 822411264 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.9885343313217163, + "objective/train/avg_token_score": 0.01146561186760664, + "objective/train/docs_used": 303734, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4793987274169922, + "objective/train/original_loss": 1.4920737743377686, + "objective/train/theoretical_loss": 3.7474219151273367, + "objective/train/tokens_used": 842936800, + "objective/train/unlikelihood_loss": 0.04166296124458313, + "theoretical_loss": 3.7474219151273367, + "tokens_seen": 822476800 + }, + { + "epoch": 0.25, + "learning_rate": 0.000606716148445336, + "loss": 1.5262, + "theoretical_loss": 3.7474219151273367, + "tokens_seen": 822476800 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006067001003009027, + "loss": 1.5138, + "theoretical_loss": 3.7473926882620328, + "tokens_seen": 822542336 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006066840521564694, + "loss": 1.5457, + "theoretical_loss": 3.7473634643772473, + "tokens_seen": 822607872 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006066680040120361, + "loss": 1.5256, + "theoretical_loss": 3.747334243472438, + "tokens_seen": 822673408 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006066519558676028, + "loss": 1.3357, + "theoretical_loss": 3.747305025547064, + "tokens_seen": 822738944 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006066359077231695, + "loss": 1.5435, + "theoretical_loss": 3.747275810600584, + "tokens_seen": 822804480 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006066198595787362, + "loss": 1.4364, + "theoretical_loss": 3.747246598632458, + "tokens_seen": 822870016 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006066038114343029, + "loss": 1.4251, + "theoretical_loss": 3.7472173896421435, + "tokens_seen": 822935552 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006065877632898697, + "loss": 1.4545, + "theoretical_loss": 3.7471881836291017, + "tokens_seen": 823001088 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006065717151454363, + "loss": 1.4175, + "theoretical_loss": 3.7471589805927907, + "tokens_seen": 823066624 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006065556670010031, + "loss": 1.438, + "theoretical_loss": 3.7471297805326707, + "tokens_seen": 823132160 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006065396188565697, + "loss": 1.5368, + "theoretical_loss": 3.7471005834482014, + "tokens_seen": 823197696 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006065235707121365, + "loss": 1.5262, + "theoretical_loss": 3.7470713893388425, + "tokens_seen": 823263232 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006065075225677031, + "loss": 1.3783, + "theoretical_loss": 3.7470421982040545, + "tokens_seen": 823328768 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006064914744232699, + "loss": 1.3806, + "theoretical_loss": 3.7470130100432972, + "tokens_seen": 823394304 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006064754262788365, + "loss": 1.4145, + "theoretical_loss": 3.746983824856031, + "tokens_seen": 823459840 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006064593781344033, + "loss": 1.3988, + "theoretical_loss": 3.7469546426417164, + "tokens_seen": 823525376 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006064433299899699, + "loss": 1.5203, + "theoretical_loss": 3.746925463399814, + "tokens_seen": 823590912 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006064272818455367, + "loss": 1.3968, + "theoretical_loss": 3.746896287129785, + "tokens_seen": 823656448 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006064112337011033, + "loss": 1.5011, + "theoretical_loss": 3.7468671138310894, + "tokens_seen": 823721984 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006063951855566701, + "loss": 1.4897, + "theoretical_loss": 3.7468379435031887, + "tokens_seen": 823787520 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006063791374122367, + "loss": 1.4052, + "theoretical_loss": 3.7468087761455444, + "tokens_seen": 823853056 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006063630892678035, + "loss": 1.4131, + "theoretical_loss": 3.746779611757617, + "tokens_seen": 823918592 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006063470411233701, + "loss": 1.29, + "theoretical_loss": 3.746750450338869, + "tokens_seen": 823984128 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006063309929789369, + "loss": 1.4711, + "theoretical_loss": 3.746721291888761, + "tokens_seen": 824049664 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006063149448345035, + "loss": 1.4401, + "theoretical_loss": 3.7466921364067556, + "tokens_seen": 824115200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006062988966900703, + "loss": 1.5055, + "theoretical_loss": 3.746662983892314, + "tokens_seen": 824180736 + }, + { + "epoch": 0.25, + "learning_rate": 0.000606282848545637, + "loss": 1.5062, + "theoretical_loss": 3.746633834344898, + "tokens_seen": 824246272 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006062668004012037, + "loss": 1.4771, + "theoretical_loss": 3.7466046877639707, + "tokens_seen": 824311808 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006062507522567703, + "loss": 1.4286, + "theoretical_loss": 3.7465755441489943, + "tokens_seen": 824377344 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006062347041123371, + "loss": 1.52, + "theoretical_loss": 3.74654640349943, + "tokens_seen": 824442880 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006062186559679037, + "loss": 1.5501, + "theoretical_loss": 3.7465172658147416, + "tokens_seen": 824508416 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006062026078234705, + "loss": 1.4274, + "theoretical_loss": 3.7464881310943916, + "tokens_seen": 824573952 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006061865596790372, + "loss": 1.3576, + "theoretical_loss": 3.7464589993378423, + "tokens_seen": 824639488 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006061705115346039, + "loss": 1.4361, + "theoretical_loss": 3.7464298705445573, + "tokens_seen": 824705024 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006061544633901706, + "loss": 1.494, + "theoretical_loss": 3.7464007447139993, + "tokens_seen": 824770560 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006061384152457373, + "loss": 1.5082, + "theoretical_loss": 3.746371621845632, + "tokens_seen": 824836096 + }, + { + "epoch": 0.25, + "learning_rate": 0.000606122367101304, + "loss": 1.3878, + "theoretical_loss": 3.7463425019389187, + "tokens_seen": 824901632 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006061063189568707, + "loss": 1.4291, + "theoretical_loss": 3.746313384993323, + "tokens_seen": 824967168 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006060902708124374, + "loss": 1.4538, + "theoretical_loss": 3.746284271008308, + "tokens_seen": 825032704 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006060742226680041, + "loss": 1.5056, + "theoretical_loss": 3.7462551599833382, + "tokens_seen": 825098240 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006060581745235708, + "loss": 1.4156, + "theoretical_loss": 3.746226051917877, + "tokens_seen": 825163776 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006060421263791375, + "loss": 1.3433, + "theoretical_loss": 3.7461969468113896, + "tokens_seen": 825229312 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006060260782347042, + "loss": 1.5596, + "theoretical_loss": 3.746167844663339, + "tokens_seen": 825294848 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006060100300902709, + "loss": 1.3986, + "theoretical_loss": 3.7461387454731905, + "tokens_seen": 825360384 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006059939819458376, + "loss": 1.343, + "theoretical_loss": 3.746109649240408, + "tokens_seen": 825425920 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006059779338014043, + "loss": 1.5811, + "theoretical_loss": 3.7460805559644568, + "tokens_seen": 825491456 + }, + { + "epoch": 0.25, + "learning_rate": 0.000605961885656971, + "loss": 1.4253, + "theoretical_loss": 3.7460514656448005, + "tokens_seen": 825556992 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006059458375125377, + "loss": 1.4634, + "theoretical_loss": 3.7460223782809057, + "tokens_seen": 825622528 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006059297893681044, + "loss": 1.5034, + "theoretical_loss": 3.7459932938722362, + "tokens_seen": 825688064 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.9951217174530029, + "objective/train/avg_token_score": 0.004878250882029533, + "objective/train/docs_used": 304950, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.3867065906524658, + "objective/train/original_loss": 1.3944652080535889, + "objective/train/theoretical_loss": 3.745964212418258, + "objective/train/tokens_used": 846213600, + "objective/train/unlikelihood_loss": 0.01057505514472723, + "theoretical_loss": 3.745964212418258, + "tokens_seen": 825753600 + }, + { + "epoch": 0.25, + "learning_rate": 0.000605913741223671, + "loss": 1.4478, + "theoretical_loss": 3.745964212418258, + "tokens_seen": 825753600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006058976930792378, + "loss": 1.5225, + "theoretical_loss": 3.7459351339184366, + "tokens_seen": 825819136 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006058816449348044, + "loss": 1.5135, + "theoretical_loss": 3.7459060583722366, + "tokens_seen": 825884672 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006058655967903712, + "loss": 1.5014, + "theoretical_loss": 3.745876985779124, + "tokens_seen": 825950208 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006058495486459378, + "loss": 1.4672, + "theoretical_loss": 3.745847916138565, + "tokens_seen": 826015744 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006058335005015046, + "loss": 1.4396, + "theoretical_loss": 3.745818849450025, + "tokens_seen": 826081280 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006058174523570712, + "loss": 1.4742, + "theoretical_loss": 3.7457897857129705, + "tokens_seen": 826146816 + }, + { + "epoch": 0.25, + "learning_rate": 0.000605801404212638, + "loss": 1.4694, + "theoretical_loss": 3.7457607249268676, + "tokens_seen": 826212352 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006057853560682046, + "loss": 1.4701, + "theoretical_loss": 3.7457316670911824, + "tokens_seen": 826277888 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006057693079237713, + "loss": 1.3713, + "theoretical_loss": 3.7457026122053816, + "tokens_seen": 826343424 + }, + { + "epoch": 0.25, + "learning_rate": 0.000605753259779338, + "loss": 1.4403, + "theoretical_loss": 3.7456735602689317, + "tokens_seen": 826408960 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006057372116349047, + "loss": 1.4149, + "theoretical_loss": 3.7456445112812995, + "tokens_seen": 826474496 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006057211634904714, + "loss": 1.4649, + "theoretical_loss": 3.745615465241952, + "tokens_seen": 826540032 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006057051153460381, + "loss": 1.3077, + "theoretical_loss": 3.745586422150356, + "tokens_seen": 826605568 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006056890672016048, + "loss": 1.5037, + "theoretical_loss": 3.745557382005979, + "tokens_seen": 826671104 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006056730190571715, + "loss": 1.4788, + "theoretical_loss": 3.7455283448082883, + "tokens_seen": 826736640 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006056569709127382, + "loss": 1.5419, + "theoretical_loss": 3.7454993105567507, + "tokens_seen": 826802176 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006056409227683049, + "loss": 1.4458, + "theoretical_loss": 3.745470279250835, + "tokens_seen": 826867712 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006056248746238716, + "loss": 1.4098, + "theoretical_loss": 3.7454412508900075, + "tokens_seen": 826933248 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006056088264794383, + "loss": 1.3865, + "theoretical_loss": 3.7454122254737374, + "tokens_seen": 826998784 + }, + { + "epoch": 0.25, + "learning_rate": 0.000605592778335005, + "loss": 1.5491, + "theoretical_loss": 3.7453832030014915, + "tokens_seen": 827064320 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006055767301905717, + "loss": 1.5453, + "theoretical_loss": 3.745354183472739, + "tokens_seen": 827129856 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006055606820461384, + "loss": 1.5896, + "theoretical_loss": 3.745325166886947, + "tokens_seen": 827195392 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006055446339017051, + "loss": 1.5107, + "theoretical_loss": 3.7452961532435856, + "tokens_seen": 827260928 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006055285857572718, + "loss": 1.5174, + "theoretical_loss": 3.7452671425421213, + "tokens_seen": 827326464 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006055125376128385, + "loss": 1.5287, + "theoretical_loss": 3.7452381347820247, + "tokens_seen": 827392000 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006054964894684052, + "loss": 1.6209, + "theoretical_loss": 3.7452091299627637, + "tokens_seen": 827457536 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006054804413239719, + "loss": 1.5024, + "theoretical_loss": 3.745180128083807, + "tokens_seen": 827523072 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006054643931795386, + "loss": 1.4563, + "theoretical_loss": 3.7451511291446242, + "tokens_seen": 827588608 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006054483450351053, + "loss": 1.4466, + "theoretical_loss": 3.745122133144685, + "tokens_seen": 827654144 + }, + { + "epoch": 0.25, + "learning_rate": 0.000605432296890672, + "loss": 1.4887, + "theoretical_loss": 3.7450931400834575, + "tokens_seen": 827719680 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006054162487462387, + "loss": 1.5009, + "theoretical_loss": 3.7450641499604123, + "tokens_seen": 827785216 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006054002006018054, + "loss": 1.5129, + "theoretical_loss": 3.7450351627750185, + "tokens_seen": 827850752 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006053841524573722, + "loss": 1.4696, + "theoretical_loss": 3.7450061785267463, + "tokens_seen": 827916288 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006053681043129388, + "loss": 1.5081, + "theoretical_loss": 3.744977197215065, + "tokens_seen": 827981824 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006053520561685056, + "loss": 1.48, + "theoretical_loss": 3.744948218839446, + "tokens_seen": 828047360 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006053360080240722, + "loss": 1.513, + "theoretical_loss": 3.744919243399358, + "tokens_seen": 828112896 + }, + { + "epoch": 0.25, + "learning_rate": 0.000605319959879639, + "loss": 1.4316, + "theoretical_loss": 3.744890270894272, + "tokens_seen": 828178432 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006053039117352056, + "loss": 1.5367, + "theoretical_loss": 3.7448613013236587, + "tokens_seen": 828243968 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006052878635907724, + "loss": 1.5634, + "theoretical_loss": 3.744832334686988, + "tokens_seen": 828309504 + }, + { + "epoch": 0.25, + "learning_rate": 0.000605271815446339, + "loss": 1.6139, + "theoretical_loss": 3.744803370983732, + "tokens_seen": 828375040 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006052557673019058, + "loss": 1.5501, + "theoretical_loss": 3.7447744102133598, + "tokens_seen": 828440576 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006052397191574724, + "loss": 1.4992, + "theoretical_loss": 3.744745452375344, + "tokens_seen": 828506112 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006052236710130392, + "loss": 1.3707, + "theoretical_loss": 3.7447164974691547, + "tokens_seen": 828571648 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006052076228686058, + "loss": 1.4394, + "theoretical_loss": 3.744687545494264, + "tokens_seen": 828637184 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006051915747241726, + "loss": 1.5046, + "theoretical_loss": 3.744658596450143, + "tokens_seen": 828702720 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006051755265797392, + "loss": 1.4739, + "theoretical_loss": 3.744629650336263, + "tokens_seen": 828768256 + }, + { + "epoch": 0.25, + "learning_rate": 0.000605159478435306, + "loss": 1.4339, + "theoretical_loss": 3.744600707152096, + "tokens_seen": 828833792 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006051434302908726, + "loss": 1.5368, + "theoretical_loss": 3.7445717668971144, + "tokens_seen": 828899328 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006051273821464394, + "loss": 1.4087, + "theoretical_loss": 3.7445428295707894, + "tokens_seen": 828964864 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.9920074343681335, + "objective/train/avg_token_score": 0.00799259077757597, + "objective/train/docs_used": 306096, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.3625860214233398, + "objective/train/original_loss": 1.3730239868164062, + "objective/train/theoretical_loss": 3.7445138951725934, + "objective/train/tokens_used": 849490400, + "objective/train/unlikelihood_loss": 0.020957142114639282, + "theoretical_loss": 3.7445138951725934, + "tokens_seen": 829030400 + }, + { + "epoch": 0.25, + "learning_rate": 0.000605111334002006, + "loss": 1.4584, + "theoretical_loss": 3.7445138951725934, + "tokens_seen": 829030400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006050952858575728, + "loss": 1.4067, + "theoretical_loss": 3.7444849637019986, + "tokens_seen": 829095936 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006050792377131394, + "loss": 1.5666, + "theoretical_loss": 3.7444560351584775, + "tokens_seen": 829161472 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006050631895687062, + "loss": 1.4422, + "theoretical_loss": 3.744427109541503, + "tokens_seen": 829227008 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006050471414242728, + "loss": 1.3611, + "theoretical_loss": 3.744398186850547, + "tokens_seen": 829292544 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006050310932798396, + "loss": 1.4177, + "theoretical_loss": 3.7443692670850828, + "tokens_seen": 829358080 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006050150451354062, + "loss": 1.478, + "theoretical_loss": 3.744340350244583, + "tokens_seen": 829423616 + }, + { + "epoch": 0.25, + "learning_rate": 0.000604998996990973, + "loss": 1.4982, + "theoretical_loss": 3.7443114363285215, + "tokens_seen": 829489152 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006049829488465396, + "loss": 1.3564, + "theoretical_loss": 3.74428252533637, + "tokens_seen": 829554688 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006049669007021064, + "loss": 1.4914, + "theoretical_loss": 3.7442536172676038, + "tokens_seen": 829620224 + }, + { + "epoch": 0.25, + "learning_rate": 0.000604950852557673, + "loss": 1.4626, + "theoretical_loss": 3.744224712121695, + "tokens_seen": 829685760 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006049348044132398, + "loss": 1.3256, + "theoretical_loss": 3.744195809898118, + "tokens_seen": 829751296 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006049187562688064, + "loss": 1.3682, + "theoretical_loss": 3.7441669105963458, + "tokens_seen": 829816832 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006049027081243732, + "loss": 1.5628, + "theoretical_loss": 3.744138014215853, + "tokens_seen": 829882368 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006048866599799398, + "loss": 1.4309, + "theoretical_loss": 3.7441091207561135, + "tokens_seen": 829947904 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006048706118355066, + "loss": 1.4487, + "theoretical_loss": 3.7440802302166016, + "tokens_seen": 830013440 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006048545636910732, + "loss": 1.4554, + "theoretical_loss": 3.7440513425967907, + "tokens_seen": 830078976 + }, + { + "epoch": 0.25, + "learning_rate": 0.00060483851554664, + "loss": 1.5015, + "theoretical_loss": 3.7440224578961563, + "tokens_seen": 830144512 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006048224674022067, + "loss": 1.4662, + "theoretical_loss": 3.743993576114173, + "tokens_seen": 830210048 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006048064192577734, + "loss": 1.3562, + "theoretical_loss": 3.7439646972503144, + "tokens_seen": 830275584 + }, + { + "epoch": 0.25, + "learning_rate": 0.00060479037111334, + "loss": 1.4746, + "theoretical_loss": 3.7439358213040568, + "tokens_seen": 830341120 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006047743229689068, + "loss": 1.4457, + "theoretical_loss": 3.743906948274874, + "tokens_seen": 830406656 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006047582748244735, + "loss": 1.5303, + "theoretical_loss": 3.7438780781622425, + "tokens_seen": 830472192 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006047422266800402, + "loss": 1.4195, + "theoretical_loss": 3.743849210965636, + "tokens_seen": 830537728 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006047261785356069, + "loss": 1.4266, + "theoretical_loss": 3.7438203466845303, + "tokens_seen": 830603264 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006047101303911736, + "loss": 1.4529, + "theoretical_loss": 3.7437914853184022, + "tokens_seen": 830668800 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006046940822467403, + "loss": 1.5167, + "theoretical_loss": 3.743762626866726, + "tokens_seen": 830734336 + }, + { + "epoch": 0.25, + "learning_rate": 0.000604678034102307, + "loss": 1.379, + "theoretical_loss": 3.7437337713289778, + "tokens_seen": 830799872 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006046619859578737, + "loss": 1.406, + "theoretical_loss": 3.743704918704634, + "tokens_seen": 830865408 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006046459378134404, + "loss": 1.4758, + "theoretical_loss": 3.74367606899317, + "tokens_seen": 830930944 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006046298896690071, + "loss": 1.4324, + "theoretical_loss": 3.743647222194063, + "tokens_seen": 830996480 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006046138415245738, + "loss": 1.5064, + "theoretical_loss": 3.7436183783067882, + "tokens_seen": 831062016 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006045977933801405, + "loss": 1.3952, + "theoretical_loss": 3.7435895373308226, + "tokens_seen": 831127552 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006045817452357072, + "loss": 1.3805, + "theoretical_loss": 3.743560699265643, + "tokens_seen": 831193088 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006045656970912739, + "loss": 1.5129, + "theoretical_loss": 3.743531864110726, + "tokens_seen": 831258624 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006045496489468406, + "loss": 1.4097, + "theoretical_loss": 3.743503031865549, + "tokens_seen": 831324160 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006045336008024073, + "loss": 1.49, + "theoretical_loss": 3.743474202529588, + "tokens_seen": 831389696 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006045175526579739, + "loss": 1.4055, + "theoretical_loss": 3.743445376102321, + "tokens_seen": 831455232 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006045015045135407, + "loss": 1.4653, + "theoretical_loss": 3.7434165525832253, + "tokens_seen": 831520768 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006044854563691073, + "loss": 1.4155, + "theoretical_loss": 3.7433877319717777, + "tokens_seen": 831586304 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006044694082246741, + "loss": 1.4959, + "theoretical_loss": 3.743358914267456, + "tokens_seen": 831651840 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006044533600802407, + "loss": 1.5976, + "theoretical_loss": 3.743330099469738, + "tokens_seen": 831717376 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006044373119358075, + "loss": 1.362, + "theoretical_loss": 3.7433012875781015, + "tokens_seen": 831782912 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006044212637913741, + "loss": 1.3482, + "theoretical_loss": 3.743272478592025, + "tokens_seen": 831848448 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006044052156469409, + "loss": 1.5866, + "theoretical_loss": 3.743243672510986, + "tokens_seen": 831913984 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006043891675025075, + "loss": 1.4911, + "theoretical_loss": 3.7432148693344627, + "tokens_seen": 831979520 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006043731193580742, + "loss": 1.5428, + "theoretical_loss": 3.7431860690619336, + "tokens_seen": 832045056 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006043570712136409, + "loss": 1.4376, + "theoretical_loss": 3.7431572716928776, + "tokens_seen": 832110592 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006043410230692076, + "loss": 1.419, + "theoretical_loss": 3.743128477226773, + "tokens_seen": 832176128 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006043249749247743, + "loss": 1.4745, + "theoretical_loss": 3.7430996856630987, + "tokens_seen": 832241664 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.9912421107292175, + "objective/train/avg_token_score": 0.008757867850363255, + "objective/train/docs_used": 307306, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4519895315170288, + "objective/train/original_loss": 1.46575129032135, + "objective/train/theoretical_loss": 3.743070897001333, + "objective/train/tokens_used": 852767200, + "objective/train/unlikelihood_loss": 0.023410141468048096, + "theoretical_loss": 3.743070897001333, + "tokens_seen": 832307200 + }, + { + "epoch": 0.25, + "learning_rate": 0.000604308926780341, + "loss": 1.3959, + "theoretical_loss": 3.743070897001333, + "tokens_seen": 832307200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006042928786359077, + "loss": 1.4793, + "theoretical_loss": 3.743042111240956, + "tokens_seen": 832372736 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006042768304914744, + "loss": 1.4225, + "theoretical_loss": 3.7430133283814464, + "tokens_seen": 832438272 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006042607823470411, + "loss": 1.388, + "theoretical_loss": 3.7429845484222835, + "tokens_seen": 832503808 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006042447342026078, + "loss": 1.5198, + "theoretical_loss": 3.742955771362947, + "tokens_seen": 832569344 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006042286860581745, + "loss": 1.4454, + "theoretical_loss": 3.7429269972029156, + "tokens_seen": 832634880 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006042126379137412, + "loss": 1.5471, + "theoretical_loss": 3.74289822594167, + "tokens_seen": 832700416 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006041965897693079, + "loss": 1.425, + "theoretical_loss": 3.7428694575786903, + "tokens_seen": 832765952 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006041805416248747, + "loss": 1.4352, + "theoretical_loss": 3.742840692113455, + "tokens_seen": 832831488 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006041644934804413, + "loss": 1.5632, + "theoretical_loss": 3.7428119295454456, + "tokens_seen": 832897024 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006041484453360081, + "loss": 1.4293, + "theoretical_loss": 3.7427831698741425, + "tokens_seen": 832962560 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006041323971915747, + "loss": 1.3596, + "theoretical_loss": 3.742754413099025, + "tokens_seen": 833028096 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006041163490471415, + "loss": 1.4901, + "theoretical_loss": 3.742725659219574, + "tokens_seen": 833093632 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006041003009027081, + "loss": 1.4476, + "theoretical_loss": 3.7426969082352706, + "tokens_seen": 833159168 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006040842527582749, + "loss": 1.4377, + "theoretical_loss": 3.7426681601455947, + "tokens_seen": 833224704 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006040682046138415, + "loss": 1.4714, + "theoretical_loss": 3.7426394149500286, + "tokens_seen": 833290240 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006040521564694083, + "loss": 1.4878, + "theoretical_loss": 3.742610672648053, + "tokens_seen": 833355776 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006040361083249749, + "loss": 1.5069, + "theoretical_loss": 3.7425819332391477, + "tokens_seen": 833421312 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006040200601805417, + "loss": 1.4519, + "theoretical_loss": 3.7425531967227954, + "tokens_seen": 833486848 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006040040120361083, + "loss": 1.4852, + "theoretical_loss": 3.7425244630984773, + "tokens_seen": 833552384 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006039879638916751, + "loss": 1.5441, + "theoretical_loss": 3.742495732365675, + "tokens_seen": 833617920 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006039719157472417, + "loss": 1.4674, + "theoretical_loss": 3.74246700452387, + "tokens_seen": 833683456 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006039558676028085, + "loss": 1.5094, + "theoretical_loss": 3.7424382795725446, + "tokens_seen": 833748992 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006039398194583751, + "loss": 1.4058, + "theoretical_loss": 3.7424095575111807, + "tokens_seen": 833814528 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006039237713139419, + "loss": 1.3509, + "theoretical_loss": 3.74238083833926, + "tokens_seen": 833880064 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006039077231695085, + "loss": 1.5028, + "theoretical_loss": 3.742352122056265, + "tokens_seen": 833945600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006038916750250753, + "loss": 1.5054, + "theoretical_loss": 3.7423234086616777, + "tokens_seen": 834011136 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006038756268806419, + "loss": 1.4769, + "theoretical_loss": 3.7422946981549816, + "tokens_seen": 834076672 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006038595787362087, + "loss": 1.5449, + "theoretical_loss": 3.7422659905356586, + "tokens_seen": 834142208 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006038435305917753, + "loss": 1.5549, + "theoretical_loss": 3.742237285803192, + "tokens_seen": 834207744 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006038274824473421, + "loss": 1.5128, + "theoretical_loss": 3.7422085839570647, + "tokens_seen": 834273280 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006038114343029087, + "loss": 1.5911, + "theoretical_loss": 3.7421798849967596, + "tokens_seen": 834338816 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006037953861584755, + "loss": 1.5319, + "theoretical_loss": 3.742151188921759, + "tokens_seen": 834404352 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006037793380140421, + "loss": 1.5263, + "theoretical_loss": 3.742122495731548, + "tokens_seen": 834469888 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006037632898696089, + "loss": 1.5495, + "theoretical_loss": 3.742093805425609, + "tokens_seen": 834535424 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006037472417251755, + "loss": 1.5746, + "theoretical_loss": 3.7420651180034255, + "tokens_seen": 834600960 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006037311935807423, + "loss": 1.5465, + "theoretical_loss": 3.742036433464482, + "tokens_seen": 834666496 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006037151454363089, + "loss": 1.488, + "theoretical_loss": 3.742007751808261, + "tokens_seen": 834732032 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006036990972918757, + "loss": 1.4258, + "theoretical_loss": 3.741979073034248, + "tokens_seen": 834797568 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006036830491474423, + "loss": 1.5499, + "theoretical_loss": 3.7419503971419266, + "tokens_seen": 834863104 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006036670010030091, + "loss": 1.3717, + "theoretical_loss": 3.7419217241307807, + "tokens_seen": 834928640 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006036509528585757, + "loss": 1.4925, + "theoretical_loss": 3.741893054000295, + "tokens_seen": 834994176 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006036349047141425, + "loss": 1.5525, + "theoretical_loss": 3.7418643867499544, + "tokens_seen": 835059712 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006036188565697091, + "loss": 1.4634, + "theoretical_loss": 3.7418357223792427, + "tokens_seen": 835125248 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006036028084252759, + "loss": 1.4708, + "theoretical_loss": 3.7418070608876453, + "tokens_seen": 835190784 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006035867602808425, + "loss": 1.3952, + "theoretical_loss": 3.741778402274647, + "tokens_seen": 835256320 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006035707121364093, + "loss": 1.4115, + "theoretical_loss": 3.7417497465397327, + "tokens_seen": 835321856 + }, + { + "epoch": 0.25, + "learning_rate": 0.000603554663991976, + "loss": 1.4267, + "theoretical_loss": 3.741721093682388, + "tokens_seen": 835387392 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006035386158475427, + "loss": 1.4134, + "theoretical_loss": 3.7416924437020977, + "tokens_seen": 835452928 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006035225677031093, + "loss": 1.4979, + "theoretical_loss": 3.741663796598348, + "tokens_seen": 835518464 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.9886983633041382, + "objective/train/avg_token_score": 0.011301620863378048, + "objective/train/docs_used": 308483, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4086856842041016, + "objective/train/original_loss": 1.4277195930480957, + "objective/train/theoretical_loss": 3.741635152370624, + "objective/train/tokens_used": 856044000, + "objective/train/unlikelihood_loss": 0.02373269572854042, + "theoretical_loss": 3.741635152370624, + "tokens_seen": 835584000 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006035065195586761, + "loss": 1.4049, + "theoretical_loss": 3.741635152370624, + "tokens_seen": 835584000 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006034904714142427, + "loss": 1.3783, + "theoretical_loss": 3.7416065110184116, + "tokens_seen": 835649536 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006034744232698095, + "loss": 1.503, + "theoretical_loss": 3.741577872541196, + "tokens_seen": 835715072 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006034583751253762, + "loss": 1.538, + "theoretical_loss": 3.741549236938464, + "tokens_seen": 835780608 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006034423269809429, + "loss": 1.449, + "theoretical_loss": 3.7415206042097013, + "tokens_seen": 835846144 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006034262788365096, + "loss": 1.5351, + "theoretical_loss": 3.7414919743543944, + "tokens_seen": 835911680 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006034102306920763, + "loss": 1.4138, + "theoretical_loss": 3.7414633473720293, + "tokens_seen": 835977216 + }, + { + "epoch": 0.25, + "learning_rate": 0.000603394182547643, + "loss": 1.4523, + "theoretical_loss": 3.741434723262093, + "tokens_seen": 836042752 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006033781344032097, + "loss": 1.487, + "theoretical_loss": 3.7414061020240723, + "tokens_seen": 836108288 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006033620862587764, + "loss": 1.5121, + "theoretical_loss": 3.741377483657453, + "tokens_seen": 836173824 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006033460381143431, + "loss": 1.408, + "theoretical_loss": 3.741348868161723, + "tokens_seen": 836239360 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006033299899699098, + "loss": 1.487, + "theoretical_loss": 3.741320255536369, + "tokens_seen": 836304896 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006033139418254765, + "loss": 1.4283, + "theoretical_loss": 3.741291645780878, + "tokens_seen": 836370432 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006032978936810432, + "loss": 1.5192, + "theoretical_loss": 3.741263038894737, + "tokens_seen": 836435968 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006032818455366099, + "loss": 1.4689, + "theoretical_loss": 3.741234434877435, + "tokens_seen": 836501504 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006032657973921766, + "loss": 1.4483, + "theoretical_loss": 3.7412058337284577, + "tokens_seen": 836567040 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006032497492477433, + "loss": 1.507, + "theoretical_loss": 3.741177235447294, + "tokens_seen": 836632576 + }, + { + "epoch": 0.25, + "learning_rate": 0.00060323370110331, + "loss": 1.5467, + "theoretical_loss": 3.7411486400334315, + "tokens_seen": 836698112 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006032176529588767, + "loss": 1.4923, + "theoretical_loss": 3.7411200474863575, + "tokens_seen": 836763648 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006032016048144434, + "loss": 1.4692, + "theoretical_loss": 3.7410914578055605, + "tokens_seen": 836829184 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006031855566700101, + "loss": 1.6207, + "theoretical_loss": 3.741062870990529, + "tokens_seen": 836894720 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006031695085255768, + "loss": 1.3887, + "theoretical_loss": 3.741034287040751, + "tokens_seen": 836960256 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006031534603811435, + "loss": 1.4972, + "theoretical_loss": 3.7410057059557156, + "tokens_seen": 837025792 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006031374122367102, + "loss": 1.4572, + "theoretical_loss": 3.7409771277349106, + "tokens_seen": 837091328 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006031213640922769, + "loss": 1.5301, + "theoretical_loss": 3.7409485523778256, + "tokens_seen": 837156864 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006031053159478436, + "loss": 1.4154, + "theoretical_loss": 3.740919979883949, + "tokens_seen": 837222400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006030892678034102, + "loss": 1.3865, + "theoretical_loss": 3.740891410252769, + "tokens_seen": 837287936 + }, + { + "epoch": 0.25, + "learning_rate": 0.000603073219658977, + "loss": 1.4609, + "theoretical_loss": 3.740862843483777, + "tokens_seen": 837353472 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006030571715145436, + "loss": 1.4622, + "theoretical_loss": 3.7408342795764598, + "tokens_seen": 837419008 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006030411233701104, + "loss": 1.4558, + "theoretical_loss": 3.740805718530308, + "tokens_seen": 837484544 + }, + { + "epoch": 0.25, + "learning_rate": 0.000603025075225677, + "loss": 1.5382, + "theoretical_loss": 3.7407771603448117, + "tokens_seen": 837550080 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006030090270812438, + "loss": 1.5719, + "theoretical_loss": 3.7407486050194594, + "tokens_seen": 837615616 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006029929789368104, + "loss": 1.4518, + "theoretical_loss": 3.7407200525537414, + "tokens_seen": 837681152 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006029769307923772, + "loss": 1.3924, + "theoretical_loss": 3.740691502947148, + "tokens_seen": 837746688 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006029608826479438, + "loss": 1.4511, + "theoretical_loss": 3.7406629561991687, + "tokens_seen": 837812224 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006029448345035106, + "loss": 1.5387, + "theoretical_loss": 3.740634412309294, + "tokens_seen": 837877760 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006029287863590772, + "loss": 1.4195, + "theoretical_loss": 3.740605871277014, + "tokens_seen": 837943296 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602912738214644, + "loss": 1.5472, + "theoretical_loss": 3.7405773331018195, + "tokens_seen": 838008832 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006028966900702106, + "loss": 1.3621, + "theoretical_loss": 3.740548797783201, + "tokens_seen": 838074368 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006028806419257774, + "loss": 1.5209, + "theoretical_loss": 3.740520265320649, + "tokens_seen": 838139904 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602864593781344, + "loss": 1.4463, + "theoretical_loss": 3.740491735713654, + "tokens_seen": 838205440 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006028485456369108, + "loss": 1.4385, + "theoretical_loss": 3.740463208961708, + "tokens_seen": 838270976 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006028324974924774, + "loss": 1.5698, + "theoretical_loss": 3.7404346850643013, + "tokens_seen": 838336512 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006028164493480442, + "loss": 1.4422, + "theoretical_loss": 3.740406164020926, + "tokens_seen": 838402048 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006028004012036108, + "loss": 1.4739, + "theoretical_loss": 3.740377645831072, + "tokens_seen": 838467584 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006027843530591776, + "loss": 1.434, + "theoretical_loss": 3.7403491304942325, + "tokens_seen": 838533120 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006027683049147442, + "loss": 1.446, + "theoretical_loss": 3.740320618009898, + "tokens_seen": 838598656 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602752256770311, + "loss": 1.4168, + "theoretical_loss": 3.7402921083775604, + "tokens_seen": 838664192 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006027362086258776, + "loss": 1.3746, + "theoretical_loss": 3.7402636015967117, + "tokens_seen": 838729728 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006027201604814444, + "loss": 1.4637, + "theoretical_loss": 3.7402350976668446, + "tokens_seen": 838795264 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.988420844078064, + "objective/train/avg_token_score": 0.011579135432839394, + "objective/train/docs_used": 309674, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6028941869735718, + "objective/train/original_loss": 1.6241681575775146, + "objective/train/theoretical_loss": 3.7402065965874502, + "objective/train/tokens_used": 859320800, + "objective/train/unlikelihood_loss": 0.022012384608387947, + "theoretical_loss": 3.7402065965874502, + "tokens_seen": 838860800 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602704112337011, + "loss": 1.4941, + "theoretical_loss": 3.7402065965874502, + "tokens_seen": 838860800 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006026880641925778, + "loss": 1.4338, + "theoretical_loss": 3.7401780983580215, + "tokens_seen": 838926336 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006026720160481444, + "loss": 1.4384, + "theoretical_loss": 3.7401496029780503, + "tokens_seen": 838991872 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006026559679037112, + "loss": 1.365, + "theoretical_loss": 3.7401211104470296, + "tokens_seen": 839057408 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006026399197592778, + "loss": 1.4097, + "theoretical_loss": 3.740092620764452, + "tokens_seen": 839122944 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006026238716148446, + "loss": 1.5415, + "theoretical_loss": 3.74006413392981, + "tokens_seen": 839188480 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006026078234704112, + "loss": 1.454, + "theoretical_loss": 3.7400356499425973, + "tokens_seen": 839254016 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602591775325978, + "loss": 1.5628, + "theoretical_loss": 3.7400071688023067, + "tokens_seen": 839319552 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006025757271815446, + "loss": 1.4041, + "theoretical_loss": 3.7399786905084307, + "tokens_seen": 839385088 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006025596790371114, + "loss": 1.4896, + "theoretical_loss": 3.739950215060463, + "tokens_seen": 839450624 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602543630892678, + "loss": 1.448, + "theoretical_loss": 3.7399217424578977, + "tokens_seen": 839516160 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006025275827482448, + "loss": 1.413, + "theoretical_loss": 3.739893272700227, + "tokens_seen": 839581696 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006025115346038114, + "loss": 1.5478, + "theoretical_loss": 3.7398648057869464, + "tokens_seen": 839647232 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006024954864593782, + "loss": 1.4561, + "theoretical_loss": 3.7398363417175484, + "tokens_seen": 839712768 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006024794383149448, + "loss": 1.4547, + "theoretical_loss": 3.7398078804915276, + "tokens_seen": 839778304 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006024633901705116, + "loss": 1.4102, + "theoretical_loss": 3.739779422108377, + "tokens_seen": 839843840 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006024473420260782, + "loss": 1.4173, + "theoretical_loss": 3.7397509665675925, + "tokens_seen": 839909376 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602431293881645, + "loss": 1.4252, + "theoretical_loss": 3.7397225138686676, + "tokens_seen": 839974912 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006024152457372116, + "loss": 1.4212, + "theoretical_loss": 3.739694064011097, + "tokens_seen": 840040448 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006023991975927784, + "loss": 1.5112, + "theoretical_loss": 3.7396656169943743, + "tokens_seen": 840105984 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602383149448345, + "loss": 1.4947, + "theoretical_loss": 3.739637172817996, + "tokens_seen": 840171520 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006023671013039118, + "loss": 1.4137, + "theoretical_loss": 3.739608731481456, + "tokens_seen": 840237056 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006023510531594784, + "loss": 1.4597, + "theoretical_loss": 3.7395802929842494, + "tokens_seen": 840302592 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006023350050150452, + "loss": 1.4811, + "theoretical_loss": 3.739551857325871, + "tokens_seen": 840368128 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006023189568706118, + "loss": 1.46, + "theoretical_loss": 3.7395234245058164, + "tokens_seen": 840433664 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006023029087261786, + "loss": 1.4484, + "theoretical_loss": 3.7394949945235814, + "tokens_seen": 840499200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006022868605817452, + "loss": 1.5408, + "theoretical_loss": 3.7394665673786607, + "tokens_seen": 840564736 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602270812437312, + "loss": 1.5447, + "theoretical_loss": 3.739438143070551, + "tokens_seen": 840630272 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006022547642928786, + "loss": 1.5123, + "theoretical_loss": 3.739409721598747, + "tokens_seen": 840695808 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006022387161484454, + "loss": 1.4417, + "theoretical_loss": 3.739381302962745, + "tokens_seen": 840761344 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602222668004012, + "loss": 1.3717, + "theoretical_loss": 3.739352887162041, + "tokens_seen": 840826880 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006022066198595788, + "loss": 1.4928, + "theoretical_loss": 3.739324474196132, + "tokens_seen": 840892416 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006021905717151454, + "loss": 1.3277, + "theoretical_loss": 3.739296064064513, + "tokens_seen": 840957952 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006021745235707122, + "loss": 1.4885, + "theoretical_loss": 3.739267656766681, + "tokens_seen": 841023488 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006021584754262788, + "loss": 1.5203, + "theoretical_loss": 3.7392392523021325, + "tokens_seen": 841089024 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006021424272818456, + "loss": 1.4918, + "theoretical_loss": 3.7392108506703647, + "tokens_seen": 841154560 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006021263791374122, + "loss": 1.4995, + "theoretical_loss": 3.7391824518708736, + "tokens_seen": 841220096 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602110330992979, + "loss": 1.461, + "theoretical_loss": 3.739154055903157, + "tokens_seen": 841285632 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006020942828485457, + "loss": 1.4488, + "theoretical_loss": 3.7391256627667104, + "tokens_seen": 841351168 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006020782347041124, + "loss": 1.4904, + "theoretical_loss": 3.7390972724610334, + "tokens_seen": 841416704 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602062186559679, + "loss": 1.3773, + "theoretical_loss": 3.7390688849856213, + "tokens_seen": 841482240 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006020461384152458, + "loss": 1.4134, + "theoretical_loss": 3.7390405003399723, + "tokens_seen": 841547776 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006020300902708125, + "loss": 1.4459, + "theoretical_loss": 3.7390121185235836, + "tokens_seen": 841613312 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006020140421263792, + "loss": 1.4658, + "theoretical_loss": 3.7389837395359535, + "tokens_seen": 841678848 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006019979939819459, + "loss": 1.4896, + "theoretical_loss": 3.7389553633765797, + "tokens_seen": 841744384 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006019819458375126, + "loss": 1.4278, + "theoretical_loss": 3.73892699004496, + "tokens_seen": 841809920 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006019658976930793, + "loss": 1.4062, + "theoretical_loss": 3.7388986195405924, + "tokens_seen": 841875456 + }, + { + "epoch": 0.26, + "learning_rate": 0.000601949849548646, + "loss": 1.3648, + "theoretical_loss": 3.7388702518629753, + "tokens_seen": 841940992 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006019338014042127, + "loss": 1.4793, + "theoretical_loss": 3.738841887011607, + "tokens_seen": 842006528 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006019177532597794, + "loss": 1.4809, + "theoretical_loss": 3.7388135249859866, + "tokens_seen": 842072064 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.9974714517593384, + "objective/train/avg_token_score": 0.0025285687297582626, + "objective/train/docs_used": 310992, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.3742161989212036, + "objective/train/original_loss": 1.3787143230438232, + "objective/train/theoretical_loss": 3.738785165785611, + "objective/train/tokens_used": 862597600, + "objective/train/unlikelihood_loss": 0.006540261209011078, + "theoretical_loss": 3.738785165785611, + "tokens_seen": 842137600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006019017051153461, + "loss": 1.4599, + "theoretical_loss": 3.738785165785611, + "tokens_seen": 842137600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006018856569709128, + "loss": 1.4205, + "theoretical_loss": 3.7387568094099812, + "tokens_seen": 842203136 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006018696088264795, + "loss": 1.5283, + "theoretical_loss": 3.738728455858594, + "tokens_seen": 842268672 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006018535606820462, + "loss": 1.5016, + "theoretical_loss": 3.73870010513095, + "tokens_seen": 842334208 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006018375125376129, + "loss": 1.5168, + "theoretical_loss": 3.7386717572265473, + "tokens_seen": 842399744 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006018214643931796, + "loss": 1.54, + "theoretical_loss": 3.738643412144885, + "tokens_seen": 842465280 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006018054162487463, + "loss": 1.5209, + "theoretical_loss": 3.738615069885464, + "tokens_seen": 842530816 + }, + { + "epoch": 0.26, + "learning_rate": 0.000601789368104313, + "loss": 1.4716, + "theoretical_loss": 3.738586730447782, + "tokens_seen": 842596352 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006017733199598797, + "loss": 1.4255, + "theoretical_loss": 3.7385583938313394, + "tokens_seen": 842661888 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006017572718154464, + "loss": 1.4438, + "theoretical_loss": 3.738530060035636, + "tokens_seen": 842727424 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006017412236710131, + "loss": 1.4918, + "theoretical_loss": 3.7385017290601716, + "tokens_seen": 842792960 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006017251755265798, + "loss": 1.435, + "theoretical_loss": 3.7384734009044465, + "tokens_seen": 842858496 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006017091273821465, + "loss": 1.5121, + "theoretical_loss": 3.7384450755679604, + "tokens_seen": 842924032 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006016930792377131, + "loss": 1.4775, + "theoretical_loss": 3.7384167530502133, + "tokens_seen": 842989568 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006016770310932799, + "loss": 1.4565, + "theoretical_loss": 3.738388433350707, + "tokens_seen": 843055104 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006016609829488465, + "loss": 1.3734, + "theoretical_loss": 3.7383601164689404, + "tokens_seen": 843120640 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006016449348044133, + "loss": 1.4627, + "theoretical_loss": 3.7383318024044145, + "tokens_seen": 843186176 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006016288866599799, + "loss": 1.5146, + "theoretical_loss": 3.738303491156631, + "tokens_seen": 843251712 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006016128385155467, + "loss": 1.4897, + "theoretical_loss": 3.73827518272509, + "tokens_seen": 843317248 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006015967903711133, + "loss": 1.53, + "theoretical_loss": 3.738246877109293, + "tokens_seen": 843382784 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006015807422266801, + "loss": 1.4212, + "theoretical_loss": 3.7382185743087404, + "tokens_seen": 843448320 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006015646940822467, + "loss": 1.542, + "theoretical_loss": 3.7381902743229345, + "tokens_seen": 843513856 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006015486459378135, + "loss": 1.5391, + "theoretical_loss": 3.7381619771513757, + "tokens_seen": 843579392 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006015325977933801, + "loss": 1.4291, + "theoretical_loss": 3.738133682793566, + "tokens_seen": 843644928 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006015165496489469, + "loss": 1.5302, + "theoretical_loss": 3.7381053912490074, + "tokens_seen": 843710464 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006015005015045135, + "loss": 1.4936, + "theoretical_loss": 3.7380771025172015, + "tokens_seen": 843776000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006014844533600803, + "loss": 1.4142, + "theoretical_loss": 3.73804881659765, + "tokens_seen": 843841536 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006014684052156469, + "loss": 1.463, + "theoretical_loss": 3.738020533489855, + "tokens_seen": 843907072 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006014523570712137, + "loss": 1.489, + "theoretical_loss": 3.737992253193319, + "tokens_seen": 843972608 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006014363089267803, + "loss": 1.461, + "theoretical_loss": 3.737963975707544, + "tokens_seen": 844038144 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006014202607823471, + "loss": 1.4624, + "theoretical_loss": 3.7379357010320318, + "tokens_seen": 844103680 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006014042126379137, + "loss": 1.3946, + "theoretical_loss": 3.7379074291662864, + "tokens_seen": 844169216 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006013881644934805, + "loss": 1.408, + "theoretical_loss": 3.7378791601098094, + "tokens_seen": 844234752 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006013721163490471, + "loss": 1.4246, + "theoretical_loss": 3.737850893862104, + "tokens_seen": 844300288 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006013560682046139, + "loss": 1.5051, + "theoretical_loss": 3.7378226304226727, + "tokens_seen": 844365824 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006013400200601805, + "loss": 1.3764, + "theoretical_loss": 3.737794369791019, + "tokens_seen": 844431360 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006013239719157473, + "loss": 1.5011, + "theoretical_loss": 3.7377661119666463, + "tokens_seen": 844496896 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006013079237713139, + "loss": 1.6091, + "theoretical_loss": 3.7377378569490576, + "tokens_seen": 844562432 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006012918756268807, + "loss": 1.4542, + "theoretical_loss": 3.7377096047377565, + "tokens_seen": 844627968 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006012758274824473, + "loss": 1.4472, + "theoretical_loss": 3.7376813553322465, + "tokens_seen": 844693504 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006012597793380141, + "loss": 1.5567, + "theoretical_loss": 3.737653108732031, + "tokens_seen": 844759040 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006012437311935807, + "loss": 1.5588, + "theoretical_loss": 3.7376248649366137, + "tokens_seen": 844824576 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006012276830491475, + "loss": 1.3941, + "theoretical_loss": 3.737596623945499, + "tokens_seen": 844890112 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006012116349047141, + "loss": 1.4239, + "theoretical_loss": 3.737568385758191, + "tokens_seen": 844955648 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006011955867602809, + "loss": 1.4437, + "theoretical_loss": 3.737540150374194, + "tokens_seen": 845021184 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006011795386158475, + "loss": 1.5071, + "theoretical_loss": 3.737511917793012, + "tokens_seen": 845086720 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006011634904714143, + "loss": 1.4275, + "theoretical_loss": 3.7374836880141493, + "tokens_seen": 845152256 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006011474423269809, + "loss": 1.5129, + "theoretical_loss": 3.737455461037111, + "tokens_seen": 845217792 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006011313941825477, + "loss": 1.4634, + "theoretical_loss": 3.737427236861401, + "tokens_seen": 845283328 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006011153460381143, + "loss": 1.442, + "theoretical_loss": 3.7373990154865253, + "tokens_seen": 845348864 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.9938036203384399, + "objective/train/avg_token_score": 0.006196343805640936, + "objective/train/docs_used": 312167, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4445252418518066, + "objective/train/original_loss": 1.4536511898040771, + "objective/train/theoretical_loss": 3.7373707969119883, + "objective/train/tokens_used": 865874400, + "objective/train/unlikelihood_loss": 0.01699359342455864, + "theoretical_loss": 3.7373707969119883, + "tokens_seen": 845414400 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006010992978936811, + "loss": 1.444, + "theoretical_loss": 3.7373707969119883, + "tokens_seen": 845414400 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006010832497492477, + "loss": 1.5307, + "theoretical_loss": 3.7373425811372942, + "tokens_seen": 845479936 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006010672016048145, + "loss": 1.5673, + "theoretical_loss": 3.7373143681619494, + "tokens_seen": 845545472 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006010511534603811, + "loss": 1.4019, + "theoretical_loss": 3.737286157985459, + "tokens_seen": 845611008 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006010351053159479, + "loss": 1.494, + "theoretical_loss": 3.737257950607328, + "tokens_seen": 845676544 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006010190571715145, + "loss": 1.472, + "theoretical_loss": 3.7372297460270625, + "tokens_seen": 845742080 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006010030090270813, + "loss": 1.5095, + "theoretical_loss": 3.7372015442441677, + "tokens_seen": 845807616 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006009869608826479, + "loss": 1.4148, + "theoretical_loss": 3.7371733452581504, + "tokens_seen": 845873152 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006009709127382147, + "loss": 1.3609, + "theoretical_loss": 3.737145149068515, + "tokens_seen": 845938688 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006009548645937813, + "loss": 1.4901, + "theoretical_loss": 3.737116955674769, + "tokens_seen": 846004224 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006009388164493481, + "loss": 1.4362, + "theoretical_loss": 3.7370887650764177, + "tokens_seen": 846069760 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006009227683049147, + "loss": 1.4571, + "theoretical_loss": 3.7370605772729677, + "tokens_seen": 846135296 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006009067201604815, + "loss": 1.4418, + "theoretical_loss": 3.7370323922639264, + "tokens_seen": 846200832 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006008906720160481, + "loss": 1.3735, + "theoretical_loss": 3.7370042100487986, + "tokens_seen": 846266368 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006008746238716149, + "loss": 1.4875, + "theoretical_loss": 3.7369760306270923, + "tokens_seen": 846331904 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006008585757271815, + "loss": 1.3701, + "theoretical_loss": 3.7369478539983145, + "tokens_seen": 846397440 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006008425275827483, + "loss": 1.4166, + "theoretical_loss": 3.736919680161971, + "tokens_seen": 846462976 + }, + { + "epoch": 0.26, + "learning_rate": 0.000600826479438315, + "loss": 1.4668, + "theoretical_loss": 3.7368915091175703, + "tokens_seen": 846528512 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006008104312938817, + "loss": 1.4769, + "theoretical_loss": 3.736863340864618, + "tokens_seen": 846594048 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006007943831494483, + "loss": 1.556, + "theoretical_loss": 3.7368351754026228, + "tokens_seen": 846659584 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006007783350050151, + "loss": 1.4395, + "theoretical_loss": 3.7368070127310915, + "tokens_seen": 846725120 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006007622868605817, + "loss": 1.4357, + "theoretical_loss": 3.736778852849532, + "tokens_seen": 846790656 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006007462387161485, + "loss": 1.5112, + "theoretical_loss": 3.736750695757452, + "tokens_seen": 846856192 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006007301905717151, + "loss": 1.4344, + "theoretical_loss": 3.736722541454359, + "tokens_seen": 846921728 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006007141424272819, + "loss": 1.4939, + "theoretical_loss": 3.7366943899397613, + "tokens_seen": 846987264 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006006980942828486, + "loss": 1.3691, + "theoretical_loss": 3.736666241213167, + "tokens_seen": 847052800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006006820461384153, + "loss": 1.494, + "theoretical_loss": 3.736638095274084, + "tokens_seen": 847118336 + }, + { + "epoch": 0.26, + "learning_rate": 0.000600665997993982, + "loss": 1.4605, + "theoretical_loss": 3.736609952122021, + "tokens_seen": 847183872 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006006499498495487, + "loss": 1.3638, + "theoretical_loss": 3.736581811756486, + "tokens_seen": 847249408 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006006339017051154, + "loss": 1.3518, + "theoretical_loss": 3.736553674176988, + "tokens_seen": 847314944 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006006178535606821, + "loss": 1.4425, + "theoretical_loss": 3.736525539383036, + "tokens_seen": 847380480 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006006018054162488, + "loss": 1.3398, + "theoretical_loss": 3.736497407374138, + "tokens_seen": 847446016 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006005857572718155, + "loss": 1.3967, + "theoretical_loss": 3.7364692781498032, + "tokens_seen": 847511552 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006005697091273823, + "loss": 1.4166, + "theoretical_loss": 3.7364411517095415, + "tokens_seen": 847577088 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006005536609829489, + "loss": 1.5205, + "theoretical_loss": 3.736413028052861, + "tokens_seen": 847642624 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006005376128385157, + "loss": 1.4754, + "theoretical_loss": 3.736384907179272, + "tokens_seen": 847708160 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006005215646940823, + "loss": 1.4663, + "theoretical_loss": 3.736356789088283, + "tokens_seen": 847773696 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006005055165496491, + "loss": 1.4026, + "theoretical_loss": 3.7363286737794046, + "tokens_seen": 847839232 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006004894684052157, + "loss": 1.5397, + "theoretical_loss": 3.7363005612521456, + "tokens_seen": 847904768 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006004734202607825, + "loss": 1.412, + "theoretical_loss": 3.736272451506016, + "tokens_seen": 847970304 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006004573721163491, + "loss": 1.4733, + "theoretical_loss": 3.7362443445405265, + "tokens_seen": 848035840 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006004413239719158, + "loss": 1.4368, + "theoretical_loss": 3.736216240355186, + "tokens_seen": 848101376 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006004252758274825, + "loss": 1.5348, + "theoretical_loss": 3.7361881389495064, + "tokens_seen": 848166912 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006004092276830492, + "loss": 1.5278, + "theoretical_loss": 3.736160040322996, + "tokens_seen": 848232448 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006003931795386159, + "loss": 1.4064, + "theoretical_loss": 3.736131944475167, + "tokens_seen": 848297984 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006003771313941826, + "loss": 1.5314, + "theoretical_loss": 3.7361038514055287, + "tokens_seen": 848363520 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006003610832497493, + "loss": 1.3971, + "theoretical_loss": 3.7360757611135926, + "tokens_seen": 848429056 + }, + { + "epoch": 0.26, + "learning_rate": 0.000600345035105316, + "loss": 1.3507, + "theoretical_loss": 3.7360476735988692, + "tokens_seen": 848494592 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006003289869608827, + "loss": 1.4264, + "theoretical_loss": 3.736019588860869, + "tokens_seen": 848560128 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006003129388164494, + "loss": 1.4639, + "theoretical_loss": 3.7359915068991043, + "tokens_seen": 848625664 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.9947525262832642, + "objective/train/avg_token_score": 0.005247475113719702, + "objective/train/docs_used": 313372, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5028502941131592, + "objective/train/original_loss": 1.5119600296020508, + "objective/train/theoretical_loss": 3.7359634277130853, + "objective/train/tokens_used": 869151200, + "objective/train/unlikelihood_loss": 0.01129594910889864, + "theoretical_loss": 3.7359634277130853, + "tokens_seen": 848691200 + }, + { + "epoch": 0.26, + "learning_rate": 0.000600296890672016, + "loss": 1.4504, + "theoretical_loss": 3.7359634277130853, + "tokens_seen": 848691200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006002808425275828, + "loss": 1.5667, + "theoretical_loss": 3.7359353513023237, + "tokens_seen": 848756736 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006002647943831494, + "loss": 1.4144, + "theoretical_loss": 3.7359072776663305, + "tokens_seen": 848822272 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006002487462387162, + "loss": 1.3928, + "theoretical_loss": 3.735879206804618, + "tokens_seen": 848887808 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006002326980942828, + "loss": 1.4949, + "theoretical_loss": 3.735851138716697, + "tokens_seen": 848953344 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006002166499498496, + "loss": 1.4647, + "theoretical_loss": 3.7358230734020803, + "tokens_seen": 849018880 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006002006018054162, + "loss": 1.3662, + "theoretical_loss": 3.7357950108602793, + "tokens_seen": 849084416 + }, + { + "epoch": 0.26, + "learning_rate": 0.000600184553660983, + "loss": 1.4205, + "theoretical_loss": 3.735766951090806, + "tokens_seen": 849149952 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006001685055165496, + "loss": 1.4827, + "theoretical_loss": 3.735738894093173, + "tokens_seen": 849215488 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006001524573721164, + "loss": 1.5992, + "theoretical_loss": 3.7357108398668917, + "tokens_seen": 849281024 + }, + { + "epoch": 0.26, + "learning_rate": 0.000600136409227683, + "loss": 1.5022, + "theoretical_loss": 3.7356827884114754, + "tokens_seen": 849346560 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006001203610832498, + "loss": 1.6339, + "theoretical_loss": 3.735654739726437, + "tokens_seen": 849412096 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006001043129388164, + "loss": 1.4638, + "theoretical_loss": 3.735626693811288, + "tokens_seen": 849477632 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006000882647943832, + "loss": 1.4952, + "theoretical_loss": 3.7355986506655414, + "tokens_seen": 849543168 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006000722166499498, + "loss": 1.3909, + "theoretical_loss": 3.735570610288711, + "tokens_seen": 849608704 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006000561685055166, + "loss": 1.5386, + "theoretical_loss": 3.73554257268031, + "tokens_seen": 849674240 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006000401203610832, + "loss": 1.3893, + "theoretical_loss": 3.73551453783985, + "tokens_seen": 849739776 + }, + { + "epoch": 0.26, + "learning_rate": 0.00060002407221665, + "loss": 1.354, + "theoretical_loss": 3.7354865057668456, + "tokens_seen": 849805312 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006000080240722166, + "loss": 1.4853, + "theoretical_loss": 3.73545847646081, + "tokens_seen": 849870848 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005999919759277834, + "loss": 1.4056, + "theoretical_loss": 3.735430449921256, + "tokens_seen": 849936384 + }, + { + "epoch": 0.26, + "learning_rate": 0.00059997592778335, + "loss": 1.3513, + "theoretical_loss": 3.735402426147698, + "tokens_seen": 850001920 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005999598796389168, + "loss": 1.3632, + "theoretical_loss": 3.7353744051396505, + "tokens_seen": 850067456 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005999438314944834, + "loss": 1.3656, + "theoretical_loss": 3.7353463868966257, + "tokens_seen": 850132992 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005999277833500502, + "loss": 1.4961, + "theoretical_loss": 3.7353183714181393, + "tokens_seen": 850198528 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005999117352056168, + "loss": 1.5691, + "theoretical_loss": 3.735290358703704, + "tokens_seen": 850264064 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005998956870611836, + "loss": 1.4068, + "theoretical_loss": 3.735262348752835, + "tokens_seen": 850329600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005998796389167502, + "loss": 1.5117, + "theoretical_loss": 3.7352343415650466, + "tokens_seen": 850395136 + }, + { + "epoch": 0.26, + "learning_rate": 0.000599863590772317, + "loss": 1.5345, + "theoretical_loss": 3.7352063371398527, + "tokens_seen": 850460672 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005998475426278836, + "loss": 1.4376, + "theoretical_loss": 3.7351783354767685, + "tokens_seen": 850526208 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005998314944834504, + "loss": 1.4401, + "theoretical_loss": 3.735150336575309, + "tokens_seen": 850591744 + }, + { + "epoch": 0.26, + "learning_rate": 0.000599815446339017, + "loss": 1.4872, + "theoretical_loss": 3.7351223404349887, + "tokens_seen": 850657280 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005997993981945838, + "loss": 1.542, + "theoretical_loss": 3.7350943470553224, + "tokens_seen": 850722816 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005997833500501504, + "loss": 1.3511, + "theoretical_loss": 3.735066356435826, + "tokens_seen": 850788352 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005997673019057172, + "loss": 1.5886, + "theoretical_loss": 3.735038368576014, + "tokens_seen": 850853888 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005997512537612838, + "loss": 1.5625, + "theoretical_loss": 3.7350103834754025, + "tokens_seen": 850919424 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005997352056168506, + "loss": 1.4918, + "theoretical_loss": 3.734982401133506, + "tokens_seen": 850984960 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005997191574724172, + "loss": 1.4759, + "theoretical_loss": 3.734954421549841, + "tokens_seen": 851050496 + }, + { + "epoch": 0.26, + "learning_rate": 0.000599703109327984, + "loss": 1.3087, + "theoretical_loss": 3.734926444723923, + "tokens_seen": 851116032 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005996870611835506, + "loss": 1.3929, + "theoretical_loss": 3.734898470655268, + "tokens_seen": 851181568 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005996710130391174, + "loss": 1.3915, + "theoretical_loss": 3.7348704993433914, + "tokens_seen": 851247104 + }, + { + "epoch": 0.26, + "learning_rate": 0.000599654964894684, + "loss": 1.4784, + "theoretical_loss": 3.7348425307878097, + "tokens_seen": 851312640 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005996389167502508, + "loss": 1.4464, + "theoretical_loss": 3.7348145649880395, + "tokens_seen": 851378176 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005996228686058174, + "loss": 1.4437, + "theoretical_loss": 3.734786601943597, + "tokens_seen": 851443712 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005996068204613842, + "loss": 1.305, + "theoretical_loss": 3.7347586416539986, + "tokens_seen": 851509248 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005995907723169508, + "loss": 1.4344, + "theoretical_loss": 3.7347306841187606, + "tokens_seen": 851574784 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005995747241725176, + "loss": 1.5414, + "theoretical_loss": 3.7347027293373998, + "tokens_seen": 851640320 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005995586760280842, + "loss": 1.4241, + "theoretical_loss": 3.7346747773094338, + "tokens_seen": 851705856 + }, + { + "epoch": 0.26, + "learning_rate": 0.000599542627883651, + "loss": 1.461, + "theoretical_loss": 3.7346468280343785, + "tokens_seen": 851771392 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005995265797392176, + "loss": 1.4091, + "theoretical_loss": 3.734618881511752, + "tokens_seen": 851836928 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005995105315947844, + "loss": 1.4138, + "theoretical_loss": 3.7345909377410704, + "tokens_seen": 851902464 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.9930643439292908, + "objective/train/avg_token_score": 0.006935643497854471, + "objective/train/docs_used": 314587, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4835004806518555, + "objective/train/original_loss": 1.4968411922454834, + "objective/train/theoretical_loss": 3.7345629967218517, + "objective/train/tokens_used": 872428000, + "objective/train/unlikelihood_loss": 0.012244139797985554, + "theoretical_loss": 3.7345629967218517, + "tokens_seen": 851968000 + }, + { + "epoch": 0.26, + "learning_rate": 0.000599494483450351, + "loss": 1.4681, + "theoretical_loss": 3.7345629967218517, + "tokens_seen": 851968000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005994784353059178, + "loss": 1.4712, + "theoretical_loss": 3.7345350584536137, + "tokens_seen": 852033536 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005994623871614844, + "loss": 1.5686, + "theoretical_loss": 3.7345071229358733, + "tokens_seen": 852099072 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005994463390170512, + "loss": 1.4892, + "theoretical_loss": 3.7344791901681487, + "tokens_seen": 852164608 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005994302908726178, + "loss": 1.4382, + "theoretical_loss": 3.734451260149957, + "tokens_seen": 852230144 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005994142427281846, + "loss": 1.4765, + "theoretical_loss": 3.7344233328808176, + "tokens_seen": 852295680 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005993981945837514, + "loss": 1.4608, + "theoretical_loss": 3.7343954083602466, + "tokens_seen": 852361216 + }, + { + "epoch": 0.26, + "learning_rate": 0.000599382146439318, + "loss": 1.5308, + "theoretical_loss": 3.734367486587764, + "tokens_seen": 852426752 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005993660982948848, + "loss": 1.443, + "theoretical_loss": 3.7343395675628868, + "tokens_seen": 852492288 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005993500501504514, + "loss": 1.3993, + "theoretical_loss": 3.734311651285134, + "tokens_seen": 852557824 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005993340020060182, + "loss": 1.4174, + "theoretical_loss": 3.7342837377540246, + "tokens_seen": 852623360 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005993179538615848, + "loss": 1.4546, + "theoretical_loss": 3.7342558269690764, + "tokens_seen": 852688896 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005993019057171516, + "loss": 1.3844, + "theoretical_loss": 3.7342279189298093, + "tokens_seen": 852754432 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005992858575727182, + "loss": 1.5718, + "theoretical_loss": 3.7342000136357405, + "tokens_seen": 852819968 + }, + { + "epoch": 0.26, + "learning_rate": 0.000599269809428285, + "loss": 1.471, + "theoretical_loss": 3.734172111086391, + "tokens_seen": 852885504 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005992537612838516, + "loss": 1.4977, + "theoretical_loss": 3.7341442112812784, + "tokens_seen": 852951040 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005992377131394184, + "loss": 1.4885, + "theoretical_loss": 3.7341163142199223, + "tokens_seen": 853016576 + }, + { + "epoch": 0.26, + "learning_rate": 0.000599221664994985, + "loss": 1.4161, + "theoretical_loss": 3.734088419901843, + "tokens_seen": 853082112 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005992056168505518, + "loss": 1.5682, + "theoretical_loss": 3.734060528326559, + "tokens_seen": 853147648 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005991895687061184, + "loss": 1.3744, + "theoretical_loss": 3.7340326394935905, + "tokens_seen": 853213184 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005991735205616852, + "loss": 1.4517, + "theoretical_loss": 3.734004753402457, + "tokens_seen": 853278720 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005991574724172518, + "loss": 1.4242, + "theoretical_loss": 3.7339768700526785, + "tokens_seen": 853344256 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005991414242728186, + "loss": 1.4914, + "theoretical_loss": 3.7339489894437756, + "tokens_seen": 853409792 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005991253761283852, + "loss": 1.4718, + "theoretical_loss": 3.733921111575267, + "tokens_seen": 853475328 + }, + { + "epoch": 0.26, + "learning_rate": 0.000599109327983952, + "loss": 1.4987, + "theoretical_loss": 3.7338932364466744, + "tokens_seen": 853540864 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005990932798395186, + "loss": 1.3815, + "theoretical_loss": 3.7338653640575172, + "tokens_seen": 853606400 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005990772316950854, + "loss": 1.4251, + "theoretical_loss": 3.7338374944073163, + "tokens_seen": 853671936 + }, + { + "epoch": 0.26, + "learning_rate": 0.000599061183550652, + "loss": 1.3971, + "theoretical_loss": 3.7338096274955923, + "tokens_seen": 853737472 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005990451354062187, + "loss": 1.5397, + "theoretical_loss": 3.733781763321866, + "tokens_seen": 853803008 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005990290872617854, + "loss": 1.5302, + "theoretical_loss": 3.7337539018856587, + "tokens_seen": 853868544 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005990130391173521, + "loss": 1.5057, + "theoretical_loss": 3.73372604318649, + "tokens_seen": 853934080 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005989969909729188, + "loss": 1.5476, + "theoretical_loss": 3.733698187223882, + "tokens_seen": 853999616 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005989809428284855, + "loss": 1.4402, + "theoretical_loss": 3.733670333997356, + "tokens_seen": 854065152 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005989648946840522, + "loss": 1.522, + "theoretical_loss": 3.733642483506433, + "tokens_seen": 854130688 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005989488465396189, + "loss": 1.3839, + "theoretical_loss": 3.7336146357506337, + "tokens_seen": 854196224 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005989327983951856, + "loss": 1.5091, + "theoretical_loss": 3.7335867907294817, + "tokens_seen": 854261760 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005989167502507523, + "loss": 1.4375, + "theoretical_loss": 3.733558948442497, + "tokens_seen": 854327296 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005989007021063189, + "loss": 1.4656, + "theoretical_loss": 3.733531108889202, + "tokens_seen": 854392832 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005988846539618857, + "loss": 1.378, + "theoretical_loss": 3.733503272069118, + "tokens_seen": 854458368 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005988686058174523, + "loss": 1.3449, + "theoretical_loss": 3.7334754379817685, + "tokens_seen": 854523904 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005988525576730191, + "loss": 1.4494, + "theoretical_loss": 3.733447606626674, + "tokens_seen": 854589440 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005988365095285857, + "loss": 1.4204, + "theoretical_loss": 3.7334197780033582, + "tokens_seen": 854654976 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005988204613841525, + "loss": 1.3279, + "theoretical_loss": 3.733391952111343, + "tokens_seen": 854720512 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005988044132397191, + "loss": 1.4711, + "theoretical_loss": 3.73336412895015, + "tokens_seen": 854786048 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005987883650952859, + "loss": 1.5319, + "theoretical_loss": 3.733336308519303, + "tokens_seen": 854851584 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005987723169508525, + "loss": 1.4948, + "theoretical_loss": 3.7333084908183247, + "tokens_seen": 854917120 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005987562688064193, + "loss": 1.5279, + "theoretical_loss": 3.7332806758467383, + "tokens_seen": 854982656 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005987402206619859, + "loss": 1.4637, + "theoretical_loss": 3.7332528636040654, + "tokens_seen": 855048192 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005987241725175527, + "loss": 1.4376, + "theoretical_loss": 3.7332250540898304, + "tokens_seen": 855113728 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005987081243731193, + "loss": 1.4623, + "theoretical_loss": 3.733197247303556, + "tokens_seen": 855179264 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.9947687387466431, + "objective/train/avg_token_score": 0.005231198854744434, + "objective/train/docs_used": 315806, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.3668560981750488, + "objective/train/original_loss": 1.3730368614196777, + "objective/train/theoretical_loss": 3.733169443244766, + "objective/train/tokens_used": 875704800, + "objective/train/unlikelihood_loss": 0.01583940163254738, + "theoretical_loss": 3.733169443244766, + "tokens_seen": 855244800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005986920762286861, + "loss": 1.4102, + "theoretical_loss": 3.733169443244766, + "tokens_seen": 855244800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005986760280842527, + "loss": 1.4533, + "theoretical_loss": 3.7331416419129835, + "tokens_seen": 855310336 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005986599799398195, + "loss": 1.4683, + "theoretical_loss": 3.7331138433077324, + "tokens_seen": 855375872 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005986439317953861, + "loss": 1.5117, + "theoretical_loss": 3.733086047428536, + "tokens_seen": 855441408 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005986278836509529, + "loss": 1.3317, + "theoretical_loss": 3.733058254274919, + "tokens_seen": 855506944 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005986118355065195, + "loss": 1.369, + "theoretical_loss": 3.7330304638464042, + "tokens_seen": 855572480 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005985957873620863, + "loss": 1.5234, + "theoretical_loss": 3.733002676142517, + "tokens_seen": 855638016 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005985797392176529, + "loss": 1.4096, + "theoretical_loss": 3.7329748911627805, + "tokens_seen": 855703552 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005985636910732197, + "loss": 1.2762, + "theoretical_loss": 3.7329471089067194, + "tokens_seen": 855769088 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005985476429287863, + "loss": 1.4186, + "theoretical_loss": 3.7329193293738587, + "tokens_seen": 855834624 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005985315947843531, + "loss": 1.4317, + "theoretical_loss": 3.7328915525637227, + "tokens_seen": 855900160 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005985155466399197, + "loss": 1.4509, + "theoretical_loss": 3.7328637784758354, + "tokens_seen": 855965696 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005984994984954865, + "loss": 1.5513, + "theoretical_loss": 3.7328360071097224, + "tokens_seen": 856031232 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005984834503510531, + "loss": 1.51, + "theoretical_loss": 3.7328082384649086, + "tokens_seen": 856096768 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005984674022066199, + "loss": 1.4412, + "theoretical_loss": 3.7327804725409184, + "tokens_seen": 856162304 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005984513540621865, + "loss": 1.4541, + "theoretical_loss": 3.7327527093372774, + "tokens_seen": 856227840 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005984353059177533, + "loss": 1.4363, + "theoretical_loss": 3.7327249488535106, + "tokens_seen": 856293376 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005984192577733199, + "loss": 1.3451, + "theoretical_loss": 3.7326971910891444, + "tokens_seen": 856358912 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005984032096288867, + "loss": 1.3963, + "theoretical_loss": 3.732669436043703, + "tokens_seen": 856424448 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005983871614844533, + "loss": 1.3936, + "theoretical_loss": 3.7326416837167127, + "tokens_seen": 856489984 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005983711133400201, + "loss": 1.3951, + "theoretical_loss": 3.7326139341076994, + "tokens_seen": 856555520 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005983550651955867, + "loss": 1.4279, + "theoretical_loss": 3.7325861872161887, + "tokens_seen": 856621056 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005983390170511535, + "loss": 1.3702, + "theoretical_loss": 3.732558443041707, + "tokens_seen": 856686592 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005983229689067201, + "loss": 1.4638, + "theoretical_loss": 3.73253070158378, + "tokens_seen": 856752128 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005983069207622869, + "loss": 1.4448, + "theoretical_loss": 3.732502962841934, + "tokens_seen": 856817664 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005982908726178535, + "loss": 1.3498, + "theoretical_loss": 3.732475226815695, + "tokens_seen": 856883200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005982748244734203, + "loss": 1.4543, + "theoretical_loss": 3.7324474935045906, + "tokens_seen": 856948736 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005982587763289869, + "loss": 1.4434, + "theoretical_loss": 3.7324197629081466, + "tokens_seen": 857014272 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005982427281845537, + "loss": 1.443, + "theoretical_loss": 3.7323920350258897, + "tokens_seen": 857079808 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005982266800401203, + "loss": 1.4062, + "theoretical_loss": 3.732364309857347, + "tokens_seen": 857145344 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005982106318956871, + "loss": 1.4251, + "theoretical_loss": 3.7323365874020453, + "tokens_seen": 857210880 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005981945837512539, + "loss": 1.4266, + "theoretical_loss": 3.732308867659511, + "tokens_seen": 857276416 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005981785356068205, + "loss": 1.3779, + "theoretical_loss": 3.732281150629273, + "tokens_seen": 857341952 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005981624874623873, + "loss": 1.5558, + "theoretical_loss": 3.732253436310857, + "tokens_seen": 857407488 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005981464393179539, + "loss": 1.4401, + "theoretical_loss": 3.732225724703791, + "tokens_seen": 857473024 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005981303911735207, + "loss": 1.4443, + "theoretical_loss": 3.7321980158076027, + "tokens_seen": 857538560 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005981143430290873, + "loss": 1.4765, + "theoretical_loss": 3.7321703096218197, + "tokens_seen": 857604096 + }, + { + "epoch": 0.26, + "learning_rate": 0.000598098294884654, + "loss": 1.4031, + "theoretical_loss": 3.7321426061459704, + "tokens_seen": 857669632 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005980822467402207, + "loss": 1.5353, + "theoretical_loss": 3.7321149053795812, + "tokens_seen": 857735168 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005980661985957875, + "loss": 1.4354, + "theoretical_loss": 3.7320872073221807, + "tokens_seen": 857800704 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005980501504513541, + "loss": 1.4582, + "theoretical_loss": 3.732059511973298, + "tokens_seen": 857866240 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005980341023069209, + "loss": 1.3962, + "theoretical_loss": 3.7320318193324598, + "tokens_seen": 857931776 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005980180541624875, + "loss": 1.435, + "theoretical_loss": 3.732004129399196, + "tokens_seen": 857997312 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005980020060180543, + "loss": 1.462, + "theoretical_loss": 3.731976442173034, + "tokens_seen": 858062848 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005979859578736209, + "loss": 1.4472, + "theoretical_loss": 3.731948757653503, + "tokens_seen": 858128384 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005979699097291877, + "loss": 1.3444, + "theoretical_loss": 3.7319210758401318, + "tokens_seen": 858193920 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005979538615847543, + "loss": 1.379, + "theoretical_loss": 3.7318933967324486, + "tokens_seen": 858259456 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005979378134403211, + "loss": 1.5041, + "theoretical_loss": 3.731865720329983, + "tokens_seen": 858324992 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005979217652958877, + "loss": 1.4912, + "theoretical_loss": 3.731838046632264, + "tokens_seen": 858390528 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005979057171514545, + "loss": 1.4351, + "theoretical_loss": 3.73181037563882, + "tokens_seen": 858456064 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.9928128719329834, + "objective/train/avg_token_score": 0.007187159266322851, + "objective/train/docs_used": 317142, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4648841619491577, + "objective/train/original_loss": 1.476088523864746, + "objective/train/theoretical_loss": 3.7317827073491814, + "objective/train/tokens_used": 878981600, + "objective/train/unlikelihood_loss": 0.017246056348085403, + "theoretical_loss": 3.7317827073491814, + "tokens_seen": 858521600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005978896690070211, + "loss": 1.4869, + "theoretical_loss": 3.7317827073491814, + "tokens_seen": 858521600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005978736208625879, + "loss": 1.4317, + "theoretical_loss": 3.7317550417628773, + "tokens_seen": 858587136 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005978575727181545, + "loss": 1.5574, + "theoretical_loss": 3.731727378879437, + "tokens_seen": 858652672 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005978415245737213, + "loss": 1.3857, + "theoretical_loss": 3.7316997186983905, + "tokens_seen": 858718208 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005978254764292879, + "loss": 1.3748, + "theoretical_loss": 3.7316720612192666, + "tokens_seen": 858783744 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005978094282848547, + "loss": 1.3847, + "theoretical_loss": 3.731644406441597, + "tokens_seen": 858849280 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005977933801404213, + "loss": 1.5901, + "theoretical_loss": 3.7316167543649104, + "tokens_seen": 858914816 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005977773319959881, + "loss": 1.4516, + "theoretical_loss": 3.7315891049887373, + "tokens_seen": 858980352 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005977612838515547, + "loss": 1.5876, + "theoretical_loss": 3.7315614583126084, + "tokens_seen": 859045888 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005977452357071215, + "loss": 1.5006, + "theoretical_loss": 3.731533814336053, + "tokens_seen": 859111424 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005977291875626881, + "loss": 1.4244, + "theoretical_loss": 3.7315061730586025, + "tokens_seen": 859176960 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005977131394182549, + "loss": 1.5485, + "theoretical_loss": 3.7314785344797867, + "tokens_seen": 859242496 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005976970912738215, + "loss": 1.5082, + "theoretical_loss": 3.731450898599138, + "tokens_seen": 859308032 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005976810431293883, + "loss": 1.4516, + "theoretical_loss": 3.731423265416185, + "tokens_seen": 859373568 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005976649949849549, + "loss": 1.4304, + "theoretical_loss": 3.7313956349304602, + "tokens_seen": 859439104 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005976489468405216, + "loss": 1.4257, + "theoretical_loss": 3.7313680071414943, + "tokens_seen": 859504640 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005976328986960883, + "loss": 1.4072, + "theoretical_loss": 3.7313403820488187, + "tokens_seen": 859570176 + }, + { + "epoch": 0.26, + "learning_rate": 0.000597616850551655, + "loss": 1.4018, + "theoretical_loss": 3.7313127596519644, + "tokens_seen": 859635712 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005976008024072217, + "loss": 1.4004, + "theoretical_loss": 3.7312851399504625, + "tokens_seen": 859701248 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005975847542627884, + "loss": 1.5806, + "theoretical_loss": 3.731257522943846, + "tokens_seen": 859766784 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005975687061183551, + "loss": 1.4569, + "theoretical_loss": 3.7312299086316445, + "tokens_seen": 859832320 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005975526579739218, + "loss": 1.5106, + "theoretical_loss": 3.731202297013391, + "tokens_seen": 859897856 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005975366098294885, + "loss": 1.4142, + "theoretical_loss": 3.7311746880886174, + "tokens_seen": 859963392 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005975205616850552, + "loss": 1.4353, + "theoretical_loss": 3.7311470818568555, + "tokens_seen": 860028928 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005975045135406218, + "loss": 1.4482, + "theoretical_loss": 3.7311194783176376, + "tokens_seen": 860094464 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005974884653961886, + "loss": 1.5381, + "theoretical_loss": 3.7310918774704955, + "tokens_seen": 860160000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005974724172517552, + "loss": 1.5225, + "theoretical_loss": 3.731064279314962, + "tokens_seen": 860225536 + }, + { + "epoch": 0.26, + "learning_rate": 0.000597456369107322, + "loss": 1.3726, + "theoretical_loss": 3.7310366838505704, + "tokens_seen": 860291072 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005974403209628886, + "loss": 1.4463, + "theoretical_loss": 3.7310090910768516, + "tokens_seen": 860356608 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005974242728184554, + "loss": 1.3652, + "theoretical_loss": 3.730981500993339, + "tokens_seen": 860422144 + }, + { + "epoch": 0.26, + "learning_rate": 0.000597408224674022, + "loss": 1.5188, + "theoretical_loss": 3.7309539135995657, + "tokens_seen": 860487680 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005973921765295888, + "loss": 1.3327, + "theoretical_loss": 3.7309263288950647, + "tokens_seen": 860553216 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005973761283851554, + "loss": 1.4179, + "theoretical_loss": 3.730898746879369, + "tokens_seen": 860618752 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005973600802407222, + "loss": 1.6437, + "theoretical_loss": 3.730871167552011, + "tokens_seen": 860684288 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005973440320962888, + "loss": 1.4572, + "theoretical_loss": 3.7308435909125253, + "tokens_seen": 860749824 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005973279839518556, + "loss": 1.5146, + "theoretical_loss": 3.7308160169604445, + "tokens_seen": 860815360 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005973119358074222, + "loss": 1.5802, + "theoretical_loss": 3.7307884456953024, + "tokens_seen": 860880896 + }, + { + "epoch": 0.26, + "learning_rate": 0.000597295887662989, + "loss": 1.3758, + "theoretical_loss": 3.7307608771166327, + "tokens_seen": 860946432 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005972798395185556, + "loss": 1.4933, + "theoretical_loss": 3.730733311223969, + "tokens_seen": 861011968 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005972637913741224, + "loss": 1.4622, + "theoretical_loss": 3.7307057480168453, + "tokens_seen": 861077504 + }, + { + "epoch": 0.26, + "learning_rate": 0.000597247743229689, + "loss": 1.4467, + "theoretical_loss": 3.730678187494796, + "tokens_seen": 861143040 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005972316950852558, + "loss": 1.3645, + "theoretical_loss": 3.7306506296573536, + "tokens_seen": 861208576 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005972156469408224, + "loss": 1.4365, + "theoretical_loss": 3.7306230745040545, + "tokens_seen": 861274112 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005971995987963892, + "loss": 1.4839, + "theoretical_loss": 3.730595522034432, + "tokens_seen": 861339648 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005971835506519558, + "loss": 1.349, + "theoretical_loss": 3.73056797224802, + "tokens_seen": 861405184 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005971675025075226, + "loss": 1.4636, + "theoretical_loss": 3.730540425144354, + "tokens_seen": 861470720 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005971514543630892, + "loss": 1.4825, + "theoretical_loss": 3.7305128807229684, + "tokens_seen": 861536256 + }, + { + "epoch": 0.26, + "learning_rate": 0.000597135406218656, + "loss": 1.459, + "theoretical_loss": 3.730485338983398, + "tokens_seen": 861601792 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005971193580742226, + "loss": 1.3991, + "theoretical_loss": 3.730457799925178, + "tokens_seen": 861667328 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005971033099297894, + "loss": 1.4055, + "theoretical_loss": 3.7304302635478432, + "tokens_seen": 861732864 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.9923645853996277, + "objective/train/avg_token_score": 0.007635429501533508, + "objective/train/docs_used": 318349, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.3821438550949097, + "objective/train/original_loss": 1.3942036628723145, + "objective/train/theoretical_loss": 3.730402729850929, + "objective/train/tokens_used": 882258400, + "objective/train/unlikelihood_loss": 0.018785608932375908, + "theoretical_loss": 3.730402729850929, + "tokens_seen": 861798400 + }, + { + "epoch": 0.26, + "learning_rate": 0.000597087261785356, + "loss": 1.4888, + "theoretical_loss": 3.730402729850929, + "tokens_seen": 861798400 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005970712136409228, + "loss": 1.5867, + "theoretical_loss": 3.7303751988339697, + "tokens_seen": 861863936 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005970551654964894, + "loss": 1.384, + "theoretical_loss": 3.730347670496502, + "tokens_seen": 861929472 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005970391173520562, + "loss": 1.4613, + "theoretical_loss": 3.7303201448380605, + "tokens_seen": 861995008 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005970230692076228, + "loss": 1.4707, + "theoretical_loss": 3.7302926218581813, + "tokens_seen": 862060544 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005970070210631896, + "loss": 1.3779, + "theoretical_loss": 3.730265101556401, + "tokens_seen": 862126080 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005969909729187563, + "loss": 1.3698, + "theoretical_loss": 3.7302375839322535, + "tokens_seen": 862191616 + }, + { + "epoch": 0.26, + "learning_rate": 0.000596974924774323, + "loss": 1.4597, + "theoretical_loss": 3.7302100689852757, + "tokens_seen": 862257152 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005969588766298897, + "loss": 1.4124, + "theoretical_loss": 3.730182556715004, + "tokens_seen": 862322688 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005969428284854564, + "loss": 1.5272, + "theoretical_loss": 3.7301550471209746, + "tokens_seen": 862388224 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005969267803410231, + "loss": 1.542, + "theoretical_loss": 3.730127540202723, + "tokens_seen": 862453760 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005969107321965898, + "loss": 1.4353, + "theoretical_loss": 3.730100035959787, + "tokens_seen": 862519296 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005968946840521565, + "loss": 1.5112, + "theoretical_loss": 3.730072534391702, + "tokens_seen": 862584832 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005968786359077232, + "loss": 1.455, + "theoretical_loss": 3.730045035498005, + "tokens_seen": 862650368 + }, + { + "epoch": 0.26, + "learning_rate": 0.00059686258776329, + "loss": 1.4674, + "theoretical_loss": 3.730017539278233, + "tokens_seen": 862715904 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005968465396188566, + "loss": 1.6167, + "theoretical_loss": 3.729990045731922, + "tokens_seen": 862781440 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005968304914744233, + "loss": 1.4353, + "theoretical_loss": 3.7299625548586106, + "tokens_seen": 862846976 + }, + { + "epoch": 0.26, + "learning_rate": 0.00059681444332999, + "loss": 1.4323, + "theoretical_loss": 3.729935066657835, + "tokens_seen": 862912512 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005967983951855568, + "loss": 1.5204, + "theoretical_loss": 3.7299075811291322, + "tokens_seen": 862978048 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005967823470411234, + "loss": 1.3828, + "theoretical_loss": 3.7298800982720404, + "tokens_seen": 863043584 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005967662988966902, + "loss": 1.5367, + "theoretical_loss": 3.7298526180860962, + "tokens_seen": 863109120 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005967502507522568, + "loss": 1.3417, + "theoretical_loss": 3.7298251405708376, + "tokens_seen": 863174656 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005967342026078236, + "loss": 1.3952, + "theoretical_loss": 3.729797665725802, + "tokens_seen": 863240192 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005967181544633902, + "loss": 1.4639, + "theoretical_loss": 3.729770193550528, + "tokens_seen": 863305728 + }, + { + "epoch": 0.26, + "learning_rate": 0.000596702106318957, + "loss": 1.3545, + "theoretical_loss": 3.729742724044553, + "tokens_seen": 863371264 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005966860581745236, + "loss": 1.4322, + "theoretical_loss": 3.7297152572074146, + "tokens_seen": 863436800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005966700100300904, + "loss": 1.4592, + "theoretical_loss": 3.729687793038652, + "tokens_seen": 863502336 + }, + { + "epoch": 0.26, + "learning_rate": 0.000596653961885657, + "loss": 1.474, + "theoretical_loss": 3.7296603315378025, + "tokens_seen": 863567872 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005966379137412238, + "loss": 1.4369, + "theoretical_loss": 3.7296328727044052, + "tokens_seen": 863633408 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005966218655967904, + "loss": 1.444, + "theoretical_loss": 3.729605416537998, + "tokens_seen": 863698944 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005966058174523572, + "loss": 1.4806, + "theoretical_loss": 3.7295779630381203, + "tokens_seen": 863764480 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005965897693079238, + "loss": 1.4478, + "theoretical_loss": 3.7295505122043098, + "tokens_seen": 863830016 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005965737211634906, + "loss": 1.4, + "theoretical_loss": 3.7295230640361066, + "tokens_seen": 863895552 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005965576730190572, + "loss": 1.4652, + "theoretical_loss": 3.729495618533049, + "tokens_seen": 863961088 + }, + { + "epoch": 0.26, + "learning_rate": 0.000596541624874624, + "loss": 1.4443, + "theoretical_loss": 3.7294681756946755, + "tokens_seen": 864026624 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005965255767301906, + "loss": 1.4232, + "theoretical_loss": 3.7294407355205257, + "tokens_seen": 864092160 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005965095285857574, + "loss": 1.5404, + "theoretical_loss": 3.72941329801014, + "tokens_seen": 864157696 + }, + { + "epoch": 0.26, + "learning_rate": 0.000596493480441324, + "loss": 1.5176, + "theoretical_loss": 3.7293858631630563, + "tokens_seen": 864223232 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005964774322968908, + "loss": 1.442, + "theoretical_loss": 3.7293584309788157, + "tokens_seen": 864288768 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005964613841524574, + "loss": 1.4654, + "theoretical_loss": 3.729331001456956, + "tokens_seen": 864354304 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005964453360080242, + "loss": 1.4644, + "theoretical_loss": 3.7293035745970182, + "tokens_seen": 864419840 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005964292878635908, + "loss": 1.4094, + "theoretical_loss": 3.729276150398542, + "tokens_seen": 864485376 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005964132397191576, + "loss": 1.4722, + "theoretical_loss": 3.7292487288610676, + "tokens_seen": 864550912 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005963971915747242, + "loss": 1.4102, + "theoretical_loss": 3.7292213099841343, + "tokens_seen": 864616448 + }, + { + "epoch": 0.26, + "learning_rate": 0.000596381143430291, + "loss": 1.4834, + "theoretical_loss": 3.7291938937672833, + "tokens_seen": 864681984 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005963650952858576, + "loss": 1.405, + "theoretical_loss": 3.729166480210054, + "tokens_seen": 864747520 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005963490471414244, + "loss": 1.5682, + "theoretical_loss": 3.729139069311988, + "tokens_seen": 864813056 + }, + { + "epoch": 0.26, + "learning_rate": 0.000596332998996991, + "loss": 1.3194, + "theoretical_loss": 3.729111661072625, + "tokens_seen": 864878592 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005963169508525578, + "loss": 1.4435, + "theoretical_loss": 3.729084255491506, + "tokens_seen": 864944128 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005963009027081244, + "loss": 1.4106, + "theoretical_loss": 3.7290568525681715, + "tokens_seen": 865009664 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.9909172654151917, + "objective/train/avg_token_score": 0.009082701057195663, + "objective/train/docs_used": 319421, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.3765337467193604, + "objective/train/original_loss": 1.3915152549743652, + "objective/train/theoretical_loss": 3.7290294523021625, + "objective/train/tokens_used": 885535200, + "objective/train/unlikelihood_loss": 0.02201116643846035, + "theoretical_loss": 3.7290294523021625, + "tokens_seen": 865075200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005962848545636912, + "loss": 1.4209, + "theoretical_loss": 3.7290294523021625, + "tokens_seen": 865075200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005962688064192578, + "loss": 1.394, + "theoretical_loss": 3.729002054693021, + "tokens_seen": 865140736 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005962527582748246, + "loss": 1.5502, + "theoretical_loss": 3.7289746597402864, + "tokens_seen": 865206272 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005962367101303912, + "loss": 1.5783, + "theoretical_loss": 3.7289472674435014, + "tokens_seen": 865271808 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005962206619859579, + "loss": 1.4583, + "theoretical_loss": 3.728919877802207, + "tokens_seen": 865337344 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005962046138415246, + "loss": 1.499, + "theoretical_loss": 3.728892490815944, + "tokens_seen": 865402880 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005961885656970913, + "loss": 1.4036, + "theoretical_loss": 3.7288651064842546, + "tokens_seen": 865468416 + }, + { + "epoch": 0.26, + "learning_rate": 0.000596172517552658, + "loss": 1.5353, + "theoretical_loss": 3.728837724806681, + "tokens_seen": 865533952 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005961564694082247, + "loss": 1.4795, + "theoretical_loss": 3.7288103457827635, + "tokens_seen": 865599488 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005961404212637914, + "loss": 1.5156, + "theoretical_loss": 3.7287829694120456, + "tokens_seen": 865665024 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005961243731193581, + "loss": 1.3214, + "theoretical_loss": 3.728755595694069, + "tokens_seen": 865730560 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005961083249749248, + "loss": 1.4938, + "theoretical_loss": 3.7287282246283757, + "tokens_seen": 865796096 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005960922768304915, + "loss": 1.5276, + "theoretical_loss": 3.728700856214507, + "tokens_seen": 865861632 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005960762286860581, + "loss": 1.4646, + "theoretical_loss": 3.7286734904520076, + "tokens_seen": 865927168 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005960601805416249, + "loss": 1.5004, + "theoretical_loss": 3.7286461273404177, + "tokens_seen": 865992704 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005960441323971915, + "loss": 1.4144, + "theoretical_loss": 3.728618766879281, + "tokens_seen": 866058240 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005960280842527583, + "loss": 1.4097, + "theoretical_loss": 3.72859140906814, + "tokens_seen": 866123776 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005960120361083249, + "loss": 1.3421, + "theoretical_loss": 3.7285640539065374, + "tokens_seen": 866189312 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005959959879638917, + "loss": 1.4539, + "theoretical_loss": 3.7285367013940167, + "tokens_seen": 866254848 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005959799398194583, + "loss": 1.4712, + "theoretical_loss": 3.7285093515301213, + "tokens_seen": 866320384 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005959638916750251, + "loss": 1.4966, + "theoretical_loss": 3.728482004314393, + "tokens_seen": 866385920 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005959478435305917, + "loss": 1.3134, + "theoretical_loss": 3.728454659746376, + "tokens_seen": 866451456 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005959317953861585, + "loss": 1.4446, + "theoretical_loss": 3.7284273178256133, + "tokens_seen": 866516992 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005959157472417251, + "loss": 1.4733, + "theoretical_loss": 3.728399978551649, + "tokens_seen": 866582528 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005958996990972919, + "loss": 1.4922, + "theoretical_loss": 3.7283726419240266, + "tokens_seen": 866648064 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005958836509528585, + "loss": 1.5208, + "theoretical_loss": 3.7283453079422895, + "tokens_seen": 866713600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005958676028084253, + "loss": 1.4221, + "theoretical_loss": 3.7283179766059824, + "tokens_seen": 866779136 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005958515546639919, + "loss": 1.465, + "theoretical_loss": 3.7282906479146476, + "tokens_seen": 866844672 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005958355065195587, + "loss": 1.4556, + "theoretical_loss": 3.728263321867831, + "tokens_seen": 866910208 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005958194583751253, + "loss": 1.3473, + "theoretical_loss": 3.7282359984650757, + "tokens_seen": 866975744 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005958034102306921, + "loss": 1.5343, + "theoretical_loss": 3.7282086777059265, + "tokens_seen": 867041280 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005957873620862588, + "loss": 1.5337, + "theoretical_loss": 3.728181359589928, + "tokens_seen": 867106816 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005957713139418255, + "loss": 1.4382, + "theoretical_loss": 3.728154044116624, + "tokens_seen": 867172352 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005957552657973922, + "loss": 1.3869, + "theoretical_loss": 3.72812673128556, + "tokens_seen": 867237888 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005957392176529589, + "loss": 1.4096, + "theoretical_loss": 3.728099421096281, + "tokens_seen": 867303424 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005957231695085256, + "loss": 1.5031, + "theoretical_loss": 3.72807211354833, + "tokens_seen": 867368960 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005957071213640923, + "loss": 1.3986, + "theoretical_loss": 3.728044808641254, + "tokens_seen": 867434496 + }, + { + "epoch": 0.26, + "learning_rate": 0.000595691073219659, + "loss": 1.6083, + "theoretical_loss": 3.7280175063745973, + "tokens_seen": 867500032 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005956750250752257, + "loss": 1.4245, + "theoretical_loss": 3.727990206747905, + "tokens_seen": 867565568 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005956589769307924, + "loss": 1.6172, + "theoretical_loss": 3.7279629097607234, + "tokens_seen": 867631104 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005956429287863591, + "loss": 1.4636, + "theoretical_loss": 3.7279356154125964, + "tokens_seen": 867696640 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005956268806419258, + "loss": 1.4798, + "theoretical_loss": 3.727908323703071, + "tokens_seen": 867762176 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005956108324974925, + "loss": 1.511, + "theoretical_loss": 3.7278810346316913, + "tokens_seen": 867827712 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005955947843530592, + "loss": 1.3992, + "theoretical_loss": 3.7278537481980045, + "tokens_seen": 867893248 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005955787362086259, + "loss": 1.5767, + "theoretical_loss": 3.7278264644015566, + "tokens_seen": 867958784 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005955626880641926, + "loss": 1.5273, + "theoretical_loss": 3.727799183241893, + "tokens_seen": 868024320 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005955466399197593, + "loss": 1.4512, + "theoretical_loss": 3.7277719047185593, + "tokens_seen": 868089856 + }, + { + "epoch": 0.26, + "learning_rate": 0.000595530591775326, + "loss": 1.3943, + "theoretical_loss": 3.7277446288311022, + "tokens_seen": 868155392 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005955145436308927, + "loss": 1.5259, + "theoretical_loss": 3.7277173555790686, + "tokens_seen": 868220928 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005954984954864594, + "loss": 1.5021, + "theoretical_loss": 3.7276900849620045, + "tokens_seen": 868286464 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.9922113418579102, + "objective/train/avg_token_score": 0.007788582239300013, + "objective/train/docs_used": 320668, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4595539569854736, + "objective/train/original_loss": 1.470514178276062, + "objective/train/theoretical_loss": 3.727662816979456, + "objective/train/tokens_used": 888812000, + "objective/train/unlikelihood_loss": 0.02342648059129715, + "theoretical_loss": 3.727662816979456, + "tokens_seen": 868352000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005954824473420261, + "loss": 1.4654, + "theoretical_loss": 3.727662816979456, + "tokens_seen": 868352000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005954663991975928, + "loss": 1.4691, + "theoretical_loss": 3.727635551630971, + "tokens_seen": 868417536 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005954503510531595, + "loss": 1.5094, + "theoretical_loss": 3.7276082889160955, + "tokens_seen": 868483072 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005954343029087263, + "loss": 1.4088, + "theoretical_loss": 3.727581028834376, + "tokens_seen": 868548608 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005954182547642929, + "loss": 1.5212, + "theoretical_loss": 3.7275537713853604, + "tokens_seen": 868614144 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005954022066198597, + "loss": 1.4087, + "theoretical_loss": 3.727526516568595, + "tokens_seen": 868679680 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005953861584754263, + "loss": 1.5827, + "theoretical_loss": 3.727499264383628, + "tokens_seen": 868745216 + }, + { + "epoch": 0.26, + "learning_rate": 0.000595370110330993, + "loss": 1.3934, + "theoretical_loss": 3.7274720148300062, + "tokens_seen": 868810752 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005953540621865597, + "loss": 1.4348, + "theoretical_loss": 3.727444767907277, + "tokens_seen": 868876288 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005953380140421265, + "loss": 1.4906, + "theoretical_loss": 3.7274175236149887, + "tokens_seen": 868941824 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005953219658976931, + "loss": 1.428, + "theoretical_loss": 3.7273902819526876, + "tokens_seen": 869007360 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005953059177532599, + "loss": 1.495, + "theoretical_loss": 3.7273630429199223, + "tokens_seen": 869072896 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005952898696088265, + "loss": 1.3263, + "theoretical_loss": 3.727335806516241, + "tokens_seen": 869138432 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005952738214643933, + "loss": 1.447, + "theoretical_loss": 3.727308572741191, + "tokens_seen": 869203968 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005952577733199599, + "loss": 1.4787, + "theoretical_loss": 3.7272813415943222, + "tokens_seen": 869269504 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005952417251755267, + "loss": 1.5028, + "theoretical_loss": 3.7272541130751806, + "tokens_seen": 869335040 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005952256770310933, + "loss": 1.3893, + "theoretical_loss": 3.727226887183315, + "tokens_seen": 869400576 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005952096288866601, + "loss": 1.5089, + "theoretical_loss": 3.7271996639182756, + "tokens_seen": 869466112 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005951935807422267, + "loss": 1.4372, + "theoretical_loss": 3.727172443279609, + "tokens_seen": 869531648 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005951775325977935, + "loss": 1.5586, + "theoretical_loss": 3.727145225266865, + "tokens_seen": 869597184 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005951614844533601, + "loss": 1.5654, + "theoretical_loss": 3.727118009879592, + "tokens_seen": 869662720 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005951454363089269, + "loss": 1.4221, + "theoretical_loss": 3.727090797117339, + "tokens_seen": 869728256 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005951293881644935, + "loss": 1.3289, + "theoretical_loss": 3.7270635869796553, + "tokens_seen": 869793792 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005951133400200603, + "loss": 1.4681, + "theoretical_loss": 3.7270363794660892, + "tokens_seen": 869859328 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005950972918756269, + "loss": 1.5043, + "theoretical_loss": 3.727009174576191, + "tokens_seen": 869924864 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005950812437311937, + "loss": 1.3267, + "theoretical_loss": 3.7269819723095097, + "tokens_seen": 869990400 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005950651955867603, + "loss": 1.453, + "theoretical_loss": 3.7269547726655947, + "tokens_seen": 870055936 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005950491474423271, + "loss": 1.5371, + "theoretical_loss": 3.7269275756439955, + "tokens_seen": 870121472 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005950330992978937, + "loss": 1.5246, + "theoretical_loss": 3.7269003812442616, + "tokens_seen": 870187008 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005950170511534605, + "loss": 1.4916, + "theoretical_loss": 3.7268731894659437, + "tokens_seen": 870252544 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005950010030090271, + "loss": 1.6228, + "theoretical_loss": 3.726846000308591, + "tokens_seen": 870318080 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005949849548645939, + "loss": 1.4817, + "theoretical_loss": 3.7268188137717537, + "tokens_seen": 870383616 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005949689067201605, + "loss": 1.428, + "theoretical_loss": 3.7267916298549815, + "tokens_seen": 870449152 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005949528585757273, + "loss": 1.5386, + "theoretical_loss": 3.7267644485578257, + "tokens_seen": 870514688 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005949368104312939, + "loss": 1.4531, + "theoretical_loss": 3.726737269879836, + "tokens_seen": 870580224 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005949207622868607, + "loss": 1.4732, + "theoretical_loss": 3.7267100938205626, + "tokens_seen": 870645760 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005949047141424273, + "loss": 1.5977, + "theoretical_loss": 3.726682920379556, + "tokens_seen": 870711296 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005948886659979941, + "loss": 1.2929, + "theoretical_loss": 3.726655749556368, + "tokens_seen": 870776832 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005948726178535607, + "loss": 1.3988, + "theoretical_loss": 3.726628581350549, + "tokens_seen": 870842368 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005948565697091275, + "loss": 1.4988, + "theoretical_loss": 3.7266014157616487, + "tokens_seen": 870907904 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005948405215646941, + "loss": 1.4727, + "theoretical_loss": 3.72657425278922, + "tokens_seen": 870973440 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005948244734202608, + "loss": 1.3444, + "theoretical_loss": 3.7265470924328126, + "tokens_seen": 871038976 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005948084252758275, + "loss": 1.4233, + "theoretical_loss": 3.7265199346919786, + "tokens_seen": 871104512 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005947923771313942, + "loss": 1.5207, + "theoretical_loss": 3.7264927795662692, + "tokens_seen": 871170048 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005947763289869609, + "loss": 1.5665, + "theoretical_loss": 3.7264656270552354, + "tokens_seen": 871235584 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005947602808425276, + "loss": 1.4216, + "theoretical_loss": 3.72643847715843, + "tokens_seen": 871301120 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005947442326980943, + "loss": 1.4129, + "theoretical_loss": 3.726411329875403, + "tokens_seen": 871366656 + }, + { + "epoch": 0.26, + "learning_rate": 0.000594728184553661, + "loss": 1.426, + "theoretical_loss": 3.726384185205707, + "tokens_seen": 871432192 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005947121364092277, + "loss": 1.4515, + "theoretical_loss": 3.7263570431488944, + "tokens_seen": 871497728 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005946960882647944, + "loss": 1.5683, + "theoretical_loss": 3.7263299037045168, + "tokens_seen": 871563264 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.9908714294433594, + "objective/train/avg_token_score": 0.009128576144576073, + "objective/train/docs_used": 321822, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6108325719833374, + "objective/train/original_loss": 1.6263738870620728, + "objective/train/theoretical_loss": 3.7263027668721262, + "objective/train/tokens_used": 892088800, + "objective/train/unlikelihood_loss": 0.01899661496281624, + "theoretical_loss": 3.7263027668721262, + "tokens_seen": 871628800 + }, + { + "epoch": 0.26, + "learning_rate": 0.000594680040120361, + "loss": 1.5437, + "theoretical_loss": 3.7263027668721262, + "tokens_seen": 871628800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005946639919759278, + "loss": 1.5198, + "theoretical_loss": 3.7262756326512747, + "tokens_seen": 871694336 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005946479438314944, + "loss": 1.3667, + "theoretical_loss": 3.7262485010415154, + "tokens_seen": 871759872 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005946318956870612, + "loss": 1.3983, + "theoretical_loss": 3.7262213720424002, + "tokens_seen": 871825408 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005946158475426278, + "loss": 1.4619, + "theoretical_loss": 3.726194245653482, + "tokens_seen": 871890944 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005945997993981946, + "loss": 1.5759, + "theoretical_loss": 3.7261671218743135, + "tokens_seen": 871956480 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005945837512537613, + "loss": 1.3971, + "theoretical_loss": 3.7261400007044467, + "tokens_seen": 872022016 + }, + { + "epoch": 0.26, + "learning_rate": 0.000594567703109328, + "loss": 1.4355, + "theoretical_loss": 3.7261128821434353, + "tokens_seen": 872087552 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005945516549648947, + "loss": 1.4323, + "theoretical_loss": 3.7260857661908324, + "tokens_seen": 872153088 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005945356068204614, + "loss": 1.5296, + "theoretical_loss": 3.726058652846191, + "tokens_seen": 872218624 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005945195586760281, + "loss": 1.3913, + "theoretical_loss": 3.726031542109064, + "tokens_seen": 872284160 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005945035105315948, + "loss": 1.3934, + "theoretical_loss": 3.7260044339790053, + "tokens_seen": 872349696 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005944874623871615, + "loss": 1.4887, + "theoretical_loss": 3.7259773284555675, + "tokens_seen": 872415232 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005944714142427282, + "loss": 1.4301, + "theoretical_loss": 3.7259502255383055, + "tokens_seen": 872480768 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005944553660982949, + "loss": 1.4652, + "theoretical_loss": 3.7259231252267715, + "tokens_seen": 872546304 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005944393179538616, + "loss": 1.4667, + "theoretical_loss": 3.72589602752052, + "tokens_seen": 872611840 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005944232698094283, + "loss": 1.508, + "theoretical_loss": 3.7258689324191057, + "tokens_seen": 872677376 + }, + { + "epoch": 0.26, + "learning_rate": 0.000594407221664995, + "loss": 1.4777, + "theoretical_loss": 3.725841839922081, + "tokens_seen": 872742912 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005943911735205617, + "loss": 1.4937, + "theoretical_loss": 3.7258147500290013, + "tokens_seen": 872808448 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005943751253761284, + "loss": 1.4687, + "theoretical_loss": 3.72578766273942, + "tokens_seen": 872873984 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005943590772316951, + "loss": 1.4306, + "theoretical_loss": 3.725760578052892, + "tokens_seen": 872939520 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005943430290872618, + "loss": 1.4281, + "theoretical_loss": 3.725733495968972, + "tokens_seen": 873005056 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005943269809428285, + "loss": 1.5431, + "theoretical_loss": 3.7257064164872142, + "tokens_seen": 873070592 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005943109327983952, + "loss": 1.4218, + "theoretical_loss": 3.7256793396071726, + "tokens_seen": 873136128 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005942948846539619, + "loss": 1.4016, + "theoretical_loss": 3.725652265328403, + "tokens_seen": 873201664 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005942788365095286, + "loss": 1.4936, + "theoretical_loss": 3.7256251936504596, + "tokens_seen": 873267200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005942627883650953, + "loss": 1.566, + "theoretical_loss": 3.725598124572898, + "tokens_seen": 873332736 + }, + { + "epoch": 0.26, + "learning_rate": 0.000594246740220662, + "loss": 1.4309, + "theoretical_loss": 3.725571058095273, + "tokens_seen": 873398272 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005942306920762287, + "loss": 1.4076, + "theoretical_loss": 3.7255439942171398, + "tokens_seen": 873463808 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005942146439317954, + "loss": 1.4179, + "theoretical_loss": 3.7255169329380537, + "tokens_seen": 873529344 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005941985957873621, + "loss": 1.5831, + "theoretical_loss": 3.72548987425757, + "tokens_seen": 873594880 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005941825476429288, + "loss": 1.4512, + "theoretical_loss": 3.725462818175245, + "tokens_seen": 873660416 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005941664994984955, + "loss": 1.3399, + "theoretical_loss": 3.7254357646906335, + "tokens_seen": 873725952 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005941504513540622, + "loss": 1.4345, + "theoretical_loss": 3.7254087138032914, + "tokens_seen": 873791488 + }, + { + "epoch": 0.26, + "learning_rate": 0.000594134403209629, + "loss": 1.4836, + "theoretical_loss": 3.725381665512775, + "tokens_seen": 873857024 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005941183550651956, + "loss": 1.4261, + "theoretical_loss": 3.7253546198186402, + "tokens_seen": 873922560 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005941023069207623, + "loss": 1.5071, + "theoretical_loss": 3.7253275767204426, + "tokens_seen": 873988096 + }, + { + "epoch": 0.26, + "learning_rate": 0.000594086258776329, + "loss": 1.5329, + "theoretical_loss": 3.7253005362177385, + "tokens_seen": 874053632 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005940702106318958, + "loss": 1.4773, + "theoretical_loss": 3.7252734983100853, + "tokens_seen": 874119168 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005940541624874624, + "loss": 1.4661, + "theoretical_loss": 3.7252464629970383, + "tokens_seen": 874184704 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005940381143430292, + "loss": 1.3476, + "theoretical_loss": 3.7252194302781536, + "tokens_seen": 874250240 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005940220661985958, + "loss": 1.5016, + "theoretical_loss": 3.7251924001529892, + "tokens_seen": 874315776 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005940060180541626, + "loss": 1.5593, + "theoretical_loss": 3.7251653726211016, + "tokens_seen": 874381312 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005939899699097292, + "loss": 1.4801, + "theoretical_loss": 3.7251383476820465, + "tokens_seen": 874446848 + }, + { + "epoch": 0.27, + "learning_rate": 0.000593973921765296, + "loss": 1.4722, + "theoretical_loss": 3.725111325335382, + "tokens_seen": 874512384 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005939578736208626, + "loss": 1.459, + "theoretical_loss": 3.7250843055806646, + "tokens_seen": 874577920 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005939418254764294, + "loss": 1.3638, + "theoretical_loss": 3.7250572884174518, + "tokens_seen": 874643456 + }, + { + "epoch": 0.27, + "learning_rate": 0.000593925777331996, + "loss": 1.5381, + "theoretical_loss": 3.7250302738453005, + "tokens_seen": 874708992 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005939097291875628, + "loss": 1.5834, + "theoretical_loss": 3.725003261863769, + "tokens_seen": 874774528 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005938936810431294, + "loss": 1.4143, + "theoretical_loss": 3.724976252472414, + "tokens_seen": 874840064 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.9936264753341675, + "objective/train/avg_token_score": 0.006373567972332239, + "objective/train/docs_used": 323008, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4573657512664795, + "objective/train/original_loss": 1.4665029048919678, + "objective/train/theoretical_loss": 3.7249492456707936, + "objective/train/tokens_used": 895365600, + "objective/train/unlikelihood_loss": 0.018805908039212227, + "theoretical_loss": 3.7249492456707936, + "tokens_seen": 874905600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005938776328986962, + "loss": 1.4016, + "theoretical_loss": 3.7249492456707936, + "tokens_seen": 874905600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005938615847542628, + "loss": 1.5758, + "theoretical_loss": 3.724922241458464, + "tokens_seen": 874971136 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005938455366098296, + "loss": 1.377, + "theoretical_loss": 3.7248952398349857, + "tokens_seen": 875036672 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005938294884653962, + "loss": 1.4374, + "theoretical_loss": 3.7248682407999145, + "tokens_seen": 875102208 + }, + { + "epoch": 0.27, + "learning_rate": 0.000593813440320963, + "loss": 1.4591, + "theoretical_loss": 3.7248412443528096, + "tokens_seen": 875167744 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005937973921765296, + "loss": 1.3981, + "theoretical_loss": 3.7248142504932282, + "tokens_seen": 875233280 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005937813440320964, + "loss": 1.4827, + "theoretical_loss": 3.7247872592207303, + "tokens_seen": 875298816 + }, + { + "epoch": 0.27, + "learning_rate": 0.000593765295887663, + "loss": 1.4882, + "theoretical_loss": 3.7247602705348717, + "tokens_seen": 875364352 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005937492477432298, + "loss": 1.4259, + "theoretical_loss": 3.7247332844352132, + "tokens_seen": 875429888 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005937331995987964, + "loss": 1.4125, + "theoretical_loss": 3.724706300921312, + "tokens_seen": 875495424 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005937171514543632, + "loss": 1.4688, + "theoretical_loss": 3.7246793199927275, + "tokens_seen": 875560960 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005937011033099298, + "loss": 1.6103, + "theoretical_loss": 3.7246523416490187, + "tokens_seen": 875626496 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005936850551654966, + "loss": 1.4565, + "theoretical_loss": 3.724625365889744, + "tokens_seen": 875692032 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005936690070210632, + "loss": 1.3636, + "theoretical_loss": 3.724598392714462, + "tokens_seen": 875757568 + }, + { + "epoch": 0.27, + "learning_rate": 0.00059365295887663, + "loss": 1.446, + "theoretical_loss": 3.724571422122733, + "tokens_seen": 875823104 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005936369107321966, + "loss": 1.4231, + "theoretical_loss": 3.724544454114115, + "tokens_seen": 875888640 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005936208625877634, + "loss": 1.378, + "theoretical_loss": 3.7245174886881687, + "tokens_seen": 875954176 + }, + { + "epoch": 0.27, + "learning_rate": 0.00059360481444333, + "loss": 1.5081, + "theoretical_loss": 3.7244905258444527, + "tokens_seen": 876019712 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005935887662988968, + "loss": 1.5187, + "theoretical_loss": 3.7244635655825267, + "tokens_seen": 876085248 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005935727181544634, + "loss": 1.541, + "theoretical_loss": 3.7244366079019504, + "tokens_seen": 876150784 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005935566700100302, + "loss": 1.4487, + "theoretical_loss": 3.724409652802284, + "tokens_seen": 876216320 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005935406218655968, + "loss": 1.4812, + "theoretical_loss": 3.7243827002830865, + "tokens_seen": 876281856 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005935245737211636, + "loss": 1.4878, + "theoretical_loss": 3.7243557503439186, + "tokens_seen": 876347392 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005935085255767302, + "loss": 1.4339, + "theoretical_loss": 3.72432880298434, + "tokens_seen": 876412928 + }, + { + "epoch": 0.27, + "learning_rate": 0.000593492477432297, + "loss": 1.5493, + "theoretical_loss": 3.7243018582039116, + "tokens_seen": 876478464 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005934764292878636, + "loss": 1.4396, + "theoretical_loss": 3.724274916002193, + "tokens_seen": 876544000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005934603811434304, + "loss": 1.4968, + "theoretical_loss": 3.724247976378745, + "tokens_seen": 876609536 + }, + { + "epoch": 0.27, + "learning_rate": 0.000593444332998997, + "loss": 1.5887, + "theoretical_loss": 3.724221039333128, + "tokens_seen": 876675072 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005934282848545637, + "loss": 1.4405, + "theoretical_loss": 3.724194104864903, + "tokens_seen": 876740608 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005934122367101304, + "loss": 1.5143, + "theoretical_loss": 3.72416717297363, + "tokens_seen": 876806144 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005933961885656971, + "loss": 1.4182, + "theoretical_loss": 3.7241402436588706, + "tokens_seen": 876871680 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005933801404212638, + "loss": 1.393, + "theoretical_loss": 3.724113316920185, + "tokens_seen": 876937216 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005933640922768305, + "loss": 1.556, + "theoretical_loss": 3.724086392757136, + "tokens_seen": 877002752 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005933480441323972, + "loss": 1.3775, + "theoretical_loss": 3.7240594711692827, + "tokens_seen": 877068288 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005933319959879639, + "loss": 1.4595, + "theoretical_loss": 3.7240325521561872, + "tokens_seen": 877133824 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005933159478435306, + "loss": 1.4304, + "theoretical_loss": 3.724005635717411, + "tokens_seen": 877199360 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005932998996990973, + "loss": 1.4463, + "theoretical_loss": 3.7239787218525158, + "tokens_seen": 877264896 + }, + { + "epoch": 0.27, + "learning_rate": 0.000593283851554664, + "loss": 1.4383, + "theoretical_loss": 3.723951810561063, + "tokens_seen": 877330432 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005932678034102307, + "loss": 1.4474, + "theoretical_loss": 3.723924901842614, + "tokens_seen": 877395968 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005932517552657974, + "loss": 1.4719, + "theoretical_loss": 3.7238979956967317, + "tokens_seen": 877461504 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005932357071213641, + "loss": 1.4674, + "theoretical_loss": 3.7238710921229767, + "tokens_seen": 877527040 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005932196589769308, + "loss": 1.3158, + "theoretical_loss": 3.7238441911209117, + "tokens_seen": 877592576 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005932036108324975, + "loss": 1.4349, + "theoretical_loss": 3.723817292690099, + "tokens_seen": 877658112 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005931875626880642, + "loss": 1.5191, + "theoretical_loss": 3.723790396830101, + "tokens_seen": 877723648 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005931715145436309, + "loss": 1.4901, + "theoretical_loss": 3.7237635035404795, + "tokens_seen": 877789184 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005931554663991976, + "loss": 1.5007, + "theoretical_loss": 3.723736612820797, + "tokens_seen": 877854720 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005931394182547643, + "loss": 1.4841, + "theoretical_loss": 3.7237097246706163, + "tokens_seen": 877920256 + }, + { + "epoch": 0.27, + "learning_rate": 0.000593123370110331, + "loss": 1.4981, + "theoretical_loss": 3.7236828390895003, + "tokens_seen": 877985792 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005931073219658977, + "loss": 1.5246, + "theoretical_loss": 3.7236559560770113, + "tokens_seen": 878051328 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005930912738214644, + "loss": 1.4844, + "theoretical_loss": 3.7236290756327133, + "tokens_seen": 878116864 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.9883785247802734, + "objective/train/avg_token_score": 0.011621516197919846, + "objective/train/docs_used": 324331, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.3458869457244873, + "objective/train/original_loss": 1.3658087253570557, + "objective/train/theoretical_loss": 3.7236021977561675, + "objective/train/tokens_used": 898642400, + "objective/train/unlikelihood_loss": 0.026009755209088326, + "theoretical_loss": 3.7236021977561675, + "tokens_seen": 878182400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005930752256770311, + "loss": 1.4641, + "theoretical_loss": 3.7236021977561675, + "tokens_seen": 878182400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005930591775325978, + "loss": 1.3974, + "theoretical_loss": 3.723575322446939, + "tokens_seen": 878247936 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005930431293881645, + "loss": 1.4948, + "theoretical_loss": 3.723548449704589, + "tokens_seen": 878313472 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005930270812437312, + "loss": 1.3565, + "theoretical_loss": 3.7235215795286827, + "tokens_seen": 878379008 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005930110330992979, + "loss": 1.5306, + "theoretical_loss": 3.7234947119187822, + "tokens_seen": 878444544 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005929949849548646, + "loss": 1.3487, + "theoretical_loss": 3.723467846874452, + "tokens_seen": 878510080 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005929789368104313, + "loss": 1.6112, + "theoretical_loss": 3.723440984395255, + "tokens_seen": 878575616 + }, + { + "epoch": 0.27, + "learning_rate": 0.000592962888665998, + "loss": 1.4416, + "theoretical_loss": 3.7234141244807555, + "tokens_seen": 878641152 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005929468405215647, + "loss": 1.3979, + "theoretical_loss": 3.7233872671305175, + "tokens_seen": 878706688 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005929307923771314, + "loss": 1.3945, + "theoretical_loss": 3.723360412344104, + "tokens_seen": 878772224 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005929147442326981, + "loss": 1.4115, + "theoretical_loss": 3.7233335601210804, + "tokens_seen": 878837760 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005928986960882648, + "loss": 1.4881, + "theoretical_loss": 3.7233067104610096, + "tokens_seen": 878903296 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005928826479438315, + "loss": 1.5152, + "theoretical_loss": 3.723279863363457, + "tokens_seen": 878968832 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005928665997993982, + "loss": 1.5261, + "theoretical_loss": 3.723253018827986, + "tokens_seen": 879034368 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005928505516549649, + "loss": 1.3332, + "theoretical_loss": 3.723226176854162, + "tokens_seen": 879099904 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005928345035105316, + "loss": 1.4527, + "theoretical_loss": 3.723199337441549, + "tokens_seen": 879165440 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005928184553660983, + "loss": 1.405, + "theoretical_loss": 3.723172500589712, + "tokens_seen": 879230976 + }, + { + "epoch": 0.27, + "learning_rate": 0.000592802407221665, + "loss": 1.5009, + "theoretical_loss": 3.7231456662982154, + "tokens_seen": 879296512 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005927863590772317, + "loss": 1.403, + "theoretical_loss": 3.723118834566625, + "tokens_seen": 879362048 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005927703109327984, + "loss": 1.53, + "theoretical_loss": 3.7230920053945056, + "tokens_seen": 879427584 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005927542627883651, + "loss": 1.4963, + "theoretical_loss": 3.7230651787814217, + "tokens_seen": 879493120 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005927382146439318, + "loss": 1.3635, + "theoretical_loss": 3.7230383547269392, + "tokens_seen": 879558656 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005927221664994985, + "loss": 1.5027, + "theoretical_loss": 3.723011533230623, + "tokens_seen": 879624192 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005927061183550653, + "loss": 1.5223, + "theoretical_loss": 3.7229847142920383, + "tokens_seen": 879689728 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005926900702106319, + "loss": 1.5201, + "theoretical_loss": 3.7229578979107516, + "tokens_seen": 879755264 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005926740220661987, + "loss": 1.4707, + "theoretical_loss": 3.722931084086328, + "tokens_seen": 879820800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005926579739217653, + "loss": 1.5623, + "theoretical_loss": 3.722904272818334, + "tokens_seen": 879886336 + }, + { + "epoch": 0.27, + "learning_rate": 0.000592641925777332, + "loss": 1.5207, + "theoretical_loss": 3.722877464106334, + "tokens_seen": 879951872 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005926258776328987, + "loss": 1.5068, + "theoretical_loss": 3.722850657949895, + "tokens_seen": 880017408 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005926098294884655, + "loss": 1.5236, + "theoretical_loss": 3.722823854348583, + "tokens_seen": 880082944 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005925937813440321, + "loss": 1.5591, + "theoretical_loss": 3.7227970533019645, + "tokens_seen": 880148480 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005925777331995989, + "loss": 1.4092, + "theoretical_loss": 3.7227702548096056, + "tokens_seen": 880214016 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005925616850551655, + "loss": 1.4283, + "theoretical_loss": 3.722743458871072, + "tokens_seen": 880279552 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005925456369107323, + "loss": 1.4405, + "theoretical_loss": 3.7227166654859314, + "tokens_seen": 880345088 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005925295887662989, + "loss": 1.4739, + "theoretical_loss": 3.7226898746537493, + "tokens_seen": 880410624 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005925135406218657, + "loss": 1.3833, + "theoretical_loss": 3.7226630863740935, + "tokens_seen": 880476160 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005924974924774323, + "loss": 1.4853, + "theoretical_loss": 3.7226363006465304, + "tokens_seen": 880541696 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005924814443329991, + "loss": 1.493, + "theoretical_loss": 3.7226095174706266, + "tokens_seen": 880607232 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005924653961885657, + "loss": 1.3682, + "theoretical_loss": 3.7225827368459496, + "tokens_seen": 880672768 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005924493480441325, + "loss": 1.3215, + "theoretical_loss": 3.722555958772066, + "tokens_seen": 880738304 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005924332998996991, + "loss": 1.3869, + "theoretical_loss": 3.722529183248544, + "tokens_seen": 880803840 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005924172517552659, + "loss": 1.3741, + "theoretical_loss": 3.7225024102749504, + "tokens_seen": 880869376 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005924012036108325, + "loss": 1.4131, + "theoretical_loss": 3.7224756398508525, + "tokens_seen": 880934912 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005923851554663993, + "loss": 1.5927, + "theoretical_loss": 3.7224488719758178, + "tokens_seen": 881000448 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005923691073219659, + "loss": 1.6109, + "theoretical_loss": 3.7224221066494145, + "tokens_seen": 881065984 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005923530591775327, + "loss": 1.5301, + "theoretical_loss": 3.7223953438712103, + "tokens_seen": 881131520 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005923370110330993, + "loss": 1.4805, + "theoretical_loss": 3.7223685836407734, + "tokens_seen": 881197056 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005923209628886661, + "loss": 1.4829, + "theoretical_loss": 3.7223418259576704, + "tokens_seen": 881262592 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005923049147442327, + "loss": 1.4407, + "theoretical_loss": 3.7223150708214714, + "tokens_seen": 881328128 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005922888665997995, + "loss": 1.4202, + "theoretical_loss": 3.722288318231743, + "tokens_seen": 881393664 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.9939919710159302, + "objective/train/avg_token_score": 0.006008072756230831, + "objective/train/docs_used": 325483, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.3953746557235718, + "objective/train/original_loss": 1.4038684368133545, + "objective/train/theoretical_loss": 3.722261568188054, + "objective/train/tokens_used": 901919200, + "objective/train/unlikelihood_loss": 0.01641520857810974, + "theoretical_loss": 3.722261568188054, + "tokens_seen": 881459200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005922728184553661, + "loss": 1.3952, + "theoretical_loss": 3.722261568188054, + "tokens_seen": 881459200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005922567703109329, + "loss": 1.5338, + "theoretical_loss": 3.722234820689973, + "tokens_seen": 881524736 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005922407221664995, + "loss": 1.4075, + "theoretical_loss": 3.722208075737069, + "tokens_seen": 881590272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005922246740220663, + "loss": 1.5057, + "theoretical_loss": 3.7221813333289093, + "tokens_seen": 881655808 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005922086258776329, + "loss": 1.4707, + "theoretical_loss": 3.722154593465064, + "tokens_seen": 881721344 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005921925777331997, + "loss": 1.5365, + "theoretical_loss": 3.722127856145101, + "tokens_seen": 881786880 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005921765295887663, + "loss": 1.44, + "theoretical_loss": 3.72210112136859, + "tokens_seen": 881852416 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005921604814443331, + "loss": 1.4731, + "theoretical_loss": 3.7220743891350994, + "tokens_seen": 881917952 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005921444332998997, + "loss": 1.4447, + "theoretical_loss": 3.7220476594441987, + "tokens_seen": 881983488 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005921283851554665, + "loss": 1.5009, + "theoretical_loss": 3.7220209322954574, + "tokens_seen": 882049024 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005921123370110331, + "loss": 1.4559, + "theoretical_loss": 3.7219942076884447, + "tokens_seen": 882114560 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005920962888665999, + "loss": 1.4668, + "theoretical_loss": 3.72196748562273, + "tokens_seen": 882180096 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005920802407221665, + "loss": 1.4961, + "theoretical_loss": 3.721940766097882, + "tokens_seen": 882245632 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005920641925777333, + "loss": 1.5071, + "theoretical_loss": 3.721914049113472, + "tokens_seen": 882311168 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005920481444332999, + "loss": 1.3998, + "theoretical_loss": 3.721887334669069, + "tokens_seen": 882376704 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005920320962888666, + "loss": 1.5562, + "theoretical_loss": 3.7218606227642423, + "tokens_seen": 882442240 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005920160481444333, + "loss": 1.3927, + "theoretical_loss": 3.721833913398563, + "tokens_seen": 882507776 + }, + { + "epoch": 0.27, + "learning_rate": 0.000592, + "loss": 1.4638, + "theoretical_loss": 3.7218072065716004, + "tokens_seen": 882573312 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005919839518555667, + "loss": 1.5457, + "theoretical_loss": 3.7217805022829253, + "tokens_seen": 882638848 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005919679037111334, + "loss": 1.5554, + "theoretical_loss": 3.721753800532108, + "tokens_seen": 882704384 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005919518555667001, + "loss": 1.4836, + "theoretical_loss": 3.7217271013187183, + "tokens_seen": 882769920 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005919358074222668, + "loss": 1.4141, + "theoretical_loss": 3.721700404642327, + "tokens_seen": 882835456 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005919197592778335, + "loss": 1.4196, + "theoretical_loss": 3.721673710502505, + "tokens_seen": 882900992 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005919037111334002, + "loss": 1.4025, + "theoretical_loss": 3.7216470188988224, + "tokens_seen": 882966528 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005918876629889669, + "loss": 1.5022, + "theoretical_loss": 3.721620329830851, + "tokens_seen": 883032064 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005918716148445336, + "loss": 1.5032, + "theoretical_loss": 3.721593643298161, + "tokens_seen": 883097600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005918555667001003, + "loss": 1.4089, + "theoretical_loss": 3.7215669593003238, + "tokens_seen": 883163136 + }, + { + "epoch": 0.27, + "learning_rate": 0.000591839518555667, + "loss": 1.5696, + "theoretical_loss": 3.72154027783691, + "tokens_seen": 883228672 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005918234704112337, + "loss": 1.4113, + "theoretical_loss": 3.7215135989074914, + "tokens_seen": 883294208 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005918074222668004, + "loss": 1.4233, + "theoretical_loss": 3.7214869225116396, + "tokens_seen": 883359744 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005917913741223671, + "loss": 1.3937, + "theoretical_loss": 3.7214602486489254, + "tokens_seen": 883425280 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005917753259779338, + "loss": 1.4496, + "theoretical_loss": 3.721433577318921, + "tokens_seen": 883490816 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005917592778335005, + "loss": 1.3336, + "theoretical_loss": 3.721406908521197, + "tokens_seen": 883556352 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005917432296890672, + "loss": 1.4113, + "theoretical_loss": 3.7213802422553264, + "tokens_seen": 883621888 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005917271815446339, + "loss": 1.4111, + "theoretical_loss": 3.7213535785208807, + "tokens_seen": 883687424 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005917111334002006, + "loss": 1.5265, + "theoretical_loss": 3.7213269173174313, + "tokens_seen": 883752960 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005916950852557673, + "loss": 1.4189, + "theoretical_loss": 3.7213002586445514, + "tokens_seen": 883818496 + }, + { + "epoch": 0.27, + "learning_rate": 0.000591679037111334, + "loss": 1.4878, + "theoretical_loss": 3.721273602501812, + "tokens_seen": 883884032 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005916629889669007, + "loss": 1.4119, + "theoretical_loss": 3.721246948888786, + "tokens_seen": 883949568 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005916469408224674, + "loss": 1.5715, + "theoretical_loss": 3.7212202978050457, + "tokens_seen": 884015104 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005916308926780341, + "loss": 1.4288, + "theoretical_loss": 3.721193649250164, + "tokens_seen": 884080640 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005916148445336008, + "loss": 1.4825, + "theoretical_loss": 3.7211670032237123, + "tokens_seen": 884146176 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005915987963891675, + "loss": 1.4078, + "theoretical_loss": 3.721140359725265, + "tokens_seen": 884211712 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005915827482447342, + "loss": 1.4515, + "theoretical_loss": 3.7211137187543937, + "tokens_seen": 884277248 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005915667001003009, + "loss": 1.5063, + "theoretical_loss": 3.7210870803106717, + "tokens_seen": 884342784 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005915506519558676, + "loss": 1.2831, + "theoretical_loss": 3.721060444393672, + "tokens_seen": 884408320 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005915346038114343, + "loss": 1.5009, + "theoretical_loss": 3.721033811002968, + "tokens_seen": 884473856 + }, + { + "epoch": 0.27, + "learning_rate": 0.000591518555667001, + "loss": 1.4516, + "theoretical_loss": 3.721007180138132, + "tokens_seen": 884539392 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005915025075225677, + "loss": 1.3267, + "theoretical_loss": 3.720980551798739, + "tokens_seen": 884604928 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005914864593781344, + "loss": 1.5279, + "theoretical_loss": 3.720953925984361, + "tokens_seen": 884670464 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.9930366277694702, + "objective/train/avg_token_score": 0.006963417399674654, + "objective/train/docs_used": 326745, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6441460847854614, + "objective/train/original_loss": 1.6571362018585205, + "objective/train/theoretical_loss": 3.720927302694572, + "objective/train/tokens_used": 905196000, + "objective/train/unlikelihood_loss": 0.0128669124096632, + "theoretical_loss": 3.720927302694572, + "tokens_seen": 884736000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005914704112337011, + "loss": 1.5175, + "theoretical_loss": 3.720927302694572, + "tokens_seen": 884736000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005914543630892678, + "loss": 1.4679, + "theoretical_loss": 3.7209006819289456, + "tokens_seen": 884801536 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005914383149448345, + "loss": 1.4746, + "theoretical_loss": 3.7208740636870554, + "tokens_seen": 884867072 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005914222668004012, + "loss": 1.4866, + "theoretical_loss": 3.7208474479684757, + "tokens_seen": 884932608 + }, + { + "epoch": 0.27, + "learning_rate": 0.000591406218655968, + "loss": 1.5277, + "theoretical_loss": 3.72082083477278, + "tokens_seen": 884998144 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005913901705115346, + "loss": 1.4704, + "theoretical_loss": 3.7207942240995426, + "tokens_seen": 885063680 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005913741223671013, + "loss": 1.4915, + "theoretical_loss": 3.7207676159483376, + "tokens_seen": 885129216 + }, + { + "epoch": 0.27, + "learning_rate": 0.000591358074222668, + "loss": 1.3147, + "theoretical_loss": 3.720741010318739, + "tokens_seen": 885194752 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005913420260782348, + "loss": 1.3988, + "theoretical_loss": 3.720714407210322, + "tokens_seen": 885260288 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005913259779338014, + "loss": 1.3587, + "theoretical_loss": 3.72068780662266, + "tokens_seen": 885325824 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005913099297893682, + "loss": 1.4306, + "theoretical_loss": 3.7206612085553283, + "tokens_seen": 885391360 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005912938816449348, + "loss": 1.4678, + "theoretical_loss": 3.7206346130079018, + "tokens_seen": 885456896 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005912778335005016, + "loss": 1.4896, + "theoretical_loss": 3.720608019979954, + "tokens_seen": 885522432 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005912617853560682, + "loss": 1.507, + "theoretical_loss": 3.7205814294710615, + "tokens_seen": 885587968 + }, + { + "epoch": 0.27, + "learning_rate": 0.000591245737211635, + "loss": 1.4395, + "theoretical_loss": 3.720554841480798, + "tokens_seen": 885653504 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005912296890672016, + "loss": 1.4608, + "theoretical_loss": 3.720528256008739, + "tokens_seen": 885719040 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005912136409227684, + "loss": 1.5308, + "theoretical_loss": 3.72050167305446, + "tokens_seen": 885784576 + }, + { + "epoch": 0.27, + "learning_rate": 0.000591197592778335, + "loss": 1.4157, + "theoretical_loss": 3.7204750926175354, + "tokens_seen": 885850112 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005911815446339018, + "loss": 1.4522, + "theoretical_loss": 3.7204485146975417, + "tokens_seen": 885915648 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005911654964894684, + "loss": 1.481, + "theoretical_loss": 3.720421939294054, + "tokens_seen": 885981184 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005911494483450352, + "loss": 1.4301, + "theoretical_loss": 3.7203953664066476, + "tokens_seen": 886046720 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005911334002006018, + "loss": 1.4898, + "theoretical_loss": 3.7203687960348986, + "tokens_seen": 886112256 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005911173520561686, + "loss": 1.3812, + "theoretical_loss": 3.7203422281783824, + "tokens_seen": 886177792 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005911013039117352, + "loss": 1.5216, + "theoretical_loss": 3.7203156628366756, + "tokens_seen": 886243328 + }, + { + "epoch": 0.27, + "learning_rate": 0.000591085255767302, + "loss": 1.5422, + "theoretical_loss": 3.720289100009353, + "tokens_seen": 886308864 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005910692076228686, + "loss": 1.3375, + "theoretical_loss": 3.7202625396959923, + "tokens_seen": 886374400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005910531594784354, + "loss": 1.4528, + "theoretical_loss": 3.7202359818961686, + "tokens_seen": 886439936 + }, + { + "epoch": 0.27, + "learning_rate": 0.000591037111334002, + "loss": 1.4909, + "theoretical_loss": 3.7202094266094585, + "tokens_seen": 886505472 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005910210631895688, + "loss": 1.4529, + "theoretical_loss": 3.720182873835439, + "tokens_seen": 886571008 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005910050150451354, + "loss": 1.3541, + "theoretical_loss": 3.7201563235736854, + "tokens_seen": 886636544 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005909889669007022, + "loss": 1.4171, + "theoretical_loss": 3.7201297758237755, + "tokens_seen": 886702080 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005909729187562689, + "loss": 1.4714, + "theoretical_loss": 3.720103230585286, + "tokens_seen": 886767616 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005909568706118356, + "loss": 1.4395, + "theoretical_loss": 3.7200766878577927, + "tokens_seen": 886833152 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005909408224674023, + "loss": 1.4538, + "theoretical_loss": 3.720050147640873, + "tokens_seen": 886898688 + }, + { + "epoch": 0.27, + "learning_rate": 0.000590924774322969, + "loss": 1.4693, + "theoretical_loss": 3.7200236099341044, + "tokens_seen": 886964224 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005909087261785357, + "loss": 1.4646, + "theoretical_loss": 3.719997074737064, + "tokens_seen": 887029760 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005908926780341024, + "loss": 1.5497, + "theoretical_loss": 3.7199705420493285, + "tokens_seen": 887095296 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005908766298896691, + "loss": 1.4931, + "theoretical_loss": 3.719944011870476, + "tokens_seen": 887160832 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005908605817452358, + "loss": 1.4443, + "theoretical_loss": 3.7199174842000833, + "tokens_seen": 887226368 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005908445336008025, + "loss": 1.504, + "theoretical_loss": 3.719890959037728, + "tokens_seen": 887291904 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005908284854563692, + "loss": 1.4509, + "theoretical_loss": 3.719864436382988, + "tokens_seen": 887357440 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005908124373119359, + "loss": 1.4794, + "theoretical_loss": 3.7198379162354414, + "tokens_seen": 887422976 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005907963891675026, + "loss": 1.4442, + "theoretical_loss": 3.7198113985946657, + "tokens_seen": 887488512 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005907803410230693, + "loss": 1.4927, + "theoretical_loss": 3.719784883460238, + "tokens_seen": 887554048 + }, + { + "epoch": 0.27, + "learning_rate": 0.000590764292878636, + "loss": 1.3669, + "theoretical_loss": 3.719758370831738, + "tokens_seen": 887619584 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005907482447342026, + "loss": 1.4023, + "theoretical_loss": 3.719731860708743, + "tokens_seen": 887685120 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005907321965897694, + "loss": 1.4068, + "theoretical_loss": 3.7197053530908315, + "tokens_seen": 887750656 + }, + { + "epoch": 0.27, + "learning_rate": 0.000590716148445336, + "loss": 1.4882, + "theoretical_loss": 3.7196788479775815, + "tokens_seen": 887816192 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005907001003009028, + "loss": 1.371, + "theoretical_loss": 3.7196523453685715, + "tokens_seen": 887881728 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005906840521564694, + "loss": 1.5059, + "theoretical_loss": 3.719625845263381, + "tokens_seen": 887947264 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.9947243928909302, + "objective/train/avg_token_score": 0.005275667179375887, + "objective/train/docs_used": 327973, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.3228715658187866, + "objective/train/original_loss": 1.3324809074401855, + "objective/train/theoretical_loss": 3.719599347661587, + "objective/train/tokens_used": 908472800, + "objective/train/unlikelihood_loss": 0.01183464378118515, + "theoretical_loss": 3.719599347661587, + "tokens_seen": 888012800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005906680040120362, + "loss": 1.3848, + "theoretical_loss": 3.719599347661587, + "tokens_seen": 888012800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005906519558676028, + "loss": 1.5214, + "theoretical_loss": 3.71957285256277, + "tokens_seen": 888078336 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005906359077231695, + "loss": 1.4335, + "theoretical_loss": 3.7195463599665075, + "tokens_seen": 888143872 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005906198595787362, + "loss": 1.4665, + "theoretical_loss": 3.7195198698723795, + "tokens_seen": 888209408 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005906038114343029, + "loss": 1.4167, + "theoretical_loss": 3.719493382279965, + "tokens_seen": 888274944 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005905877632898696, + "loss": 1.4223, + "theoretical_loss": 3.719466897188843, + "tokens_seen": 888340480 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005905717151454363, + "loss": 1.4772, + "theoretical_loss": 3.7194404145985924, + "tokens_seen": 888406016 + }, + { + "epoch": 0.27, + "learning_rate": 0.000590555667001003, + "loss": 1.3567, + "theoretical_loss": 3.719413934508793, + "tokens_seen": 888471552 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005905396188565697, + "loss": 1.4758, + "theoretical_loss": 3.719387456919024, + "tokens_seen": 888537088 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005905235707121364, + "loss": 1.4605, + "theoretical_loss": 3.7193609818288653, + "tokens_seen": 888602624 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005905075225677031, + "loss": 1.3639, + "theoretical_loss": 3.7193345092378967, + "tokens_seen": 888668160 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005904914744232698, + "loss": 1.4282, + "theoretical_loss": 3.719308039145698, + "tokens_seen": 888733696 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005904754262788365, + "loss": 1.4833, + "theoretical_loss": 3.7192815715518486, + "tokens_seen": 888799232 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005904593781344032, + "loss": 1.4262, + "theoretical_loss": 3.719255106455929, + "tokens_seen": 888864768 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005904433299899699, + "loss": 1.5778, + "theoretical_loss": 3.7192286438575186, + "tokens_seen": 888930304 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005904272818455366, + "loss": 1.418, + "theoretical_loss": 3.7192021837561984, + "tokens_seen": 888995840 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005904112337011033, + "loss": 1.4436, + "theoretical_loss": 3.7191757261515486, + "tokens_seen": 889061376 + }, + { + "epoch": 0.27, + "learning_rate": 0.00059039518555667, + "loss": 1.4268, + "theoretical_loss": 3.7191492710431495, + "tokens_seen": 889126912 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005903791374122367, + "loss": 1.3869, + "theoretical_loss": 3.7191228184305816, + "tokens_seen": 889192448 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005903630892678034, + "loss": 1.4839, + "theoretical_loss": 3.719096368313425, + "tokens_seen": 889257984 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005903470411233701, + "loss": 1.501, + "theoretical_loss": 3.719069920691261, + "tokens_seen": 889323520 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005903309929789368, + "loss": 1.4967, + "theoretical_loss": 3.7190434755636708, + "tokens_seen": 889389056 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005903149448345035, + "loss": 1.4606, + "theoretical_loss": 3.7190170329302346, + "tokens_seen": 889454592 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005902988966900702, + "loss": 1.5159, + "theoretical_loss": 3.718990592790534, + "tokens_seen": 889520128 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005902828485456369, + "loss": 1.4836, + "theoretical_loss": 3.718964155144149, + "tokens_seen": 889585664 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005902668004012036, + "loss": 1.354, + "theoretical_loss": 3.718937719990662, + "tokens_seen": 889651200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005902507522567703, + "loss": 1.4629, + "theoretical_loss": 3.7189112873296537, + "tokens_seen": 889716736 + }, + { + "epoch": 0.27, + "learning_rate": 0.000590234704112337, + "loss": 1.4492, + "theoretical_loss": 3.718884857160706, + "tokens_seen": 889782272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005902186559679037, + "loss": 1.5182, + "theoretical_loss": 3.7188584294834004, + "tokens_seen": 889847808 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005902026078234704, + "loss": 1.4672, + "theoretical_loss": 3.718832004297318, + "tokens_seen": 889913344 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005901865596790371, + "loss": 1.4894, + "theoretical_loss": 3.718805581602041, + "tokens_seen": 889978880 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005901705115346038, + "loss": 1.5014, + "theoretical_loss": 3.7187791613971504, + "tokens_seen": 890044416 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005901544633901705, + "loss": 1.473, + "theoretical_loss": 3.7187527436822294, + "tokens_seen": 890109952 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005901384152457372, + "loss": 1.4628, + "theoretical_loss": 3.7187263284568592, + "tokens_seen": 890175488 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005901223671013039, + "loss": 1.4089, + "theoretical_loss": 3.7186999157206224, + "tokens_seen": 890241024 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005901063189568706, + "loss": 1.3585, + "theoretical_loss": 3.7186735054731006, + "tokens_seen": 890306560 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005900902708124373, + "loss": 1.403, + "theoretical_loss": 3.718647097713877, + "tokens_seen": 890372096 + }, + { + "epoch": 0.27, + "learning_rate": 0.000590074222668004, + "loss": 1.4316, + "theoretical_loss": 3.718620692442533, + "tokens_seen": 890437632 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005900581745235707, + "loss": 1.4429, + "theoretical_loss": 3.718594289658652, + "tokens_seen": 890503168 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005900421263791374, + "loss": 1.4473, + "theoretical_loss": 3.7185678893618164, + "tokens_seen": 890568704 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005900260782347041, + "loss": 1.4753, + "theoretical_loss": 3.7185414915516084, + "tokens_seen": 890634240 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005900100300902708, + "loss": 1.4501, + "theoretical_loss": 3.7185150962276117, + "tokens_seen": 890699776 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005899939819458375, + "loss": 1.4701, + "theoretical_loss": 3.7184887033894087, + "tokens_seen": 890765312 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005899779338014042, + "loss": 1.3651, + "theoretical_loss": 3.718462313036582, + "tokens_seen": 890830848 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005899618856569709, + "loss": 1.4274, + "theoretical_loss": 3.7184359251687162, + "tokens_seen": 890896384 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005899458375125377, + "loss": 1.4524, + "theoretical_loss": 3.718409539785393, + "tokens_seen": 890961920 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005899297893681043, + "loss": 1.485, + "theoretical_loss": 3.7183831568861967, + "tokens_seen": 891027456 + }, + { + "epoch": 0.27, + "learning_rate": 0.000589913741223671, + "loss": 1.4163, + "theoretical_loss": 3.718356776470711, + "tokens_seen": 891092992 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005898976930792377, + "loss": 1.5674, + "theoretical_loss": 3.7183303985385177, + "tokens_seen": 891158528 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005898816449348045, + "loss": 1.5689, + "theoretical_loss": 3.7183040230892024, + "tokens_seen": 891224064 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.9911864995956421, + "objective/train/avg_token_score": 0.008813573978841305, + "objective/train/docs_used": 329220, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.3726884126663208, + "objective/train/original_loss": 1.3873560428619385, + "objective/train/theoretical_loss": 3.7182776501223476, + "objective/train/tokens_used": 911749600, + "objective/train/unlikelihood_loss": 0.02063899300992489, + "theoretical_loss": 3.7182776501223476, + "tokens_seen": 891289600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005898655967903711, + "loss": 1.4063, + "theoretical_loss": 3.7182776501223476, + "tokens_seen": 891289600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005898495486459379, + "loss": 1.3585, + "theoretical_loss": 3.7182512796375375, + "tokens_seen": 891355136 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005898335005015045, + "loss": 1.4335, + "theoretical_loss": 3.7182249116343566, + "tokens_seen": 891420672 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005898174523570713, + "loss": 1.5318, + "theoretical_loss": 3.7181985461123883, + "tokens_seen": 891486208 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005898014042126379, + "loss": 1.4886, + "theoretical_loss": 3.718172183071217, + "tokens_seen": 891551744 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005897853560682047, + "loss": 1.5339, + "theoretical_loss": 3.7181458225104267, + "tokens_seen": 891617280 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005897693079237714, + "loss": 1.3315, + "theoretical_loss": 3.718119464429602, + "tokens_seen": 891682816 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005897532597793381, + "loss": 1.4907, + "theoretical_loss": 3.718093108828327, + "tokens_seen": 891748352 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005897372116349048, + "loss": 1.4222, + "theoretical_loss": 3.718066755706187, + "tokens_seen": 891813888 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005897211634904715, + "loss": 1.4103, + "theoretical_loss": 3.718040405062766, + "tokens_seen": 891879424 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005897051153460382, + "loss": 1.4886, + "theoretical_loss": 3.7180140568976485, + "tokens_seen": 891944960 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005896890672016049, + "loss": 1.4745, + "theoretical_loss": 3.7179877112104194, + "tokens_seen": 892010496 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005896730190571716, + "loss": 1.4034, + "theoretical_loss": 3.7179613680006645, + "tokens_seen": 892076032 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005896569709127383, + "loss": 1.482, + "theoretical_loss": 3.7179350272679685, + "tokens_seen": 892141568 + }, + { + "epoch": 0.27, + "learning_rate": 0.000589640922768305, + "loss": 1.5266, + "theoretical_loss": 3.7179086890119164, + "tokens_seen": 892207104 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005896248746238717, + "loss": 1.4624, + "theoretical_loss": 3.717882353232093, + "tokens_seen": 892272640 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005896088264794384, + "loss": 1.509, + "theoretical_loss": 3.7178560199280843, + "tokens_seen": 892338176 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005895927783350051, + "loss": 1.4533, + "theoretical_loss": 3.7178296890994753, + "tokens_seen": 892403712 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005895767301905718, + "loss": 1.5549, + "theoretical_loss": 3.717803360745852, + "tokens_seen": 892469248 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005895606820461385, + "loss": 1.5223, + "theoretical_loss": 3.717777034866799, + "tokens_seen": 892534784 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005895446339017052, + "loss": 1.4681, + "theoretical_loss": 3.7177507114619037, + "tokens_seen": 892600320 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005895285857572719, + "loss": 1.5703, + "theoretical_loss": 3.7177243905307504, + "tokens_seen": 892665856 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005895125376128386, + "loss": 1.4522, + "theoretical_loss": 3.7176980720729267, + "tokens_seen": 892731392 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005894964894684053, + "loss": 1.5776, + "theoretical_loss": 3.717671756088017, + "tokens_seen": 892796928 + }, + { + "epoch": 0.27, + "learning_rate": 0.000589480441323972, + "loss": 1.4161, + "theoretical_loss": 3.717645442575608, + "tokens_seen": 892862464 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005894643931795387, + "loss": 1.4958, + "theoretical_loss": 3.717619131535286, + "tokens_seen": 892928000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005894483450351054, + "loss": 1.564, + "theoretical_loss": 3.7175928229666377, + "tokens_seen": 892993536 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005894322968906721, + "loss": 1.5186, + "theoretical_loss": 3.717566516869249, + "tokens_seen": 893059072 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005894162487462388, + "loss": 1.3009, + "theoretical_loss": 3.7175402132427067, + "tokens_seen": 893124608 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005894002006018055, + "loss": 1.5012, + "theoretical_loss": 3.7175139120865976, + "tokens_seen": 893190144 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005893841524573722, + "loss": 1.3941, + "theoretical_loss": 3.7174876134005075, + "tokens_seen": 893255680 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005893681043129389, + "loss": 1.2958, + "theoretical_loss": 3.717461317184025, + "tokens_seen": 893321216 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005893520561685055, + "loss": 1.4681, + "theoretical_loss": 3.7174350234367353, + "tokens_seen": 893386752 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005893360080240723, + "loss": 1.4589, + "theoretical_loss": 3.7174087321582263, + "tokens_seen": 893452288 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005893199598796389, + "loss": 1.432, + "theoretical_loss": 3.717382443348085, + "tokens_seen": 893517824 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005893039117352057, + "loss": 1.4386, + "theoretical_loss": 3.717356157005899, + "tokens_seen": 893583360 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005892878635907723, + "loss": 1.5397, + "theoretical_loss": 3.717329873131255, + "tokens_seen": 893648896 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005892718154463391, + "loss": 1.5269, + "theoretical_loss": 3.7173035917237405, + "tokens_seen": 893714432 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005892557673019057, + "loss": 1.4809, + "theoretical_loss": 3.7172773127829437, + "tokens_seen": 893779968 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005892397191574725, + "loss": 1.4941, + "theoretical_loss": 3.7172510363084514, + "tokens_seen": 893845504 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005892236710130391, + "loss": 1.3647, + "theoretical_loss": 3.717224762299852, + "tokens_seen": 893911040 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005892076228686058, + "loss": 1.4491, + "theoretical_loss": 3.717198490756733, + "tokens_seen": 893976576 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005891915747241725, + "loss": 1.4195, + "theoretical_loss": 3.717172221678682, + "tokens_seen": 894042112 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005891755265797392, + "loss": 1.487, + "theoretical_loss": 3.7171459550652877, + "tokens_seen": 894107648 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005891594784353059, + "loss": 1.5369, + "theoretical_loss": 3.717119690916138, + "tokens_seen": 894173184 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005891434302908726, + "loss": 1.5236, + "theoretical_loss": 3.717093429230821, + "tokens_seen": 894238720 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005891273821464393, + "loss": 1.4425, + "theoretical_loss": 3.7170671700089244, + "tokens_seen": 894304256 + }, + { + "epoch": 0.27, + "learning_rate": 0.000589111334002006, + "loss": 1.4545, + "theoretical_loss": 3.717040913250038, + "tokens_seen": 894369792 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005890952858575727, + "loss": 1.3535, + "theoretical_loss": 3.7170146589537496, + "tokens_seen": 894435328 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005890792377131394, + "loss": 1.6409, + "theoretical_loss": 3.716988407119648, + "tokens_seen": 894500864 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.9950731992721558, + "objective/train/avg_token_score": 0.004926789551973343, + "objective/train/docs_used": 330336, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.353309154510498, + "objective/train/original_loss": 1.3595290184020996, + "objective/train/theoretical_loss": 3.7169621577473215, + "objective/train/tokens_used": 915026400, + "objective/train/unlikelihood_loss": 0.01784222386777401, + "theoretical_loss": 3.7169621577473215, + "tokens_seen": 894566400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005890631895687061, + "loss": 1.4237, + "theoretical_loss": 3.7169621577473215, + "tokens_seen": 894566400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005890471414242728, + "loss": 1.4468, + "theoretical_loss": 3.716935910836359, + "tokens_seen": 894631936 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005890310932798395, + "loss": 1.4473, + "theoretical_loss": 3.71690966638635, + "tokens_seen": 894697472 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005890150451354062, + "loss": 1.4049, + "theoretical_loss": 3.716883424396883, + "tokens_seen": 894763008 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005889989969909729, + "loss": 1.5647, + "theoretical_loss": 3.716857184867547, + "tokens_seen": 894828544 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005889829488465396, + "loss": 1.408, + "theoretical_loss": 3.716830947797932, + "tokens_seen": 894894080 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005889669007021063, + "loss": 1.4177, + "theoretical_loss": 3.7168047131876265, + "tokens_seen": 894959616 + }, + { + "epoch": 0.27, + "learning_rate": 0.000588950852557673, + "loss": 1.4366, + "theoretical_loss": 3.7167784810362203, + "tokens_seen": 895025152 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005889348044132397, + "loss": 1.5098, + "theoretical_loss": 3.7167522513433022, + "tokens_seen": 895090688 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005889187562688064, + "loss": 1.4595, + "theoretical_loss": 3.716726024108463, + "tokens_seen": 895156224 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005889027081243731, + "loss": 1.4673, + "theoretical_loss": 3.716699799331292, + "tokens_seen": 895221760 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005888866599799398, + "loss": 1.4839, + "theoretical_loss": 3.716673577011379, + "tokens_seen": 895287296 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005888706118355065, + "loss": 1.5191, + "theoretical_loss": 3.716647357148313, + "tokens_seen": 895352832 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005888545636910732, + "loss": 1.5429, + "theoretical_loss": 3.7166211397416853, + "tokens_seen": 895418368 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005888385155466399, + "loss": 1.4286, + "theoretical_loss": 3.716594924791085, + "tokens_seen": 895483904 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005888224674022066, + "loss": 1.5201, + "theoretical_loss": 3.7165687122961035, + "tokens_seen": 895549440 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005888064192577733, + "loss": 1.3308, + "theoretical_loss": 3.71654250225633, + "tokens_seen": 895614976 + }, + { + "epoch": 0.27, + "learning_rate": 0.00058879037111334, + "loss": 1.359, + "theoretical_loss": 3.7165162946713552, + "tokens_seen": 895680512 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005887743229689067, + "loss": 1.4271, + "theoretical_loss": 3.7164900895407698, + "tokens_seen": 895746048 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005887582748244734, + "loss": 1.5427, + "theoretical_loss": 3.7164638868641635, + "tokens_seen": 895811584 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005887422266800401, + "loss": 1.5192, + "theoretical_loss": 3.716437686641129, + "tokens_seen": 895877120 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005887261785356068, + "loss": 1.4629, + "theoretical_loss": 3.7164114888712545, + "tokens_seen": 895942656 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005887101303911735, + "loss": 1.4133, + "theoretical_loss": 3.7163852935541333, + "tokens_seen": 896008192 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005886940822467402, + "loss": 1.4778, + "theoretical_loss": 3.716359100689355, + "tokens_seen": 896073728 + }, + { + "epoch": 0.27, + "learning_rate": 0.000588678034102307, + "loss": 1.4265, + "theoretical_loss": 3.7163329102765106, + "tokens_seen": 896139264 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005886619859578736, + "loss": 1.4007, + "theoretical_loss": 3.7163067223151924, + "tokens_seen": 896204800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005886459378134403, + "loss": 1.3541, + "theoretical_loss": 3.7162805368049905, + "tokens_seen": 896270336 + }, + { + "epoch": 0.27, + "learning_rate": 0.000588629889669007, + "loss": 1.4994, + "theoretical_loss": 3.716254353745497, + "tokens_seen": 896335872 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005886138415245737, + "loss": 1.4515, + "theoretical_loss": 3.7162281731363027, + "tokens_seen": 896401408 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005885977933801404, + "loss": 1.412, + "theoretical_loss": 3.7162019949769998, + "tokens_seen": 896466944 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005885817452357072, + "loss": 1.4771, + "theoretical_loss": 3.71617581926718, + "tokens_seen": 896532480 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005885656970912739, + "loss": 1.3673, + "theoretical_loss": 3.7161496460064347, + "tokens_seen": 896598016 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005885496489468406, + "loss": 1.4818, + "theoretical_loss": 3.716123475194356, + "tokens_seen": 896663552 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005885336008024073, + "loss": 1.3556, + "theoretical_loss": 3.7160973068305356, + "tokens_seen": 896729088 + }, + { + "epoch": 0.27, + "learning_rate": 0.000588517552657974, + "loss": 1.4666, + "theoretical_loss": 3.7160711409145657, + "tokens_seen": 896794624 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005885015045135407, + "loss": 1.442, + "theoretical_loss": 3.716044977446039, + "tokens_seen": 896860160 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005884854563691074, + "loss": 1.4325, + "theoretical_loss": 3.7160188164245467, + "tokens_seen": 896925696 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005884694082246741, + "loss": 1.4346, + "theoretical_loss": 3.7159926578496822, + "tokens_seen": 896991232 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005884533600802408, + "loss": 1.3632, + "theoretical_loss": 3.715966501721037, + "tokens_seen": 897056768 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005884373119358075, + "loss": 1.4756, + "theoretical_loss": 3.7159403480382043, + "tokens_seen": 897122304 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005884212637913742, + "loss": 1.5951, + "theoretical_loss": 3.715914196800777, + "tokens_seen": 897187840 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005884052156469409, + "loss": 1.4055, + "theoretical_loss": 3.715888048008347, + "tokens_seen": 897253376 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005883891675025076, + "loss": 1.4714, + "theoretical_loss": 3.7158619016605074, + "tokens_seen": 897318912 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005883731193580743, + "loss": 1.5126, + "theoretical_loss": 3.715835757756852, + "tokens_seen": 897384448 + }, + { + "epoch": 0.27, + "learning_rate": 0.000588357071213641, + "loss": 1.3888, + "theoretical_loss": 3.7158096162969727, + "tokens_seen": 897449984 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005883410230692077, + "loss": 1.3785, + "theoretical_loss": 3.715783477280463, + "tokens_seen": 897515520 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005883249749247744, + "loss": 1.4401, + "theoretical_loss": 3.715757340706917, + "tokens_seen": 897581056 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005883089267803411, + "loss": 1.3129, + "theoretical_loss": 3.715731206575927, + "tokens_seen": 897646592 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005882928786359078, + "loss": 1.3388, + "theoretical_loss": 3.7157050748870866, + "tokens_seen": 897712128 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005882768304914745, + "loss": 1.375, + "theoretical_loss": 3.715678945639989, + "tokens_seen": 897777664 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.9933853149414062, + "objective/train/avg_token_score": 0.006614702753722668, + "objective/train/docs_used": 331577, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.436472773551941, + "objective/train/original_loss": 1.447684645652771, + "objective/train/theoretical_loss": 3.7156528188342284, + "objective/train/tokens_used": 918303200, + "objective/train/unlikelihood_loss": 0.01672881469130516, + "theoretical_loss": 3.7156528188342284, + "tokens_seen": 897843200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005882607823470412, + "loss": 1.4862, + "theoretical_loss": 3.7156528188342284, + "tokens_seen": 897843200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005882447342026079, + "loss": 1.4541, + "theoretical_loss": 3.7156266944693987, + "tokens_seen": 897908736 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005882286860581746, + "loss": 1.576, + "theoretical_loss": 3.7156005725450933, + "tokens_seen": 897974272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005882126379137413, + "loss": 1.3996, + "theoretical_loss": 3.7155744530609063, + "tokens_seen": 898039808 + }, + { + "epoch": 0.27, + "learning_rate": 0.000588196589769308, + "loss": 1.4797, + "theoretical_loss": 3.7155483360164316, + "tokens_seen": 898105344 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005881805416248747, + "loss": 1.3513, + "theoretical_loss": 3.7155222214112635, + "tokens_seen": 898170880 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005881644934804414, + "loss": 1.4105, + "theoretical_loss": 3.715496109244996, + "tokens_seen": 898236416 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005881484453360081, + "loss": 1.5678, + "theoretical_loss": 3.715469999517224, + "tokens_seen": 898301952 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005881323971915748, + "loss": 1.4605, + "theoretical_loss": 3.7154438922275403, + "tokens_seen": 898367488 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005881163490471415, + "loss": 1.4626, + "theoretical_loss": 3.7154177873755416, + "tokens_seen": 898433024 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005881003009027082, + "loss": 1.4326, + "theoretical_loss": 3.715391684960821, + "tokens_seen": 898498560 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005880842527582749, + "loss": 1.4466, + "theoretical_loss": 3.715365584982973, + "tokens_seen": 898564096 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005880682046138416, + "loss": 1.4521, + "theoretical_loss": 3.7153394874415935, + "tokens_seen": 898629632 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005880521564694083, + "loss": 1.4081, + "theoretical_loss": 3.7153133923362764, + "tokens_seen": 898695168 + }, + { + "epoch": 0.27, + "learning_rate": 0.000588036108324975, + "loss": 1.4074, + "theoretical_loss": 3.7152872996666177, + "tokens_seen": 898760704 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005880200601805417, + "loss": 1.4918, + "theoretical_loss": 3.715261209432212, + "tokens_seen": 898826240 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005880040120361084, + "loss": 1.4892, + "theoretical_loss": 3.715235121632654, + "tokens_seen": 898891776 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005879879638916751, + "loss": 1.4354, + "theoretical_loss": 3.7152090362675394, + "tokens_seen": 898957312 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005879719157472418, + "loss": 1.4289, + "theoretical_loss": 3.7151829533364635, + "tokens_seen": 899022848 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005879558676028084, + "loss": 1.534, + "theoretical_loss": 3.715156872839022, + "tokens_seen": 899088384 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005879398194583752, + "loss": 1.3977, + "theoretical_loss": 3.7151307947748107, + "tokens_seen": 899153920 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005879237713139418, + "loss": 1.4626, + "theoretical_loss": 3.715104719143424, + "tokens_seen": 899219456 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005879077231695086, + "loss": 1.4014, + "theoretical_loss": 3.715078645944459, + "tokens_seen": 899284992 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005878916750250752, + "loss": 1.4166, + "theoretical_loss": 3.715052575177511, + "tokens_seen": 899350528 + }, + { + "epoch": 0.27, + "learning_rate": 0.000587875626880642, + "loss": 1.4227, + "theoretical_loss": 3.715026506842176, + "tokens_seen": 899416064 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005878595787362086, + "loss": 1.3629, + "theoretical_loss": 3.7150004409380504, + "tokens_seen": 899481600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005878435305917754, + "loss": 1.5489, + "theoretical_loss": 3.7149743774647295, + "tokens_seen": 899547136 + }, + { + "epoch": 0.27, + "learning_rate": 0.000587827482447342, + "loss": 1.4575, + "theoretical_loss": 3.7149483164218102, + "tokens_seen": 899612672 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005878114343029087, + "loss": 1.3848, + "theoretical_loss": 3.714922257808889, + "tokens_seen": 899678208 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005877953861584754, + "loss": 1.4155, + "theoretical_loss": 3.7148962016255616, + "tokens_seen": 899743744 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005877793380140421, + "loss": 1.5281, + "theoretical_loss": 3.714870147871425, + "tokens_seen": 899809280 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005877632898696088, + "loss": 1.4031, + "theoretical_loss": 3.7148440965460763, + "tokens_seen": 899874816 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005877472417251755, + "loss": 1.3603, + "theoretical_loss": 3.7148180476491115, + "tokens_seen": 899940352 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005877311935807422, + "loss": 1.4812, + "theoretical_loss": 3.714792001180127, + "tokens_seen": 900005888 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005877151454363089, + "loss": 1.4488, + "theoretical_loss": 3.714765957138721, + "tokens_seen": 900071424 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005876990972918756, + "loss": 1.4056, + "theoretical_loss": 3.71473991552449, + "tokens_seen": 900136960 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005876830491474423, + "loss": 1.3489, + "theoretical_loss": 3.71471387633703, + "tokens_seen": 900202496 + }, + { + "epoch": 0.27, + "learning_rate": 0.000587667001003009, + "loss": 1.5141, + "theoretical_loss": 3.7146878395759404, + "tokens_seen": 900268032 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005876509528585757, + "loss": 1.5637, + "theoretical_loss": 3.7146618052408167, + "tokens_seen": 900333568 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005876349047141424, + "loss": 1.4688, + "theoretical_loss": 3.714635773331257, + "tokens_seen": 900399104 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005876188565697091, + "loss": 1.5055, + "theoretical_loss": 3.7146097438468586, + "tokens_seen": 900464640 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005876028084252758, + "loss": 1.4759, + "theoretical_loss": 3.714583716787219, + "tokens_seen": 900530176 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005875867602808425, + "loss": 1.3894, + "theoretical_loss": 3.7145576921519363, + "tokens_seen": 900595712 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005875707121364092, + "loss": 1.4191, + "theoretical_loss": 3.714531669940608, + "tokens_seen": 900661248 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005875546639919759, + "loss": 1.4736, + "theoretical_loss": 3.714505650152832, + "tokens_seen": 900726784 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005875386158475426, + "loss": 1.3987, + "theoretical_loss": 3.7144796327882066, + "tokens_seen": 900792320 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005875225677031093, + "loss": 1.4164, + "theoretical_loss": 3.714453617846329, + "tokens_seen": 900857856 + }, + { + "epoch": 0.27, + "learning_rate": 0.000587506519558676, + "loss": 1.4428, + "theoretical_loss": 3.714427605326798, + "tokens_seen": 900923392 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005874904714142427, + "loss": 1.5368, + "theoretical_loss": 3.714401595229212, + "tokens_seen": 900988928 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005874744232698094, + "loss": 1.4156, + "theoretical_loss": 3.714375587553169, + "tokens_seen": 901054464 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.994195282459259, + "objective/train/avg_token_score": 0.005804713349789381, + "objective/train/docs_used": 332819, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.567984938621521, + "objective/train/original_loss": 1.5783530473709106, + "objective/train/theoretical_loss": 3.714349582298268, + "objective/train/tokens_used": 921580000, + "objective/train/unlikelihood_loss": 0.012281695380806923, + "theoretical_loss": 3.714349582298268, + "tokens_seen": 901120000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005874583751253761, + "loss": 1.5109, + "theoretical_loss": 3.714349582298268, + "tokens_seen": 901120000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005874423269809428, + "loss": 1.3721, + "theoretical_loss": 3.7143235794641067, + "tokens_seen": 901185536 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005874262788365095, + "loss": 1.4581, + "theoretical_loss": 3.7142975790502843, + "tokens_seen": 901251072 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005874102306920762, + "loss": 1.5074, + "theoretical_loss": 3.7142715810563995, + "tokens_seen": 901316608 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005873941825476429, + "loss": 1.4788, + "theoretical_loss": 3.7142455854820513, + "tokens_seen": 901382144 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005873781344032096, + "loss": 1.4362, + "theoretical_loss": 3.714219592326838, + "tokens_seen": 901447680 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005873620862587764, + "loss": 1.4618, + "theoretical_loss": 3.714193601590359, + "tokens_seen": 901513216 + }, + { + "epoch": 0.27, + "learning_rate": 0.000587346038114343, + "loss": 1.5158, + "theoretical_loss": 3.7141676132722137, + "tokens_seen": 901578752 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005873299899699098, + "loss": 1.4503, + "theoretical_loss": 3.714141627372001, + "tokens_seen": 901644288 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005873139418254764, + "loss": 1.4432, + "theoretical_loss": 3.7141156438893206, + "tokens_seen": 901709824 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005872978936810432, + "loss": 1.4962, + "theoretical_loss": 3.7140896628237714, + "tokens_seen": 901775360 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005872818455366098, + "loss": 1.4063, + "theoretical_loss": 3.7140636841749526, + "tokens_seen": 901840896 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005872657973921766, + "loss": 1.3884, + "theoretical_loss": 3.714037707942465, + "tokens_seen": 901906432 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005872497492477432, + "loss": 1.539, + "theoretical_loss": 3.714011734125907, + "tokens_seen": 901971968 + }, + { + "epoch": 0.27, + "learning_rate": 0.00058723370110331, + "loss": 1.4751, + "theoretical_loss": 3.7139857627248793, + "tokens_seen": 902037504 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005872176529588767, + "loss": 1.5001, + "theoretical_loss": 3.7139597937389817, + "tokens_seen": 902103040 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005872016048144434, + "loss": 1.5185, + "theoretical_loss": 3.7139338271678133, + "tokens_seen": 902168576 + }, + { + "epoch": 0.27, + "learning_rate": 0.00058718555667001, + "loss": 1.4149, + "theoretical_loss": 3.7139078630109754, + "tokens_seen": 902234112 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005871695085255768, + "loss": 1.352, + "theoretical_loss": 3.713881901268068, + "tokens_seen": 902299648 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005871534603811435, + "loss": 1.4532, + "theoretical_loss": 3.7138559419386903, + "tokens_seen": 902365184 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005871374122367102, + "loss": 1.4212, + "theoretical_loss": 3.713829985022443, + "tokens_seen": 902430720 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005871213640922769, + "loss": 1.3911, + "theoretical_loss": 3.7138040305189275, + "tokens_seen": 902496256 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005871053159478436, + "loss": 1.5622, + "theoretical_loss": 3.713778078427744, + "tokens_seen": 902561792 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005870892678034103, + "loss": 1.4473, + "theoretical_loss": 3.7137521287484923, + "tokens_seen": 902627328 + }, + { + "epoch": 0.27, + "learning_rate": 0.000587073219658977, + "loss": 1.4254, + "theoretical_loss": 3.713726181480774, + "tokens_seen": 902692864 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005870571715145437, + "loss": 1.4155, + "theoretical_loss": 3.71370023662419, + "tokens_seen": 902758400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005870411233701104, + "loss": 1.5194, + "theoretical_loss": 3.7136742941783405, + "tokens_seen": 902823936 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005870250752256771, + "loss": 1.4456, + "theoretical_loss": 3.713648354142827, + "tokens_seen": 902889472 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005870090270812438, + "loss": 1.5713, + "theoretical_loss": 3.713622416517251, + "tokens_seen": 902955008 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005869929789368105, + "loss": 1.4178, + "theoretical_loss": 3.7135964813012126, + "tokens_seen": 903020544 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005869769307923772, + "loss": 1.3915, + "theoretical_loss": 3.7135705484943142, + "tokens_seen": 903086080 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005869608826479439, + "loss": 1.5576, + "theoretical_loss": 3.713544618096157, + "tokens_seen": 903151616 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005869448345035106, + "loss": 1.2979, + "theoretical_loss": 3.713518690106342, + "tokens_seen": 903217152 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005869287863590773, + "loss": 1.4722, + "theoretical_loss": 3.7134927645244713, + "tokens_seen": 903282688 + }, + { + "epoch": 0.27, + "learning_rate": 0.000586912738214644, + "loss": 1.3787, + "theoretical_loss": 3.7134668413501464, + "tokens_seen": 903348224 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005868966900702107, + "loss": 1.4404, + "theoretical_loss": 3.713440920582969, + "tokens_seen": 903413760 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005868806419257774, + "loss": 1.3815, + "theoretical_loss": 3.713415002222541, + "tokens_seen": 903479296 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005868645937813441, + "loss": 1.3322, + "theoretical_loss": 3.7133890862684646, + "tokens_seen": 903544832 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005868485456369108, + "loss": 1.4291, + "theoretical_loss": 3.7133631727203418, + "tokens_seen": 903610368 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005868324974924775, + "loss": 1.5939, + "theoretical_loss": 3.7133372615777747, + "tokens_seen": 903675904 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005868164493480442, + "loss": 1.5436, + "theoretical_loss": 3.713311352840366, + "tokens_seen": 903741440 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005868004012036109, + "loss": 1.3498, + "theoretical_loss": 3.713285446507717, + "tokens_seen": 903806976 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005867843530591776, + "loss": 1.4807, + "theoretical_loss": 3.7132595425794306, + "tokens_seen": 903872512 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005867683049147443, + "loss": 1.4902, + "theoretical_loss": 3.7132336410551097, + "tokens_seen": 903938048 + }, + { + "epoch": 0.27, + "learning_rate": 0.000586752256770311, + "loss": 1.5222, + "theoretical_loss": 3.713207741934357, + "tokens_seen": 904003584 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005867362086258777, + "loss": 1.437, + "theoretical_loss": 3.713181845216775, + "tokens_seen": 904069120 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005867201604814444, + "loss": 1.3965, + "theoretical_loss": 3.7131559509019665, + "tokens_seen": 904134656 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005867041123370111, + "loss": 1.3606, + "theoretical_loss": 3.713130058989534, + "tokens_seen": 904200192 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005866880641925778, + "loss": 1.3417, + "theoretical_loss": 3.713104169479082, + "tokens_seen": 904265728 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005866720160481445, + "loss": 1.4754, + "theoretical_loss": 3.713078282370211, + "tokens_seen": 904331264 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.9902017712593079, + "objective/train/avg_token_score": 0.009798144921660423, + "objective/train/docs_used": 333943, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.3829900026321411, + "objective/train/original_loss": 1.3980164527893066, + "objective/train/theoretical_loss": 3.713052397662527, + "objective/train/tokens_used": 924856800, + "objective/train/unlikelihood_loss": 0.02360416017472744, + "theoretical_loss": 3.713052397662527, + "tokens_seen": 904396800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005866559679037112, + "loss": 1.3926, + "theoretical_loss": 3.713052397662527, + "tokens_seen": 904396800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005866399197592779, + "loss": 1.4329, + "theoretical_loss": 3.713026515355632, + "tokens_seen": 904462336 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005866238716148446, + "loss": 1.3905, + "theoretical_loss": 3.713000635449129, + "tokens_seen": 904527872 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005866078234704113, + "loss": 1.3444, + "theoretical_loss": 3.7129747579426224, + "tokens_seen": 904593408 + }, + { + "epoch": 0.27, + "learning_rate": 0.000586591775325978, + "loss": 1.429, + "theoretical_loss": 3.712948882835715, + "tokens_seen": 904658944 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005865757271815447, + "loss": 1.4867, + "theoretical_loss": 3.7129230101280113, + "tokens_seen": 904724480 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005865596790371113, + "loss": 1.4131, + "theoretical_loss": 3.7128971398191144, + "tokens_seen": 904790016 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005865436308926781, + "loss": 1.5921, + "theoretical_loss": 3.712871271908628, + "tokens_seen": 904855552 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005865275827482447, + "loss": 1.4439, + "theoretical_loss": 3.7128454063961565, + "tokens_seen": 904921088 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005865115346038115, + "loss": 1.4489, + "theoretical_loss": 3.712819543281304, + "tokens_seen": 904986624 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005864954864593781, + "loss": 1.4599, + "theoretical_loss": 3.712793682563675, + "tokens_seen": 905052160 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005864794383149449, + "loss": 1.4375, + "theoretical_loss": 3.712767824242873, + "tokens_seen": 905117696 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005864633901705115, + "loss": 1.3855, + "theoretical_loss": 3.712741968318502, + "tokens_seen": 905183232 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005864473420260783, + "loss": 1.4666, + "theoretical_loss": 3.712716114790168, + "tokens_seen": 905248768 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005864312938816449, + "loss": 1.4161, + "theoretical_loss": 3.712690263657474, + "tokens_seen": 905314304 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005864152457372116, + "loss": 1.4119, + "theoretical_loss": 3.7126644149200247, + "tokens_seen": 905379840 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005863991975927783, + "loss": 1.5846, + "theoretical_loss": 3.712638568577426, + "tokens_seen": 905445376 + }, + { + "epoch": 0.27, + "learning_rate": 0.000586383149448345, + "loss": 1.4156, + "theoretical_loss": 3.712612724629282, + "tokens_seen": 905510912 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005863671013039117, + "loss": 1.3694, + "theoretical_loss": 3.712586883075198, + "tokens_seen": 905576448 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005863510531594784, + "loss": 1.358, + "theoretical_loss": 3.7125610439147776, + "tokens_seen": 905641984 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005863350050150451, + "loss": 1.4794, + "theoretical_loss": 3.7125352071476274, + "tokens_seen": 905707520 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005863189568706118, + "loss": 1.3448, + "theoretical_loss": 3.7125093727733516, + "tokens_seen": 905773056 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005863029087261785, + "loss": 1.3238, + "theoretical_loss": 3.7124835407915566, + "tokens_seen": 905838592 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005862868605817452, + "loss": 1.5009, + "theoretical_loss": 3.712457711201847, + "tokens_seen": 905904128 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005862708124373119, + "loss": 1.5385, + "theoretical_loss": 3.712431884003828, + "tokens_seen": 905969664 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005862547642928786, + "loss": 1.4149, + "theoretical_loss": 3.7124060591971055, + "tokens_seen": 906035200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005862387161484453, + "loss": 1.4655, + "theoretical_loss": 3.7123802367812857, + "tokens_seen": 906100736 + }, + { + "epoch": 0.27, + "learning_rate": 0.000586222668004012, + "loss": 1.3468, + "theoretical_loss": 3.712354416755973, + "tokens_seen": 906166272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005862066198595787, + "loss": 1.5339, + "theoretical_loss": 3.7123285991207746, + "tokens_seen": 906231808 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005861905717151454, + "loss": 1.4543, + "theoretical_loss": 3.712302783875296, + "tokens_seen": 906297344 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005861745235707121, + "loss": 1.3991, + "theoretical_loss": 3.7122769710191426, + "tokens_seen": 906362880 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005861584754262789, + "loss": 1.5038, + "theoretical_loss": 3.7122511605519217, + "tokens_seen": 906428416 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005861424272818455, + "loss": 1.2757, + "theoretical_loss": 3.7122253524732383, + "tokens_seen": 906493952 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005861263791374123, + "loss": 1.3537, + "theoretical_loss": 3.712199546782699, + "tokens_seen": 906559488 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005861103309929789, + "loss": 1.4119, + "theoretical_loss": 3.7121737434799114, + "tokens_seen": 906625024 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005860942828485457, + "loss": 1.4314, + "theoretical_loss": 3.71214794256448, + "tokens_seen": 906690560 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005860782347041123, + "loss": 1.418, + "theoretical_loss": 3.712122144036013, + "tokens_seen": 906756096 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005860621865596791, + "loss": 1.5179, + "theoretical_loss": 3.712096347894116, + "tokens_seen": 906821632 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005860461384152457, + "loss": 1.3757, + "theoretical_loss": 3.7120705541383967, + "tokens_seen": 906887168 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005860300902708125, + "loss": 1.398, + "theoretical_loss": 3.7120447627684614, + "tokens_seen": 906952704 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005860140421263791, + "loss": 1.5148, + "theoretical_loss": 3.712018973783917, + "tokens_seen": 907018240 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005859979939819459, + "loss": 1.4137, + "theoretical_loss": 3.7119931871843708, + "tokens_seen": 907083776 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005859819458375125, + "loss": 1.3288, + "theoretical_loss": 3.71196740296943, + "tokens_seen": 907149312 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005859658976930793, + "loss": 1.4883, + "theoretical_loss": 3.7119416211387013, + "tokens_seen": 907214848 + }, + { + "epoch": 0.27, + "learning_rate": 0.000585949849548646, + "loss": 1.4568, + "theoretical_loss": 3.7119158416917926, + "tokens_seen": 907280384 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005859338014042127, + "loss": 1.3775, + "theoretical_loss": 3.711890064628311, + "tokens_seen": 907345920 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005859177532597793, + "loss": 1.4096, + "theoretical_loss": 3.711864289947864, + "tokens_seen": 907411456 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005859017051153461, + "loss": 1.4167, + "theoretical_loss": 3.71183851765006, + "tokens_seen": 907476992 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005858856569709127, + "loss": 1.4293, + "theoretical_loss": 3.7118127477345055, + "tokens_seen": 907542528 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005858696088264795, + "loss": 1.3841, + "theoretical_loss": 3.7117869802008094, + "tokens_seen": 907608064 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.9930924773216248, + "objective/train/avg_token_score": 0.006907503120601177, + "objective/train/docs_used": 335091, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.3492159843444824, + "objective/train/original_loss": 1.3591556549072266, + "objective/train/theoretical_loss": 3.7117612150485786, + "objective/train/tokens_used": 928133600, + "objective/train/unlikelihood_loss": 0.01792854070663452, + "theoretical_loss": 3.7117612150485786, + "tokens_seen": 907673600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005858535606820462, + "loss": 1.3757, + "theoretical_loss": 3.7117612150485786, + "tokens_seen": 907673600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005858375125376129, + "loss": 1.4391, + "theoretical_loss": 3.7117354522774217, + "tokens_seen": 907739136 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005858214643931796, + "loss": 1.4203, + "theoretical_loss": 3.7117096918869468, + "tokens_seen": 907804672 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005858054162487463, + "loss": 1.4906, + "theoretical_loss": 3.7116839338767615, + "tokens_seen": 907870208 + }, + { + "epoch": 0.28, + "learning_rate": 0.000585789368104313, + "loss": 1.3774, + "theoretical_loss": 3.7116581782464744, + "tokens_seen": 907935744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005857733199598797, + "loss": 1.5118, + "theoretical_loss": 3.711632424995694, + "tokens_seen": 908001280 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005857572718154464, + "loss": 1.5334, + "theoretical_loss": 3.7116066741240283, + "tokens_seen": 908066816 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005857412236710131, + "loss": 1.3676, + "theoretical_loss": 3.7115809256310865, + "tokens_seen": 908132352 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005857251755265798, + "loss": 1.3843, + "theoretical_loss": 3.711555179516477, + "tokens_seen": 908197888 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005857091273821465, + "loss": 1.3448, + "theoretical_loss": 3.7115294357798083, + "tokens_seen": 908263424 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005856930792377132, + "loss": 1.4431, + "theoretical_loss": 3.7115036944206885, + "tokens_seen": 908328960 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005856770310932799, + "loss": 1.517, + "theoretical_loss": 3.7114779554387285, + "tokens_seen": 908394496 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005856609829488466, + "loss": 1.4854, + "theoretical_loss": 3.7114522188335357, + "tokens_seen": 908460032 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005856449348044133, + "loss": 1.394, + "theoretical_loss": 3.7114264846047194, + "tokens_seen": 908525568 + }, + { + "epoch": 0.28, + "learning_rate": 0.00058562888665998, + "loss": 1.4351, + "theoretical_loss": 3.7114007527518886, + "tokens_seen": 908591104 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005856128385155467, + "loss": 1.3891, + "theoretical_loss": 3.7113750232746536, + "tokens_seen": 908656640 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005855967903711134, + "loss": 1.4523, + "theoretical_loss": 3.711349296172623, + "tokens_seen": 908722176 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005855807422266801, + "loss": 1.4268, + "theoretical_loss": 3.7113235714454063, + "tokens_seen": 908787712 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005855646940822468, + "loss": 1.4876, + "theoretical_loss": 3.711297849092613, + "tokens_seen": 908853248 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005855486459378135, + "loss": 1.4353, + "theoretical_loss": 3.711272129113853, + "tokens_seen": 908918784 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005855325977933802, + "loss": 1.3787, + "theoretical_loss": 3.7112464115087365, + "tokens_seen": 908984320 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005855165496489469, + "loss": 1.4181, + "theoretical_loss": 3.7112206962768717, + "tokens_seen": 909049856 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005855005015045136, + "loss": 1.3098, + "theoretical_loss": 3.7111949834178706, + "tokens_seen": 909115392 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005854844533600803, + "loss": 1.3774, + "theoretical_loss": 3.7111692729313415, + "tokens_seen": 909180928 + }, + { + "epoch": 0.28, + "learning_rate": 0.000585468405215647, + "loss": 1.4875, + "theoretical_loss": 3.7111435648168953, + "tokens_seen": 909246464 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005854523570712137, + "loss": 1.4691, + "theoretical_loss": 3.7111178590741423, + "tokens_seen": 909312000 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005854363089267804, + "loss": 1.368, + "theoretical_loss": 3.7110921557026924, + "tokens_seen": 909377536 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005854202607823471, + "loss": 1.5036, + "theoretical_loss": 3.7110664547021566, + "tokens_seen": 909443072 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005854042126379138, + "loss": 1.3298, + "theoretical_loss": 3.711040756072144, + "tokens_seen": 909508608 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005853881644934805, + "loss": 1.5999, + "theoretical_loss": 3.711015059812267, + "tokens_seen": 909574144 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005853721163490472, + "loss": 1.3733, + "theoretical_loss": 3.710989365922135, + "tokens_seen": 909639680 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005853560682046139, + "loss": 1.4766, + "theoretical_loss": 3.710963674401359, + "tokens_seen": 909705216 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005853400200601806, + "loss": 1.4417, + "theoretical_loss": 3.7109379852495503, + "tokens_seen": 909770752 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005853239719157473, + "loss": 1.4672, + "theoretical_loss": 3.7109122984663196, + "tokens_seen": 909836288 + }, + { + "epoch": 0.28, + "learning_rate": 0.000585307923771314, + "loss": 1.4328, + "theoretical_loss": 3.7108866140512777, + "tokens_seen": 909901824 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005852918756268807, + "loss": 1.3856, + "theoretical_loss": 3.7108609320040356, + "tokens_seen": 909967360 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005852758274824474, + "loss": 1.4602, + "theoretical_loss": 3.7108352523242045, + "tokens_seen": 910032896 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005852597793380141, + "loss": 1.4498, + "theoretical_loss": 3.710809575011397, + "tokens_seen": 910098432 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005852437311935808, + "loss": 1.4391, + "theoretical_loss": 3.7107839000652225, + "tokens_seen": 910163968 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005852276830491475, + "loss": 1.4898, + "theoretical_loss": 3.710758227485294, + "tokens_seen": 910229504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005852116349047142, + "loss": 1.38, + "theoretical_loss": 3.710732557271222, + "tokens_seen": 910295040 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005851955867602809, + "loss": 1.3804, + "theoretical_loss": 3.710706889422619, + "tokens_seen": 910360576 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005851795386158476, + "loss": 1.4584, + "theoretical_loss": 3.7106812239390967, + "tokens_seen": 910426112 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005851634904714142, + "loss": 1.3525, + "theoretical_loss": 3.7106555608202667, + "tokens_seen": 910491648 + }, + { + "epoch": 0.28, + "learning_rate": 0.000585147442326981, + "loss": 1.4309, + "theoretical_loss": 3.71062990006574, + "tokens_seen": 910557184 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005851313941825476, + "loss": 1.3603, + "theoretical_loss": 3.7106042416751306, + "tokens_seen": 910622720 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005851153460381144, + "loss": 1.4185, + "theoretical_loss": 3.7105785856480495, + "tokens_seen": 910688256 + }, + { + "epoch": 0.28, + "learning_rate": 0.000585099297893681, + "loss": 1.4117, + "theoretical_loss": 3.710552931984109, + "tokens_seen": 910753792 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005850832497492478, + "loss": 1.3651, + "theoretical_loss": 3.7105272806829213, + "tokens_seen": 910819328 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005850672016048144, + "loss": 1.3882, + "theoretical_loss": 3.7105016317440986, + "tokens_seen": 910884864 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.995114803314209, + "objective/train/avg_token_score": 0.004885133821517229, + "objective/train/docs_used": 336141, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4863137006759644, + "objective/train/original_loss": 1.4945149421691895, + "objective/train/theoretical_loss": 3.710475985167254, + "objective/train/tokens_used": 931410400, + "objective/train/unlikelihood_loss": 0.011246008798480034, + "theoretical_loss": 3.710475985167254, + "tokens_seen": 910950400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005850511534603812, + "loss": 1.4992, + "theoretical_loss": 3.710475985167254, + "tokens_seen": 910950400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005850351053159478, + "loss": 1.4005, + "theoretical_loss": 3.710450340952, + "tokens_seen": 911015936 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005850190571715145, + "loss": 1.4079, + "theoretical_loss": 3.710424699097949, + "tokens_seen": 911081472 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005850030090270812, + "loss": 1.4166, + "theoretical_loss": 3.710399059604714, + "tokens_seen": 911147008 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005849869608826479, + "loss": 1.4507, + "theoretical_loss": 3.710373422471908, + "tokens_seen": 911212544 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005849709127382146, + "loss": 1.3258, + "theoretical_loss": 3.710347787699143, + "tokens_seen": 911278080 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005849548645937814, + "loss": 1.3496, + "theoretical_loss": 3.710322155286033, + "tokens_seen": 911343616 + }, + { + "epoch": 0.28, + "learning_rate": 0.000584938816449348, + "loss": 1.397, + "theoretical_loss": 3.710296525232191, + "tokens_seen": 911409152 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005849227683049148, + "loss": 1.3821, + "theoretical_loss": 3.71027089753723, + "tokens_seen": 911474688 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005849067201604814, + "loss": 1.3738, + "theoretical_loss": 3.7102452722007637, + "tokens_seen": 911540224 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005848906720160482, + "loss": 1.2968, + "theoretical_loss": 3.710219649222404, + "tokens_seen": 911605760 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005848746238716148, + "loss": 1.4133, + "theoretical_loss": 3.710194028601767, + "tokens_seen": 911671296 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005848585757271816, + "loss": 1.4617, + "theoretical_loss": 3.7101684103384645, + "tokens_seen": 911736832 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005848425275827482, + "loss": 1.3418, + "theoretical_loss": 3.7101427944321106, + "tokens_seen": 911802368 + }, + { + "epoch": 0.28, + "learning_rate": 0.000584826479438315, + "loss": 1.5474, + "theoretical_loss": 3.7101171808823192, + "tokens_seen": 911867904 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005848104312938816, + "loss": 1.4754, + "theoretical_loss": 3.7100915696887036, + "tokens_seen": 911933440 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005847943831494484, + "loss": 1.3521, + "theoretical_loss": 3.710065960850878, + "tokens_seen": 911998976 + }, + { + "epoch": 0.28, + "learning_rate": 0.000584778335005015, + "loss": 1.455, + "theoretical_loss": 3.710040354368457, + "tokens_seen": 912064512 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005847622868605818, + "loss": 1.5232, + "theoretical_loss": 3.7100147502410543, + "tokens_seen": 912130048 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005847462387161484, + "loss": 1.4573, + "theoretical_loss": 3.709989148468284, + "tokens_seen": 912195584 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005847301905717152, + "loss": 1.446, + "theoretical_loss": 3.7099635490497604, + "tokens_seen": 912261120 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005847141424272818, + "loss": 1.4323, + "theoretical_loss": 3.709937951985099, + "tokens_seen": 912326656 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005846980942828486, + "loss": 1.5375, + "theoretical_loss": 3.7099123572739128, + "tokens_seen": 912392192 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005846820461384152, + "loss": 1.3677, + "theoretical_loss": 3.709886764915817, + "tokens_seen": 912457728 + }, + { + "epoch": 0.28, + "learning_rate": 0.000584665997993982, + "loss": 1.4191, + "theoretical_loss": 3.7098611749104258, + "tokens_seen": 912523264 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005846499498495486, + "loss": 1.4406, + "theoretical_loss": 3.709835587257355, + "tokens_seen": 912588800 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005846339017051154, + "loss": 1.4194, + "theoretical_loss": 3.7098100019562192, + "tokens_seen": 912654336 + }, + { + "epoch": 0.28, + "learning_rate": 0.000584617853560682, + "loss": 1.3046, + "theoretical_loss": 3.7097844190066325, + "tokens_seen": 912719872 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005846018054162488, + "loss": 1.4189, + "theoretical_loss": 3.7097588384082107, + "tokens_seen": 912785408 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005845857572718154, + "loss": 1.4609, + "theoretical_loss": 3.7097332601605686, + "tokens_seen": 912850944 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005845697091273822, + "loss": 1.3607, + "theoretical_loss": 3.7097076842633214, + "tokens_seen": 912916480 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005845536609829488, + "loss": 1.3628, + "theoretical_loss": 3.709682110716085, + "tokens_seen": 912982016 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005845376128385156, + "loss": 1.4621, + "theoretical_loss": 3.709656539518474, + "tokens_seen": 913047552 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005845215646940822, + "loss": 1.443, + "theoretical_loss": 3.709630970670105, + "tokens_seen": 913113088 + }, + { + "epoch": 0.28, + "learning_rate": 0.000584505516549649, + "loss": 1.4786, + "theoretical_loss": 3.709605404170592, + "tokens_seen": 913178624 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005844894684052156, + "loss": 1.4222, + "theoretical_loss": 3.7095798400195523, + "tokens_seen": 913244160 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005844734202607824, + "loss": 1.4293, + "theoretical_loss": 3.7095542782166, + "tokens_seen": 913309696 + }, + { + "epoch": 0.28, + "learning_rate": 0.000584457372116349, + "loss": 1.3778, + "theoretical_loss": 3.7095287187613524, + "tokens_seen": 913375232 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005844413239719158, + "loss": 1.3488, + "theoretical_loss": 3.709503161653425, + "tokens_seen": 913440768 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005844252758274825, + "loss": 1.518, + "theoretical_loss": 3.7094776068924338, + "tokens_seen": 913506304 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005844092276830492, + "loss": 1.4182, + "theoretical_loss": 3.7094520544779948, + "tokens_seen": 913571840 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005843931795386159, + "loss": 1.4695, + "theoretical_loss": 3.7094265044097243, + "tokens_seen": 913637376 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005843771313941826, + "loss": 1.5022, + "theoretical_loss": 3.7094009566872383, + "tokens_seen": 913702912 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005843610832497493, + "loss": 1.3849, + "theoretical_loss": 3.7093754113101536, + "tokens_seen": 913768448 + }, + { + "epoch": 0.28, + "learning_rate": 0.000584345035105316, + "loss": 1.3134, + "theoretical_loss": 3.709349868278087, + "tokens_seen": 913833984 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005843289869608827, + "loss": 1.3767, + "theoretical_loss": 3.7093243275906547, + "tokens_seen": 913899520 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005843129388164494, + "loss": 1.4626, + "theoretical_loss": 3.7092987892474727, + "tokens_seen": 913965056 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005842968906720161, + "loss": 1.4661, + "theoretical_loss": 3.709273253248159, + "tokens_seen": 914030592 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005842808425275828, + "loss": 1.3739, + "theoretical_loss": 3.7092477195923292, + "tokens_seen": 914096128 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005842647943831495, + "loss": 1.3577, + "theoretical_loss": 3.709222188279602, + "tokens_seen": 914161664 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.989127516746521, + "objective/train/avg_token_score": 0.01087252702564001, + "objective/train/docs_used": 337261, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4259281158447266, + "objective/train/original_loss": 1.442481517791748, + "objective/train/theoretical_loss": 3.709196659309592, + "objective/train/tokens_used": 934687200, + "objective/train/unlikelihood_loss": 0.025753257796168327, + "theoretical_loss": 3.709196659309592, + "tokens_seen": 914227200 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005842487462387162, + "loss": 1.5034, + "theoretical_loss": 3.709196659309592, + "tokens_seen": 914227200 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005842326980942829, + "loss": 1.3826, + "theoretical_loss": 3.7091711326819192, + "tokens_seen": 914292736 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005842166499498496, + "loss": 1.4421, + "theoretical_loss": 3.7091456083961982, + "tokens_seen": 914358272 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005842006018054163, + "loss": 1.3935, + "theoretical_loss": 3.7091200864520477, + "tokens_seen": 914423808 + }, + { + "epoch": 0.28, + "learning_rate": 0.000584184553660983, + "loss": 1.4348, + "theoretical_loss": 3.709094566849085, + "tokens_seen": 914489344 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005841685055165497, + "loss": 1.4113, + "theoretical_loss": 3.709069049586927, + "tokens_seen": 914554880 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005841524573721164, + "loss": 1.4232, + "theoretical_loss": 3.7090435346651915, + "tokens_seen": 914620416 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005841364092276831, + "loss": 1.4206, + "theoretical_loss": 3.7090180220834967, + "tokens_seen": 914685952 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005841203610832498, + "loss": 1.3986, + "theoretical_loss": 3.7089925118414597, + "tokens_seen": 914751488 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005841043129388165, + "loss": 1.4055, + "theoretical_loss": 3.7089670039386986, + "tokens_seen": 914817024 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005840882647943832, + "loss": 1.3969, + "theoretical_loss": 3.7089414983748314, + "tokens_seen": 914882560 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005840722166499499, + "loss": 1.3471, + "theoretical_loss": 3.708915995149476, + "tokens_seen": 914948096 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005840561685055166, + "loss": 1.418, + "theoretical_loss": 3.70889049426225, + "tokens_seen": 915013632 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005840401203610833, + "loss": 1.4562, + "theoretical_loss": 3.708864995712773, + "tokens_seen": 915079168 + }, + { + "epoch": 0.28, + "learning_rate": 0.00058402407221665, + "loss": 1.4495, + "theoretical_loss": 3.708839499500662, + "tokens_seen": 915144704 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005840080240722167, + "loss": 1.46, + "theoretical_loss": 3.708814005625536, + "tokens_seen": 915210240 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005839919759277834, + "loss": 1.4491, + "theoretical_loss": 3.708788514087013, + "tokens_seen": 915275776 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005839759277833501, + "loss": 1.438, + "theoretical_loss": 3.7087630248847123, + "tokens_seen": 915341312 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005839598796389168, + "loss": 1.4137, + "theoretical_loss": 3.7087375380182515, + "tokens_seen": 915406848 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005839438314944835, + "loss": 1.3589, + "theoretical_loss": 3.7087120534872504, + "tokens_seen": 915472384 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005839277833500502, + "loss": 1.4752, + "theoretical_loss": 3.708686571291327, + "tokens_seen": 915537920 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005839117352056169, + "loss": 1.4323, + "theoretical_loss": 3.7086610914301006, + "tokens_seen": 915603456 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005838956870611836, + "loss": 1.3611, + "theoretical_loss": 3.70863561390319, + "tokens_seen": 915668992 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005838796389167503, + "loss": 1.4367, + "theoretical_loss": 3.708610138710215, + "tokens_seen": 915734528 + }, + { + "epoch": 0.28, + "learning_rate": 0.000583863590772317, + "loss": 1.4328, + "theoretical_loss": 3.708584665850794, + "tokens_seen": 915800064 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005838475426278837, + "loss": 1.4078, + "theoretical_loss": 3.7085591953245465, + "tokens_seen": 915865600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005838314944834504, + "loss": 1.3896, + "theoretical_loss": 3.708533727131092, + "tokens_seen": 915931136 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005838154463390171, + "loss": 1.4268, + "theoretical_loss": 3.7085082612700493, + "tokens_seen": 915996672 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005837993981945838, + "loss": 1.4491, + "theoretical_loss": 3.7084827977410395, + "tokens_seen": 916062208 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005837833500501505, + "loss": 1.4302, + "theoretical_loss": 3.70845733654368, + "tokens_seen": 916127744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005837673019057172, + "loss": 1.5681, + "theoretical_loss": 3.708431877677593, + "tokens_seen": 916193280 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005837512537612839, + "loss": 1.4665, + "theoretical_loss": 3.708406421142396, + "tokens_seen": 916258816 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005837352056168505, + "loss": 1.4664, + "theoretical_loss": 3.7083809669377104, + "tokens_seen": 916324352 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005837191574724173, + "loss": 1.4791, + "theoretical_loss": 3.7083555150631557, + "tokens_seen": 916389888 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005837031093279839, + "loss": 1.3392, + "theoretical_loss": 3.708330065518352, + "tokens_seen": 916455424 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005836870611835507, + "loss": 1.5665, + "theoretical_loss": 3.70830461830292, + "tokens_seen": 916520960 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005836710130391173, + "loss": 1.2824, + "theoretical_loss": 3.7082791734164786, + "tokens_seen": 916586496 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005836549648946841, + "loss": 1.363, + "theoretical_loss": 3.708253730858649, + "tokens_seen": 916652032 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005836389167502507, + "loss": 1.4309, + "theoretical_loss": 3.7082282906290525, + "tokens_seen": 916717568 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005836228686058175, + "loss": 1.3915, + "theoretical_loss": 3.7082028527273083, + "tokens_seen": 916783104 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005836068204613841, + "loss": 1.3681, + "theoretical_loss": 3.708177417153037, + "tokens_seen": 916848640 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005835907723169509, + "loss": 1.4261, + "theoretical_loss": 3.70815198390586, + "tokens_seen": 916914176 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005835747241725175, + "loss": 1.4087, + "theoretical_loss": 3.708126552985398, + "tokens_seen": 916979712 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005835586760280843, + "loss": 1.2549, + "theoretical_loss": 3.7081011243912716, + "tokens_seen": 917045248 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005835426278836509, + "loss": 1.3853, + "theoretical_loss": 3.7080756981231016, + "tokens_seen": 917110784 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005835265797392177, + "loss": 1.3323, + "theoretical_loss": 3.7080502741805104, + "tokens_seen": 917176320 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005835105315947843, + "loss": 1.5226, + "theoretical_loss": 3.708024852563117, + "tokens_seen": 917241856 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005834944834503511, + "loss": 1.4249, + "theoretical_loss": 3.707999433270544, + "tokens_seen": 917307392 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005834784353059177, + "loss": 1.398, + "theoretical_loss": 3.7079740163024124, + "tokens_seen": 917372928 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005834623871614845, + "loss": 1.4464, + "theoretical_loss": 3.7079486016583436, + "tokens_seen": 917438464 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.9928538799285889, + "objective/train/avg_token_score": 0.007146106567233801, + "objective/train/docs_used": 338425, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.499230146408081, + "objective/train/original_loss": 1.5123393535614014, + "objective/train/theoretical_loss": 3.7079231893379596, + "objective/train/tokens_used": 937964000, + "objective/train/unlikelihood_loss": 0.01600590907037258, + "theoretical_loss": 3.7079231893379596, + "tokens_seen": 917504000 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005834463390170511, + "loss": 1.5274, + "theoretical_loss": 3.7079231893379596, + "tokens_seen": 917504000 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005834302908726179, + "loss": 1.3536, + "theoretical_loss": 3.707897779340881, + "tokens_seen": 917569536 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005834142427281845, + "loss": 1.468, + "theoretical_loss": 3.7078723716667303, + "tokens_seen": 917635072 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005833981945837513, + "loss": 1.363, + "theoretical_loss": 3.7078469663151292, + "tokens_seen": 917700608 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005833821464393179, + "loss": 1.3887, + "theoretical_loss": 3.707821563285699, + "tokens_seen": 917766144 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005833660982948847, + "loss": 1.4899, + "theoretical_loss": 3.707796162578062, + "tokens_seen": 917831680 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005833500501504513, + "loss": 1.4352, + "theoretical_loss": 3.7077707641918405, + "tokens_seen": 917897216 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005833340020060181, + "loss": 1.4788, + "theoretical_loss": 3.7077453681266563, + "tokens_seen": 917962752 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005833179538615847, + "loss": 1.3609, + "theoretical_loss": 3.7077199743821314, + "tokens_seen": 918028288 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005833019057171515, + "loss": 1.4597, + "theoretical_loss": 3.7076945829578882, + "tokens_seen": 918093824 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005832858575727181, + "loss": 1.4641, + "theoretical_loss": 3.7076691938535493, + "tokens_seen": 918159360 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005832698094282849, + "loss": 1.3932, + "theoretical_loss": 3.707643807068737, + "tokens_seen": 918224896 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005832537612838515, + "loss": 1.4209, + "theoretical_loss": 3.7076184226030744, + "tokens_seen": 918290432 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005832377131394183, + "loss": 1.4281, + "theoretical_loss": 3.707593040456184, + "tokens_seen": 918355968 + }, + { + "epoch": 0.28, + "learning_rate": 0.000583221664994985, + "loss": 1.4566, + "theoretical_loss": 3.7075676606276877, + "tokens_seen": 918421504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005832056168505517, + "loss": 1.4528, + "theoretical_loss": 3.707542283117209, + "tokens_seen": 918487040 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005831895687061183, + "loss": 1.3927, + "theoretical_loss": 3.7075169079243704, + "tokens_seen": 918552576 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005831735205616851, + "loss": 1.5246, + "theoretical_loss": 3.7074915350487956, + "tokens_seen": 918618112 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005831574724172517, + "loss": 1.4058, + "theoretical_loss": 3.707466164490107, + "tokens_seen": 918683648 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005831414242728185, + "loss": 1.5542, + "theoretical_loss": 3.707440796247928, + "tokens_seen": 918749184 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005831253761283851, + "loss": 1.406, + "theoretical_loss": 3.7074154303218823, + "tokens_seen": 918814720 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005831093279839519, + "loss": 1.5631, + "theoretical_loss": 3.707390066711593, + "tokens_seen": 918880256 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005830932798395186, + "loss": 1.4559, + "theoretical_loss": 3.7073647054166834, + "tokens_seen": 918945792 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005830772316950853, + "loss": 1.3253, + "theoretical_loss": 3.7073393464367768, + "tokens_seen": 919011328 + }, + { + "epoch": 0.28, + "learning_rate": 0.000583061183550652, + "loss": 1.5303, + "theoretical_loss": 3.707313989771497, + "tokens_seen": 919076864 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005830451354062187, + "loss": 1.512, + "theoretical_loss": 3.707288635420468, + "tokens_seen": 919142400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005830290872617854, + "loss": 1.3834, + "theoretical_loss": 3.7072632833833135, + "tokens_seen": 919207936 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005830130391173521, + "loss": 1.4234, + "theoretical_loss": 3.707237933659657, + "tokens_seen": 919273472 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005829969909729188, + "loss": 1.3898, + "theoretical_loss": 3.7072125862491228, + "tokens_seen": 919339008 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005829809428284855, + "loss": 1.4016, + "theoretical_loss": 3.7071872411513347, + "tokens_seen": 919404544 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005829648946840522, + "loss": 1.4747, + "theoretical_loss": 3.707161898365918, + "tokens_seen": 919470080 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005829488465396189, + "loss": 1.4534, + "theoretical_loss": 3.707136557892495, + "tokens_seen": 919535616 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005829327983951856, + "loss": 1.4593, + "theoretical_loss": 3.7071112197306917, + "tokens_seen": 919601152 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005829167502507523, + "loss": 1.3262, + "theoretical_loss": 3.7070858838801315, + "tokens_seen": 919666688 + }, + { + "epoch": 0.28, + "learning_rate": 0.000582900702106319, + "loss": 1.4766, + "theoretical_loss": 3.7070605503404392, + "tokens_seen": 919732224 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005828846539618857, + "loss": 1.5222, + "theoretical_loss": 3.7070352191112397, + "tokens_seen": 919797760 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005828686058174524, + "loss": 1.4335, + "theoretical_loss": 3.707009890192157, + "tokens_seen": 919863296 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005828525576730191, + "loss": 1.5404, + "theoretical_loss": 3.7069845635828167, + "tokens_seen": 919928832 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005828365095285858, + "loss": 1.3992, + "theoretical_loss": 3.7069592392828428, + "tokens_seen": 919994368 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005828204613841525, + "loss": 1.3574, + "theoretical_loss": 3.706933917291861, + "tokens_seen": 920059904 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005828044132397192, + "loss": 1.2992, + "theoretical_loss": 3.7069085976094955, + "tokens_seen": 920125440 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005827883650952859, + "loss": 1.3279, + "theoretical_loss": 3.7068832802353726, + "tokens_seen": 920190976 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005827723169508526, + "loss": 1.4596, + "theoretical_loss": 3.7068579651691165, + "tokens_seen": 920256512 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005827562688064193, + "loss": 1.3719, + "theoretical_loss": 3.7068326524103528, + "tokens_seen": 920322048 + }, + { + "epoch": 0.28, + "learning_rate": 0.000582740220661986, + "loss": 1.4492, + "theoretical_loss": 3.706807341958706, + "tokens_seen": 920387584 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005827241725175527, + "loss": 1.487, + "theoretical_loss": 3.706782033813804, + "tokens_seen": 920453120 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005827081243731194, + "loss": 1.4225, + "theoretical_loss": 3.70675672797527, + "tokens_seen": 920518656 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005826920762286861, + "loss": 1.4231, + "theoretical_loss": 3.7067314244427303, + "tokens_seen": 920584192 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005826760280842528, + "loss": 1.469, + "theoretical_loss": 3.7067061232158105, + "tokens_seen": 920649728 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005826599799398195, + "loss": 1.625, + "theoretical_loss": 3.7066808242941374, + "tokens_seen": 920715264 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.9904934167861938, + "objective/train/avg_token_score": 0.009506529197096825, + "objective/train/docs_used": 339576, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4011276960372925, + "objective/train/original_loss": 1.414494276046753, + "objective/train/theoretical_loss": 3.706655527677335, + "objective/train/tokens_used": 941240800, + "objective/train/unlikelihood_loss": 0.023052001371979713, + "theoretical_loss": 3.706655527677335, + "tokens_seen": 920780800 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005826439317953862, + "loss": 1.5327, + "theoretical_loss": 3.706655527677335, + "tokens_seen": 920780800 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005826278836509529, + "loss": 1.5712, + "theoretical_loss": 3.706630233365032, + "tokens_seen": 920846336 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005826118355065196, + "loss": 1.4886, + "theoretical_loss": 3.7066049413568516, + "tokens_seen": 920911872 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005825957873620863, + "loss": 1.5549, + "theoretical_loss": 3.7065796516524214, + "tokens_seen": 920977408 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005825797392176531, + "loss": 1.4502, + "theoretical_loss": 3.7065543642513683, + "tokens_seen": 921042944 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005825636910732197, + "loss": 1.3953, + "theoretical_loss": 3.7065290791533174, + "tokens_seen": 921108480 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005825476429287865, + "loss": 1.4476, + "theoretical_loss": 3.7065037963578957, + "tokens_seen": 921174016 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005825315947843531, + "loss": 1.34, + "theoretical_loss": 3.7064785158647293, + "tokens_seen": 921239552 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005825155466399199, + "loss": 1.3291, + "theoretical_loss": 3.7064532376734456, + "tokens_seen": 921305088 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005824994984954865, + "loss": 1.4549, + "theoretical_loss": 3.70642796178367, + "tokens_seen": 921370624 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005824834503510532, + "loss": 1.4045, + "theoretical_loss": 3.706402688195031, + "tokens_seen": 921436160 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005824674022066199, + "loss": 1.3776, + "theoretical_loss": 3.706377416907154, + "tokens_seen": 921501696 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005824513540621866, + "loss": 1.48, + "theoretical_loss": 3.7063521479196666, + "tokens_seen": 921567232 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005824353059177533, + "loss": 1.4944, + "theoretical_loss": 3.7063268812321954, + "tokens_seen": 921632768 + }, + { + "epoch": 0.28, + "learning_rate": 0.00058241925777332, + "loss": 1.4955, + "theoretical_loss": 3.7063016168443683, + "tokens_seen": 921698304 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005824032096288867, + "loss": 1.4325, + "theoretical_loss": 3.706276354755812, + "tokens_seen": 921763840 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005823871614844534, + "loss": 1.3037, + "theoretical_loss": 3.706251094966153, + "tokens_seen": 921829376 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005823711133400201, + "loss": 1.5116, + "theoretical_loss": 3.7062258374750208, + "tokens_seen": 921894912 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005823550651955868, + "loss": 1.4313, + "theoretical_loss": 3.7062005822820403, + "tokens_seen": 921960448 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005823390170511534, + "loss": 1.4482, + "theoretical_loss": 3.7061753293868414, + "tokens_seen": 922025984 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005823229689067202, + "loss": 1.569, + "theoretical_loss": 3.70615007878905, + "tokens_seen": 922091520 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005823069207622868, + "loss": 1.3078, + "theoretical_loss": 3.706124830488295, + "tokens_seen": 922157056 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005822908726178536, + "loss": 1.3698, + "theoretical_loss": 3.7060995844842033, + "tokens_seen": 922222592 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005822748244734202, + "loss": 1.5771, + "theoretical_loss": 3.706074340776403, + "tokens_seen": 922288128 + }, + { + "epoch": 0.28, + "learning_rate": 0.000582258776328987, + "loss": 1.4085, + "theoretical_loss": 3.7060490993645225, + "tokens_seen": 922353664 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005822427281845536, + "loss": 1.3849, + "theoretical_loss": 3.70602386024819, + "tokens_seen": 922419200 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005822266800401204, + "loss": 1.3448, + "theoretical_loss": 3.7059986234270332, + "tokens_seen": 922484736 + }, + { + "epoch": 0.28, + "learning_rate": 0.000582210631895687, + "loss": 1.4181, + "theoretical_loss": 3.7059733889006807, + "tokens_seen": 922550272 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005821945837512538, + "loss": 1.3553, + "theoretical_loss": 3.70594815666876, + "tokens_seen": 922615808 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005821785356068204, + "loss": 1.4127, + "theoretical_loss": 3.7059229267309006, + "tokens_seen": 922681344 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005821624874623872, + "loss": 1.383, + "theoretical_loss": 3.7058976990867305, + "tokens_seen": 922746880 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005821464393179538, + "loss": 1.3793, + "theoretical_loss": 3.7058724737358784, + "tokens_seen": 922812416 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005821303911735206, + "loss": 1.3901, + "theoretical_loss": 3.705847250677973, + "tokens_seen": 922877952 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005821143430290872, + "loss": 1.4053, + "theoretical_loss": 3.7058220299126425, + "tokens_seen": 922943488 + }, + { + "epoch": 0.28, + "learning_rate": 0.000582098294884654, + "loss": 1.5232, + "theoretical_loss": 3.705796811439517, + "tokens_seen": 923009024 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005820822467402206, + "loss": 1.4351, + "theoretical_loss": 3.7057715952582244, + "tokens_seen": 923074560 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005820661985957874, + "loss": 1.4879, + "theoretical_loss": 3.705746381368394, + "tokens_seen": 923140096 + }, + { + "epoch": 0.28, + "learning_rate": 0.000582050150451354, + "loss": 1.5277, + "theoretical_loss": 3.705721169769655, + "tokens_seen": 923205632 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005820341023069208, + "loss": 1.3949, + "theoretical_loss": 3.705695960461636, + "tokens_seen": 923271168 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005820180541624874, + "loss": 1.4501, + "theoretical_loss": 3.7056707534439672, + "tokens_seen": 923336704 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005820020060180542, + "loss": 1.4263, + "theoretical_loss": 3.7056455487162783, + "tokens_seen": 923402240 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005819859578736208, + "loss": 1.3457, + "theoretical_loss": 3.705620346278198, + "tokens_seen": 923467776 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005819699097291876, + "loss": 1.455, + "theoretical_loss": 3.7055951461293546, + "tokens_seen": 923533312 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005819538615847542, + "loss": 1.403, + "theoretical_loss": 3.70556994826938, + "tokens_seen": 923598848 + }, + { + "epoch": 0.28, + "learning_rate": 0.000581937813440321, + "loss": 1.2964, + "theoretical_loss": 3.705544752697903, + "tokens_seen": 923664384 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005819217652958876, + "loss": 1.3551, + "theoretical_loss": 3.705519559414553, + "tokens_seen": 923729920 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005819057171514544, + "loss": 1.341, + "theoretical_loss": 3.7054943684189605, + "tokens_seen": 923795456 + }, + { + "epoch": 0.28, + "learning_rate": 0.000581889669007021, + "loss": 1.371, + "theoretical_loss": 3.705469179710755, + "tokens_seen": 923860992 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005818736208625878, + "loss": 1.4723, + "theoretical_loss": 3.705443993289567, + "tokens_seen": 923926528 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005818575727181544, + "loss": 1.3277, + "theoretical_loss": 3.7054188091550264, + "tokens_seen": 923992064 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.9935050010681152, + "objective/train/avg_token_score": 0.006495039444416761, + "objective/train/docs_used": 340768, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5517058372497559, + "objective/train/original_loss": 1.5644750595092773, + "objective/train/theoretical_loss": 3.705393627306764, + "objective/train/tokens_used": 944517600, + "objective/train/unlikelihood_loss": 0.012546461075544357, + "theoretical_loss": 3.705393627306764, + "tokens_seen": 924057600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005818415245737212, + "loss": 1.4572, + "theoretical_loss": 3.705393627306764, + "tokens_seen": 924057600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005818254764292878, + "loss": 1.422, + "theoretical_loss": 3.7053684477444087, + "tokens_seen": 924123136 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005818094282848546, + "loss": 1.372, + "theoretical_loss": 3.7053432704675924, + "tokens_seen": 924188672 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005817933801404212, + "loss": 1.3361, + "theoretical_loss": 3.7053180954759446, + "tokens_seen": 924254208 + }, + { + "epoch": 0.28, + "learning_rate": 0.000581777331995988, + "loss": 1.3576, + "theoretical_loss": 3.7052929227690967, + "tokens_seen": 924319744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005817612838515546, + "loss": 1.5956, + "theoretical_loss": 3.7052677523466784, + "tokens_seen": 924385280 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005817452357071214, + "loss": 1.3594, + "theoretical_loss": 3.7052425842083214, + "tokens_seen": 924450816 + }, + { + "epoch": 0.28, + "learning_rate": 0.000581729187562688, + "loss": 1.3833, + "theoretical_loss": 3.705217418353656, + "tokens_seen": 924516352 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005817131394182548, + "loss": 1.3918, + "theoretical_loss": 3.7051922547823137, + "tokens_seen": 924581888 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005816970912738215, + "loss": 1.3186, + "theoretical_loss": 3.7051670934939245, + "tokens_seen": 924647424 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005816810431293882, + "loss": 1.3929, + "theoretical_loss": 3.70514193448812, + "tokens_seen": 924712960 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005816649949849549, + "loss": 1.34, + "theoretical_loss": 3.705116777764532, + "tokens_seen": 924778496 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005816489468405216, + "loss": 1.399, + "theoretical_loss": 3.7050916233227906, + "tokens_seen": 924844032 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005816328986960883, + "loss": 1.542, + "theoretical_loss": 3.705066471162528, + "tokens_seen": 924909568 + }, + { + "epoch": 0.28, + "learning_rate": 0.000581616850551655, + "loss": 1.4745, + "theoretical_loss": 3.7050413212833755, + "tokens_seen": 924975104 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005816008024072217, + "loss": 1.3492, + "theoretical_loss": 3.7050161736849647, + "tokens_seen": 925040640 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005815847542627884, + "loss": 1.3381, + "theoretical_loss": 3.7049910283669263, + "tokens_seen": 925106176 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005815687061183551, + "loss": 1.4715, + "theoretical_loss": 3.7049658853288934, + "tokens_seen": 925171712 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005815526579739218, + "loss": 1.4122, + "theoretical_loss": 3.7049407445704965, + "tokens_seen": 925237248 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005815366098294885, + "loss": 1.3376, + "theoretical_loss": 3.704915606091369, + "tokens_seen": 925302784 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005815205616850552, + "loss": 1.4409, + "theoretical_loss": 3.704890469891141, + "tokens_seen": 925368320 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005815045135406219, + "loss": 1.3865, + "theoretical_loss": 3.704865335969446, + "tokens_seen": 925433856 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005814884653961886, + "loss": 1.3754, + "theoretical_loss": 3.7048402043259148, + "tokens_seen": 925499392 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005814724172517553, + "loss": 1.417, + "theoretical_loss": 3.704815074960181, + "tokens_seen": 925564928 + }, + { + "epoch": 0.28, + "learning_rate": 0.000581456369107322, + "loss": 1.4623, + "theoretical_loss": 3.704789947871876, + "tokens_seen": 925630464 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005814403209628887, + "loss": 1.3664, + "theoretical_loss": 3.7047648230606325, + "tokens_seen": 925696000 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005814242728184554, + "loss": 1.4335, + "theoretical_loss": 3.7047397005260834, + "tokens_seen": 925761536 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005814082246740221, + "loss": 1.3735, + "theoretical_loss": 3.70471458026786, + "tokens_seen": 925827072 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005813921765295888, + "loss": 1.5561, + "theoretical_loss": 3.7046894622855957, + "tokens_seen": 925892608 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005813761283851556, + "loss": 1.455, + "theoretical_loss": 3.704664346578924, + "tokens_seen": 925958144 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005813600802407222, + "loss": 1.4035, + "theoretical_loss": 3.704639233147476, + "tokens_seen": 926023680 + }, + { + "epoch": 0.28, + "learning_rate": 0.000581344032096289, + "loss": 1.4448, + "theoretical_loss": 3.704614121990886, + "tokens_seen": 926089216 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005813279839518556, + "loss": 1.5043, + "theoretical_loss": 3.7045890131087864, + "tokens_seen": 926154752 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005813119358074224, + "loss": 1.3831, + "theoretical_loss": 3.70456390650081, + "tokens_seen": 926220288 + }, + { + "epoch": 0.28, + "learning_rate": 0.000581295887662989, + "loss": 1.3491, + "theoretical_loss": 3.7045388021665904, + "tokens_seen": 926285824 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005812798395185558, + "loss": 1.3531, + "theoretical_loss": 3.7045137001057613, + "tokens_seen": 926351360 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005812637913741224, + "loss": 1.4089, + "theoretical_loss": 3.704488600317955, + "tokens_seen": 926416896 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005812477432296892, + "loss": 1.4482, + "theoretical_loss": 3.7044635028028052, + "tokens_seen": 926482432 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005812316950852558, + "loss": 1.4896, + "theoretical_loss": 3.7044384075599455, + "tokens_seen": 926547968 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005812156469408226, + "loss": 1.4019, + "theoretical_loss": 3.7044133145890097, + "tokens_seen": 926613504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005811995987963892, + "loss": 1.2952, + "theoretical_loss": 3.7043882238896306, + "tokens_seen": 926679040 + }, + { + "epoch": 0.28, + "learning_rate": 0.000581183550651956, + "loss": 1.3274, + "theoretical_loss": 3.704363135461443, + "tokens_seen": 926744576 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005811675025075226, + "loss": 1.3766, + "theoretical_loss": 3.7043380493040807, + "tokens_seen": 926810112 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005811514543630894, + "loss": 1.4297, + "theoretical_loss": 3.7043129654171767, + "tokens_seen": 926875648 + }, + { + "epoch": 0.28, + "learning_rate": 0.000581135406218656, + "loss": 1.3497, + "theoretical_loss": 3.704287883800365, + "tokens_seen": 926941184 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005811193580742228, + "loss": 1.4724, + "theoretical_loss": 3.7042628044532813, + "tokens_seen": 927006720 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005811033099297894, + "loss": 1.3547, + "theoretical_loss": 3.704237727375558, + "tokens_seen": 927072256 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005810872617853561, + "loss": 1.3713, + "theoretical_loss": 3.70421265256683, + "tokens_seen": 927137792 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005810712136409228, + "loss": 1.4913, + "theoretical_loss": 3.7041875800267317, + "tokens_seen": 927203328 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005810551654964895, + "loss": 1.3871, + "theoretical_loss": 3.704162509754898, + "tokens_seen": 927268864 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.9912850260734558, + "objective/train/avg_token_score": 0.008714958094060421, + "objective/train/docs_used": 341815, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.3266854286193848, + "objective/train/original_loss": 1.3388975858688354, + "objective/train/theoretical_loss": 3.7041374417509623, + "objective/train/tokens_used": 947794400, + "objective/train/unlikelihood_loss": 0.025181926786899567, + "theoretical_loss": 3.7041374417509623, + "tokens_seen": 927334400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005810391173520562, + "loss": 1.3657, + "theoretical_loss": 3.7041374417509623, + "tokens_seen": 927334400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005810230692076229, + "loss": 1.3831, + "theoretical_loss": 3.7041123760145593, + "tokens_seen": 927399936 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005810070210631896, + "loss": 1.442, + "theoretical_loss": 3.704087312545325, + "tokens_seen": 927465472 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005809909729187563, + "loss": 1.4792, + "theoretical_loss": 3.7040622513428927, + "tokens_seen": 927531008 + }, + { + "epoch": 0.28, + "learning_rate": 0.000580974924774323, + "loss": 1.2686, + "theoretical_loss": 3.704037192406898, + "tokens_seen": 927596544 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005809588766298897, + "loss": 1.3867, + "theoretical_loss": 3.704012135736975, + "tokens_seen": 927662080 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005809428284854563, + "loss": 1.4291, + "theoretical_loss": 3.7039870813327602, + "tokens_seen": 927727616 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005809267803410231, + "loss": 1.3114, + "theoretical_loss": 3.703962029193888, + "tokens_seen": 927793152 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005809107321965897, + "loss": 1.4371, + "theoretical_loss": 3.7039369793199928, + "tokens_seen": 927858688 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005808946840521565, + "loss": 1.3436, + "theoretical_loss": 3.7039119317107105, + "tokens_seen": 927924224 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005808786359077231, + "loss": 1.3191, + "theoretical_loss": 3.7038868863656766, + "tokens_seen": 927989760 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005808625877632899, + "loss": 1.4783, + "theoretical_loss": 3.7038618432845265, + "tokens_seen": 928055296 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005808465396188565, + "loss": 1.3861, + "theoretical_loss": 3.7038368024668955, + "tokens_seen": 928120832 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005808304914744233, + "loss": 1.4834, + "theoretical_loss": 3.703811763912419, + "tokens_seen": 928186368 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005808144433299899, + "loss": 1.3845, + "theoretical_loss": 3.7037867276207335, + "tokens_seen": 928251904 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005807983951855567, + "loss": 1.5505, + "theoretical_loss": 3.703761693591474, + "tokens_seen": 928317440 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005807823470411233, + "loss": 1.4492, + "theoretical_loss": 3.7037366618242764, + "tokens_seen": 928382976 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005807662988966901, + "loss": 1.4145, + "theoretical_loss": 3.703711632318777, + "tokens_seen": 928448512 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005807502507522567, + "loss": 1.4494, + "theoretical_loss": 3.703686605074612, + "tokens_seen": 928514048 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005807342026078235, + "loss": 1.3918, + "theoretical_loss": 3.7036615800914165, + "tokens_seen": 928579584 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005807181544633901, + "loss": 1.4057, + "theoretical_loss": 3.703636557368828, + "tokens_seen": 928645120 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005807021063189569, + "loss": 1.5261, + "theoretical_loss": 3.7036115369064815, + "tokens_seen": 928710656 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005806860581745235, + "loss": 1.4967, + "theoretical_loss": 3.7035865187040145, + "tokens_seen": 928776192 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005806700100300903, + "loss": 1.4196, + "theoretical_loss": 3.703561502761063, + "tokens_seen": 928841728 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005806539618856569, + "loss": 1.416, + "theoretical_loss": 3.7035364890772637, + "tokens_seen": 928907264 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005806379137412237, + "loss": 1.4881, + "theoretical_loss": 3.7035114776522526, + "tokens_seen": 928972800 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005806218655967903, + "loss": 1.3077, + "theoretical_loss": 3.703486468485667, + "tokens_seen": 929038336 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005806058174523571, + "loss": 1.5736, + "theoretical_loss": 3.7034614615771435, + "tokens_seen": 929103872 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005805897693079237, + "loss": 1.5564, + "theoretical_loss": 3.7034364569263185, + "tokens_seen": 929169408 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005805737211634905, + "loss": 1.5051, + "theoretical_loss": 3.70341145453283, + "tokens_seen": 929234944 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005805576730190571, + "loss": 1.5022, + "theoretical_loss": 3.703386454396314, + "tokens_seen": 929300480 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005805416248746239, + "loss": 1.4524, + "theoretical_loss": 3.703361456516408, + "tokens_seen": 929366016 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005805255767301905, + "loss": 1.4226, + "theoretical_loss": 3.7033364608927495, + "tokens_seen": 929431552 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005805095285857573, + "loss": 1.366, + "theoretical_loss": 3.7033114675249754, + "tokens_seen": 929497088 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005804934804413239, + "loss": 1.3974, + "theoretical_loss": 3.703286476412724, + "tokens_seen": 929562624 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005804774322968907, + "loss": 1.306, + "theoretical_loss": 3.703261487555631, + "tokens_seen": 929628160 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005804613841524573, + "loss": 1.3992, + "theoretical_loss": 3.703236500953335, + "tokens_seen": 929693696 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005804453360080241, + "loss": 1.4299, + "theoretical_loss": 3.7032115166054735, + "tokens_seen": 929759232 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005804292878635907, + "loss": 1.4742, + "theoretical_loss": 3.7031865345116843, + "tokens_seen": 929824768 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005804132397191575, + "loss": 1.2972, + "theoretical_loss": 3.7031615546716052, + "tokens_seen": 929890304 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005803971915747241, + "loss": 1.4228, + "theoretical_loss": 3.7031365770848743, + "tokens_seen": 929955840 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005803811434302909, + "loss": 1.3957, + "theoretical_loss": 3.7031116017511287, + "tokens_seen": 930021376 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005803650952858576, + "loss": 1.3415, + "theoretical_loss": 3.7030866286700075, + "tokens_seen": 930086912 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005803490471414243, + "loss": 1.3453, + "theoretical_loss": 3.703061657841148, + "tokens_seen": 930152448 + }, + { + "epoch": 0.28, + "learning_rate": 0.000580332998996991, + "loss": 1.4022, + "theoretical_loss": 3.703036689264189, + "tokens_seen": 930217984 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005803169508525577, + "loss": 1.4353, + "theoretical_loss": 3.703011722938768, + "tokens_seen": 930283520 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005803009027081244, + "loss": 1.5657, + "theoretical_loss": 3.702986758864524, + "tokens_seen": 930349056 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005802848545636911, + "loss": 1.4614, + "theoretical_loss": 3.7029617970410955, + "tokens_seen": 930414592 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005802688064192578, + "loss": 1.4181, + "theoretical_loss": 3.702936837468121, + "tokens_seen": 930480128 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005802527582748245, + "loss": 1.3032, + "theoretical_loss": 3.7029118801452388, + "tokens_seen": 930545664 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.994653046131134, + "objective/train/avg_token_score": 0.005346918944269419, + "objective/train/docs_used": 343003, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5275816917419434, + "objective/train/original_loss": 1.5367906093597412, + "objective/train/theoretical_loss": 3.7028869250720877, + "objective/train/tokens_used": 951071200, + "objective/train/unlikelihood_loss": 0.01104492787271738, + "theoretical_loss": 3.7028869250720877, + "tokens_seen": 930611200 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005802367101303912, + "loss": 1.5271, + "theoretical_loss": 3.7028869250720877, + "tokens_seen": 930611200 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005802206619859579, + "loss": 1.3447, + "theoretical_loss": 3.702861972248307, + "tokens_seen": 930676736 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005802046138415246, + "loss": 1.4005, + "theoretical_loss": 3.702837021673535, + "tokens_seen": 930742272 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005801885656970913, + "loss": 1.4224, + "theoretical_loss": 3.7028120733474106, + "tokens_seen": 930807808 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005801725175526581, + "loss": 1.4167, + "theoretical_loss": 3.7027871272695734, + "tokens_seen": 930873344 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005801564694082247, + "loss": 1.4454, + "theoretical_loss": 3.7027621834396625, + "tokens_seen": 930938880 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005801404212637915, + "loss": 1.4888, + "theoretical_loss": 3.7027372418573163, + "tokens_seen": 931004416 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005801243731193581, + "loss": 1.5605, + "theoretical_loss": 3.7027123025221744, + "tokens_seen": 931069952 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005801083249749249, + "loss": 1.3385, + "theoretical_loss": 3.702687365433877, + "tokens_seen": 931135488 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005800922768304915, + "loss": 1.4446, + "theoretical_loss": 3.7026624305920626, + "tokens_seen": 931201024 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005800762286860583, + "loss": 1.4398, + "theoretical_loss": 3.702637497996371, + "tokens_seen": 931266560 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005800601805416249, + "loss": 1.419, + "theoretical_loss": 3.702612567646442, + "tokens_seen": 931332096 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005800441323971917, + "loss": 1.5249, + "theoretical_loss": 3.7025876395419153, + "tokens_seen": 931397632 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005800280842527583, + "loss": 1.585, + "theoretical_loss": 3.702562713682431, + "tokens_seen": 931463168 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005800120361083251, + "loss": 1.3712, + "theoretical_loss": 3.7025377900676277, + "tokens_seen": 931528704 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005799959879638917, + "loss": 1.3688, + "theoretical_loss": 3.7025128686971467, + "tokens_seen": 931594240 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005799799398194585, + "loss": 1.369, + "theoretical_loss": 3.7024879495706275, + "tokens_seen": 931659776 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005799638916750251, + "loss": 1.4273, + "theoretical_loss": 3.70246303268771, + "tokens_seen": 931725312 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005799478435305919, + "loss": 1.4047, + "theoretical_loss": 3.7024381180480352, + "tokens_seen": 931790848 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005799317953861585, + "loss": 1.4048, + "theoretical_loss": 3.702413205651243, + "tokens_seen": 931856384 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005799157472417253, + "loss": 1.4167, + "theoretical_loss": 3.7023882954969736, + "tokens_seen": 931921920 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005798996990972919, + "loss": 1.3679, + "theoretical_loss": 3.7023633875848674, + "tokens_seen": 931987456 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005798836509528587, + "loss": 1.5157, + "theoretical_loss": 3.7023384819145644, + "tokens_seen": 932052992 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005798676028084253, + "loss": 1.4508, + "theoretical_loss": 3.7023135784857066, + "tokens_seen": 932118528 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005798515546639921, + "loss": 1.4684, + "theoretical_loss": 3.7022886772979344, + "tokens_seen": 932184064 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005798355065195587, + "loss": 1.374, + "theoretical_loss": 3.702263778350887, + "tokens_seen": 932249600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005798194583751255, + "loss": 1.3475, + "theoretical_loss": 3.702238881644208, + "tokens_seen": 932315136 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005798034102306921, + "loss": 1.4218, + "theoretical_loss": 3.7022139871775357, + "tokens_seen": 932380672 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005797873620862589, + "loss": 1.4224, + "theoretical_loss": 3.702189094950512, + "tokens_seen": 932446208 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005797713139418255, + "loss": 1.3923, + "theoretical_loss": 3.702164204962779, + "tokens_seen": 932511744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005797552657973923, + "loss": 1.3988, + "theoretical_loss": 3.7021393172139767, + "tokens_seen": 932577280 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005797392176529589, + "loss": 1.4707, + "theoretical_loss": 3.7021144317037473, + "tokens_seen": 932642816 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005797231695085257, + "loss": 1.4348, + "theoretical_loss": 3.702089548431731, + "tokens_seen": 932708352 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005797071213640923, + "loss": 1.4195, + "theoretical_loss": 3.7020646673975706, + "tokens_seen": 932773888 + }, + { + "epoch": 0.28, + "learning_rate": 0.000579691073219659, + "loss": 1.4101, + "theoretical_loss": 3.7020397886009064, + "tokens_seen": 932839424 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005796750250752257, + "loss": 1.314, + "theoretical_loss": 3.7020149120413812, + "tokens_seen": 932904960 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005796589769307924, + "loss": 1.3711, + "theoretical_loss": 3.7019900377186357, + "tokens_seen": 932970496 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005796429287863591, + "loss": 1.5406, + "theoretical_loss": 3.7019651656323127, + "tokens_seen": 933036032 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005796268806419258, + "loss": 1.5978, + "theoretical_loss": 3.7019402957820526, + "tokens_seen": 933101568 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005796108324974925, + "loss": 1.3967, + "theoretical_loss": 3.7019154281674984, + "tokens_seen": 933167104 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005795947843530592, + "loss": 1.5696, + "theoretical_loss": 3.701890562788292, + "tokens_seen": 933232640 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005795787362086259, + "loss": 1.2834, + "theoretical_loss": 3.7018656996440757, + "tokens_seen": 933298176 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005795626880641926, + "loss": 1.4074, + "theoretical_loss": 3.701840838734491, + "tokens_seen": 933363712 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005795466399197592, + "loss": 1.3996, + "theoretical_loss": 3.701815980059181, + "tokens_seen": 933429248 + }, + { + "epoch": 0.28, + "learning_rate": 0.000579530591775326, + "loss": 1.3333, + "theoretical_loss": 3.701791123617787, + "tokens_seen": 933494784 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005795145436308926, + "loss": 1.44, + "theoretical_loss": 3.7017662694099527, + "tokens_seen": 933560320 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005794984954864594, + "loss": 1.3923, + "theoretical_loss": 3.70174141743532, + "tokens_seen": 933625856 + }, + { + "epoch": 0.28, + "learning_rate": 0.000579482447342026, + "loss": 1.3912, + "theoretical_loss": 3.7017165676935315, + "tokens_seen": 933691392 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005794663991975928, + "loss": 1.3427, + "theoretical_loss": 3.701691720184229, + "tokens_seen": 933756928 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005794503510531594, + "loss": 1.4547, + "theoretical_loss": 3.7016668749070574, + "tokens_seen": 933822464 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.9944117069244385, + "objective/train/avg_token_score": 0.0055882902815938, + "objective/train/docs_used": 344002, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5405638217926025, + "objective/train/original_loss": 1.5499482154846191, + "objective/train/theoretical_loss": 3.7016420318616574, + "objective/train/tokens_used": 954348000, + "objective/train/unlikelihood_loss": 0.013551376760005951, + "theoretical_loss": 3.7016420318616574, + "tokens_seen": 933888000 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005794343029087262, + "loss": 1.5227, + "theoretical_loss": 3.7016420318616574, + "tokens_seen": 933888000 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005794182547642928, + "loss": 1.4382, + "theoretical_loss": 3.7016171910476734, + "tokens_seen": 933953536 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005794022066198596, + "loss": 1.4615, + "theoretical_loss": 3.701592352464748, + "tokens_seen": 934019072 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005793861584754262, + "loss": 1.4556, + "theoretical_loss": 3.701567516112523, + "tokens_seen": 934084608 + }, + { + "epoch": 0.28, + "learning_rate": 0.000579370110330993, + "loss": 1.4724, + "theoretical_loss": 3.701542681990644, + "tokens_seen": 934150144 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005793540621865596, + "loss": 1.514, + "theoretical_loss": 3.7015178500987522, + "tokens_seen": 934215680 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005793380140421264, + "loss": 1.4305, + "theoretical_loss": 3.701493020436492, + "tokens_seen": 934281216 + }, + { + "epoch": 0.28, + "learning_rate": 0.000579321965897693, + "loss": 1.5209, + "theoretical_loss": 3.7014681930035067, + "tokens_seen": 934346752 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005793059177532598, + "loss": 1.4254, + "theoretical_loss": 3.7014433677994396, + "tokens_seen": 934412288 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005792898696088264, + "loss": 1.4202, + "theoretical_loss": 3.7014185448239343, + "tokens_seen": 934477824 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005792738214643932, + "loss": 1.4915, + "theoretical_loss": 3.7013937240766346, + "tokens_seen": 934543360 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005792577733199598, + "loss": 1.4594, + "theoretical_loss": 3.7013689055571843, + "tokens_seen": 934608896 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005792417251755266, + "loss": 1.4622, + "theoretical_loss": 3.701344089265227, + "tokens_seen": 934674432 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005792256770310932, + "loss": 1.3234, + "theoretical_loss": 3.7013192752004063, + "tokens_seen": 934739968 + }, + { + "epoch": 0.28, + "learning_rate": 0.00057920962888666, + "loss": 1.4447, + "theoretical_loss": 3.701294463362367, + "tokens_seen": 934805504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005791935807422266, + "loss": 1.4125, + "theoretical_loss": 3.701269653750753, + "tokens_seen": 934871040 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005791775325977934, + "loss": 1.4685, + "theoretical_loss": 3.701244846365208, + "tokens_seen": 934936576 + }, + { + "epoch": 0.28, + "learning_rate": 0.00057916148445336, + "loss": 1.4645, + "theoretical_loss": 3.701220041205377, + "tokens_seen": 935002112 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005791454363089268, + "loss": 1.4352, + "theoretical_loss": 3.701195238270903, + "tokens_seen": 935067648 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005791293881644934, + "loss": 1.3402, + "theoretical_loss": 3.701170437561432, + "tokens_seen": 935133184 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005791133400200602, + "loss": 1.4545, + "theoretical_loss": 3.701145639076607, + "tokens_seen": 935198720 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005790972918756268, + "loss": 1.3166, + "theoretical_loss": 3.7011208428160742, + "tokens_seen": 935264256 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005790812437311936, + "loss": 1.3922, + "theoretical_loss": 3.701096048779477, + "tokens_seen": 935329792 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005790651955867602, + "loss": 1.4314, + "theoretical_loss": 3.7010712569664603, + "tokens_seen": 935395328 + }, + { + "epoch": 0.28, + "learning_rate": 0.000579049147442327, + "loss": 1.3646, + "theoretical_loss": 3.701046467376669, + "tokens_seen": 935460864 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005790330992978936, + "loss": 1.5779, + "theoretical_loss": 3.701021680009749, + "tokens_seen": 935526400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005790170511534604, + "loss": 1.3972, + "theoretical_loss": 3.7009968948653436, + "tokens_seen": 935591936 + }, + { + "epoch": 0.28, + "learning_rate": 0.000579001003009027, + "loss": 1.3726, + "theoretical_loss": 3.7009721119430985, + "tokens_seen": 935657472 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005789849548645938, + "loss": 1.3916, + "theoretical_loss": 3.7009473312426593, + "tokens_seen": 935723008 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005789689067201606, + "loss": 1.4292, + "theoretical_loss": 3.700922552763671, + "tokens_seen": 935788544 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005789528585757272, + "loss": 1.3853, + "theoretical_loss": 3.700897776505779, + "tokens_seen": 935854080 + }, + { + "epoch": 0.28, + "learning_rate": 0.000578936810431294, + "loss": 1.4112, + "theoretical_loss": 3.7008730024686285, + "tokens_seen": 935919616 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005789207622868606, + "loss": 1.4523, + "theoretical_loss": 3.7008482306518644, + "tokens_seen": 935985152 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005789047141424274, + "loss": 1.4152, + "theoretical_loss": 3.7008234610551334, + "tokens_seen": 936050688 + }, + { + "epoch": 0.28, + "learning_rate": 0.000578888665997994, + "loss": 1.5083, + "theoretical_loss": 3.70079869367808, + "tokens_seen": 936116224 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005788726178535608, + "loss": 1.4505, + "theoretical_loss": 3.700773928520351, + "tokens_seen": 936181760 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005788565697091274, + "loss": 1.3706, + "theoretical_loss": 3.7007491655815916, + "tokens_seen": 936247296 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005788405215646942, + "loss": 1.3626, + "theoretical_loss": 3.7007244048614476, + "tokens_seen": 936312832 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005788244734202608, + "loss": 1.4508, + "theoretical_loss": 3.700699646359565, + "tokens_seen": 936378368 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005788084252758276, + "loss": 1.4102, + "theoretical_loss": 3.7006748900755904, + "tokens_seen": 936443904 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005787923771313942, + "loss": 1.4014, + "theoretical_loss": 3.700650136009169, + "tokens_seen": 936509440 + }, + { + "epoch": 0.28, + "learning_rate": 0.000578776328986961, + "loss": 1.4303, + "theoretical_loss": 3.700625384159948, + "tokens_seen": 936574976 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005787602808425276, + "loss": 1.467, + "theoretical_loss": 3.7006006345275724, + "tokens_seen": 936640512 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005787442326980944, + "loss": 1.4048, + "theoretical_loss": 3.70057588711169, + "tokens_seen": 936706048 + }, + { + "epoch": 0.28, + "learning_rate": 0.000578728184553661, + "loss": 1.4448, + "theoretical_loss": 3.700551141911946, + "tokens_seen": 936771584 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005787121364092278, + "loss": 1.4111, + "theoretical_loss": 3.700526398927988, + "tokens_seen": 936837120 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005786960882647944, + "loss": 1.363, + "theoretical_loss": 3.700501658159462, + "tokens_seen": 936902656 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005786800401203612, + "loss": 1.3606, + "theoretical_loss": 3.7004769196060145, + "tokens_seen": 936968192 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005786639919759278, + "loss": 1.3604, + "theoretical_loss": 3.7004521832672923, + "tokens_seen": 937033728 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005786479438314946, + "loss": 1.3605, + "theoretical_loss": 3.700427449142943, + "tokens_seen": 937099264 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.9892734289169312, + "objective/train/avg_token_score": 0.01072653941810131, + "objective/train/docs_used": 344763, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.3649702072143555, + "objective/train/original_loss": 1.3805736303329468, + "objective/train/theoretical_loss": 3.700402717232613, + "objective/train/tokens_used": 957624800, + "objective/train/unlikelihood_loss": 0.02666456438601017, + "theoretical_loss": 3.700402717232613, + "tokens_seen": 937164800 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005786318956870612, + "loss": 1.3823, + "theoretical_loss": 3.700402717232613, + "tokens_seen": 937164800 + }, + { + "epoch": 0.28, + "learning_rate": 0.000578615847542628, + "loss": 1.2715, + "theoretical_loss": 3.700377987535949, + "tokens_seen": 937230336 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005785997993981946, + "loss": 1.4736, + "theoretical_loss": 3.700353260052599, + "tokens_seen": 937295872 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005785837512537614, + "loss": 1.4541, + "theoretical_loss": 3.7003285347822086, + "tokens_seen": 937361408 + }, + { + "epoch": 0.28, + "learning_rate": 0.000578567703109328, + "loss": 1.3851, + "theoretical_loss": 3.7003038117244267, + "tokens_seen": 937426944 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005785516549648948, + "loss": 1.4597, + "theoretical_loss": 3.7002790908789, + "tokens_seen": 937492480 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005785356068204614, + "loss": 1.3374, + "theoretical_loss": 3.7002543722452756, + "tokens_seen": 937558016 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005785195586760282, + "loss": 1.3878, + "theoretical_loss": 3.7002296558232013, + "tokens_seen": 937623552 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005785035105315948, + "loss": 1.4466, + "theoretical_loss": 3.7002049416123253, + "tokens_seen": 937689088 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005784874623871616, + "loss": 1.4908, + "theoretical_loss": 3.700180229612294, + "tokens_seen": 937754624 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005784714142427282, + "loss": 1.4482, + "theoretical_loss": 3.700155519822756, + "tokens_seen": 937820160 + }, + { + "epoch": 0.28, + "learning_rate": 0.000578455366098295, + "loss": 1.3894, + "theoretical_loss": 3.700130812243359, + "tokens_seen": 937885696 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005784393179538616, + "loss": 1.4606, + "theoretical_loss": 3.7001061068737506, + "tokens_seen": 937951232 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005784232698094284, + "loss": 1.4663, + "theoretical_loss": 3.700081403713579, + "tokens_seen": 938016768 + }, + { + "epoch": 0.28, + "learning_rate": 0.000578407221664995, + "loss": 1.3926, + "theoretical_loss": 3.700056702762492, + "tokens_seen": 938082304 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005783911735205618, + "loss": 1.3933, + "theoretical_loss": 3.700032004020138, + "tokens_seen": 938147840 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005783751253761284, + "loss": 1.5182, + "theoretical_loss": 3.7000073074861657, + "tokens_seen": 938213376 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005783590772316952, + "loss": 1.4861, + "theoretical_loss": 3.699982613160223, + "tokens_seen": 938278912 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005783430290872618, + "loss": 1.4259, + "theoretical_loss": 3.6999579210419578, + "tokens_seen": 938344448 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005783269809428286, + "loss": 1.4873, + "theoretical_loss": 3.699933231131019, + "tokens_seen": 938409984 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005783109327983952, + "loss": 1.5081, + "theoretical_loss": 3.6999085434270547, + "tokens_seen": 938475520 + }, + { + "epoch": 0.28, + "learning_rate": 0.000578294884653962, + "loss": 1.3231, + "theoretical_loss": 3.6998838579297146, + "tokens_seen": 938541056 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005782788365095286, + "loss": 1.3478, + "theoretical_loss": 3.699859174638646, + "tokens_seen": 938606592 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005782627883650953, + "loss": 1.3703, + "theoretical_loss": 3.6998344935534995, + "tokens_seen": 938672128 + }, + { + "epoch": 0.28, + "learning_rate": 0.000578246740220662, + "loss": 1.4119, + "theoretical_loss": 3.699809814673922, + "tokens_seen": 938737664 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005782306920762287, + "loss": 1.4233, + "theoretical_loss": 3.699785137999563, + "tokens_seen": 938803200 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005782146439317954, + "loss": 1.4203, + "theoretical_loss": 3.6997604635300725, + "tokens_seen": 938868736 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005781985957873621, + "loss": 1.454, + "theoretical_loss": 3.699735791265099, + "tokens_seen": 938934272 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005781825476429288, + "loss": 1.3398, + "theoretical_loss": 3.6997111212042917, + "tokens_seen": 938999808 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005781664994984955, + "loss": 1.4911, + "theoretical_loss": 3.6996864533473, + "tokens_seen": 939065344 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005781504513540621, + "loss": 1.5145, + "theoretical_loss": 3.6996617876937723, + "tokens_seen": 939130880 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005781344032096289, + "loss": 1.4553, + "theoretical_loss": 3.6996371242433597, + "tokens_seen": 939196416 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005781183550651955, + "loss": 1.4171, + "theoretical_loss": 3.69961246299571, + "tokens_seen": 939261952 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005781023069207623, + "loss": 1.4035, + "theoretical_loss": 3.699587803950474, + "tokens_seen": 939327488 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005780862587763289, + "loss": 1.4346, + "theoretical_loss": 3.699563147107301, + "tokens_seen": 939393024 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005780702106318957, + "loss": 1.274, + "theoretical_loss": 3.6995384924658405, + "tokens_seen": 939458560 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005780541624874623, + "loss": 1.2948, + "theoretical_loss": 3.699513840025743, + "tokens_seen": 939524096 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005780381143430291, + "loss": 1.4759, + "theoretical_loss": 3.6994891897866573, + "tokens_seen": 939589632 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005780220661985957, + "loss": 1.3249, + "theoretical_loss": 3.6994645417482346, + "tokens_seen": 939655168 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005780060180541625, + "loss": 1.4523, + "theoretical_loss": 3.6994398959101242, + "tokens_seen": 939720704 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005779899699097291, + "loss": 1.4231, + "theoretical_loss": 3.6994152522719768, + "tokens_seen": 939786240 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005779739217652959, + "loss": 1.4661, + "theoretical_loss": 3.699390610833442, + "tokens_seen": 939851776 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005779578736208625, + "loss": 1.4734, + "theoretical_loss": 3.6993659715941702, + "tokens_seen": 939917312 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005779418254764293, + "loss": 1.5114, + "theoretical_loss": 3.699341334553812, + "tokens_seen": 939982848 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005779257773319959, + "loss": 1.3315, + "theoretical_loss": 3.6993166997120186, + "tokens_seen": 940048384 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005779097291875627, + "loss": 1.4147, + "theoretical_loss": 3.6992920670684395, + "tokens_seen": 940113920 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005778936810431293, + "loss": 1.4586, + "theoretical_loss": 3.6992674366227254, + "tokens_seen": 940179456 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005778776328986961, + "loss": 1.4302, + "theoretical_loss": 3.6992428083745272, + "tokens_seen": 940244992 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005778615847542627, + "loss": 1.3334, + "theoretical_loss": 3.699218182323496, + "tokens_seen": 940310528 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005778455366098295, + "loss": 1.4294, + "theoretical_loss": 3.6991935584692826, + "tokens_seen": 940376064 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.9920151829719543, + "objective/train/avg_token_score": 0.00798475369811058, + "objective/train/docs_used": 345855, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.3710527420043945, + "objective/train/original_loss": 1.3840241432189941, + "objective/train/theoretical_loss": 3.6991689368115375, + "objective/train/tokens_used": 960901600, + "objective/train/unlikelihood_loss": 0.019247159361839294, + "theoretical_loss": 3.6991689368115375, + "tokens_seen": 940441600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005778294884653961, + "loss": 1.4115, + "theoretical_loss": 3.6991689368115375, + "tokens_seen": 940441600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005778134403209629, + "loss": 1.5344, + "theoretical_loss": 3.699144317349912, + "tokens_seen": 940507136 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005777973921765295, + "loss": 1.3335, + "theoretical_loss": 3.6991197000840574, + "tokens_seen": 940572672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005777813440320963, + "loss": 1.4403, + "theoretical_loss": 3.6990950850136244, + "tokens_seen": 940638208 + }, + { + "epoch": 0.29, + "learning_rate": 0.000577765295887663, + "loss": 1.4074, + "theoretical_loss": 3.6990704721382652, + "tokens_seen": 940703744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005777492477432297, + "loss": 1.2706, + "theoretical_loss": 3.69904586145763, + "tokens_seen": 940769280 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005777331995987965, + "loss": 1.4499, + "theoretical_loss": 3.699021252971371, + "tokens_seen": 940834816 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005777171514543631, + "loss": 1.3979, + "theoretical_loss": 3.698996646679139, + "tokens_seen": 940900352 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005777011033099299, + "loss": 1.462, + "theoretical_loss": 3.6989720425805865, + "tokens_seen": 940965888 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005776850551654965, + "loss": 1.4773, + "theoretical_loss": 3.6989474406753646, + "tokens_seen": 941031424 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005776690070210633, + "loss": 1.5004, + "theoretical_loss": 3.6989228409631254, + "tokens_seen": 941096960 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005776529588766299, + "loss": 1.5127, + "theoretical_loss": 3.6988982434435203, + "tokens_seen": 941162496 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005776369107321967, + "loss": 1.4489, + "theoretical_loss": 3.6988736481162015, + "tokens_seen": 941228032 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005776208625877633, + "loss": 1.4075, + "theoretical_loss": 3.698849054980821, + "tokens_seen": 941293568 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005776048144433301, + "loss": 1.4131, + "theoretical_loss": 3.6988244640370302, + "tokens_seen": 941359104 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005775887662988967, + "loss": 1.4461, + "theoretical_loss": 3.6987998752844815, + "tokens_seen": 941424640 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005775727181544635, + "loss": 1.4575, + "theoretical_loss": 3.6987752887228282, + "tokens_seen": 941490176 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005775566700100301, + "loss": 1.4071, + "theoretical_loss": 3.698750704351722, + "tokens_seen": 941555712 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005775406218655969, + "loss": 1.5015, + "theoretical_loss": 3.6987261221708145, + "tokens_seen": 941621248 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005775245737211635, + "loss": 1.5053, + "theoretical_loss": 3.6987015421797587, + "tokens_seen": 941686784 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005775085255767303, + "loss": 1.5652, + "theoretical_loss": 3.698676964378208, + "tokens_seen": 941752320 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005774924774322969, + "loss": 1.2983, + "theoretical_loss": 3.6986523887658134, + "tokens_seen": 941817856 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005774764292878637, + "loss": 1.3376, + "theoretical_loss": 3.6986278153422285, + "tokens_seen": 941883392 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005774603811434303, + "loss": 1.4544, + "theoretical_loss": 3.6986032441071064, + "tokens_seen": 941948928 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005774443329989971, + "loss": 1.4646, + "theoretical_loss": 3.6985786750600993, + "tokens_seen": 942014464 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005774282848545637, + "loss": 1.3808, + "theoretical_loss": 3.6985541082008604, + "tokens_seen": 942080000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005774122367101305, + "loss": 1.405, + "theoretical_loss": 3.698529543529043, + "tokens_seen": 942145536 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005773961885656971, + "loss": 1.3763, + "theoretical_loss": 3.6985049810443, + "tokens_seen": 942211072 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005773801404212639, + "loss": 1.4254, + "theoretical_loss": 3.6984804207462836, + "tokens_seen": 942276608 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005773640922768305, + "loss": 1.4555, + "theoretical_loss": 3.698455862634648, + "tokens_seen": 942342144 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005773480441323973, + "loss": 1.4384, + "theoretical_loss": 3.6984313067090477, + "tokens_seen": 942407680 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005773319959879639, + "loss": 1.5153, + "theoretical_loss": 3.698406752969134, + "tokens_seen": 942473216 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005773159478435307, + "loss": 1.4654, + "theoretical_loss": 3.698382201414561, + "tokens_seen": 942538752 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005772998996990973, + "loss": 1.4083, + "theoretical_loss": 3.6983576520449835, + "tokens_seen": 942604288 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005772838515546641, + "loss": 1.3192, + "theoretical_loss": 3.6983331048600534, + "tokens_seen": 942669824 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005772678034102307, + "loss": 1.4228, + "theoretical_loss": 3.6983085598594254, + "tokens_seen": 942735360 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005772517552657975, + "loss": 1.5055, + "theoretical_loss": 3.698284017042753, + "tokens_seen": 942800896 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005772357071213641, + "loss": 1.5546, + "theoretical_loss": 3.6982594764096905, + "tokens_seen": 942866432 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005772196589769309, + "loss": 1.4477, + "theoretical_loss": 3.6982349379598913, + "tokens_seen": 942931968 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005772036108324975, + "loss": 1.3151, + "theoretical_loss": 3.69821040169301, + "tokens_seen": 942997504 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005771875626880643, + "loss": 1.4668, + "theoretical_loss": 3.6981858676087005, + "tokens_seen": 943063040 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005771715145436309, + "loss": 1.4375, + "theoretical_loss": 3.698161335706617, + "tokens_seen": 943128576 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005771554663991977, + "loss": 1.4107, + "theoretical_loss": 3.6981368059864135, + "tokens_seen": 943194112 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005771394182547643, + "loss": 1.5313, + "theoretical_loss": 3.698112278447744, + "tokens_seen": 943259648 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005771233701103311, + "loss": 1.5481, + "theoretical_loss": 3.6980877530902645, + "tokens_seen": 943325184 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005771073219658977, + "loss": 1.4579, + "theoretical_loss": 3.698063229913628, + "tokens_seen": 943390720 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005770912738214645, + "loss": 1.4459, + "theoretical_loss": 3.6980387089174895, + "tokens_seen": 943456256 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005770752256770311, + "loss": 1.4788, + "theoretical_loss": 3.6980141901015045, + "tokens_seen": 943521792 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005770591775325979, + "loss": 1.4671, + "theoretical_loss": 3.697989673465327, + "tokens_seen": 943587328 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005770431293881645, + "loss": 1.5214, + "theoretical_loss": 3.6979651590086107, + "tokens_seen": 943652864 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.992470920085907, + "objective/train/avg_token_score": 0.007529129274189472, + "objective/train/docs_used": 346959, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.3488062620162964, + "objective/train/original_loss": 1.3596160411834717, + "objective/train/theoretical_loss": 3.6979406467310127, + "objective/train/tokens_used": 964178400, + "objective/train/unlikelihood_loss": 0.019912954419851303, + "theoretical_loss": 3.6979406467310127, + "tokens_seen": 943718400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005770270812437313, + "loss": 1.3363, + "theoretical_loss": 3.6979406467310127, + "tokens_seen": 943718400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005770110330992979, + "loss": 1.4589, + "theoretical_loss": 3.6979161366321867, + "tokens_seen": 943783936 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005769949849548647, + "loss": 1.4288, + "theoretical_loss": 3.6978916287117882, + "tokens_seen": 943849472 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005769789368104313, + "loss": 1.4338, + "theoretical_loss": 3.6978671229694724, + "tokens_seen": 943915008 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005769628886659981, + "loss": 1.5017, + "theoretical_loss": 3.697842619404894, + "tokens_seen": 943980544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005769468405215647, + "loss": 1.4147, + "theoretical_loss": 3.6978181180177083, + "tokens_seen": 944046080 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005769307923771315, + "loss": 1.5014, + "theoretical_loss": 3.6977936188075717, + "tokens_seen": 944111616 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005769147442326981, + "loss": 1.4348, + "theoretical_loss": 3.6977691217741384, + "tokens_seen": 944177152 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005768986960882649, + "loss": 1.4235, + "theoretical_loss": 3.697744626917065, + "tokens_seen": 944242688 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005768826479438315, + "loss": 1.4023, + "theoretical_loss": 3.6977201342360067, + "tokens_seen": 944308224 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005768665997993982, + "loss": 1.4513, + "theoretical_loss": 3.6976956437306185, + "tokens_seen": 944373760 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005768505516549649, + "loss": 1.4144, + "theoretical_loss": 3.697671155400557, + "tokens_seen": 944439296 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005768345035105316, + "loss": 1.4534, + "theoretical_loss": 3.6976466692454784, + "tokens_seen": 944504832 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005768184553660983, + "loss": 1.4427, + "theoretical_loss": 3.697622185265037, + "tokens_seen": 944570368 + }, + { + "epoch": 0.29, + "learning_rate": 0.000576802407221665, + "loss": 1.4541, + "theoretical_loss": 3.6975977034588907, + "tokens_seen": 944635904 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005767863590772317, + "loss": 1.3582, + "theoretical_loss": 3.6975732238266943, + "tokens_seen": 944701440 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005767703109327984, + "loss": 1.4606, + "theoretical_loss": 3.697548746368105, + "tokens_seen": 944766976 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005767542627883651, + "loss": 1.4175, + "theoretical_loss": 3.697524271082778, + "tokens_seen": 944832512 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005767382146439318, + "loss": 1.4369, + "theoretical_loss": 3.6974997979703703, + "tokens_seen": 944898048 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005767221664994984, + "loss": 1.4698, + "theoretical_loss": 3.697475327030538, + "tokens_seen": 944963584 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005767061183550652, + "loss": 1.5151, + "theoretical_loss": 3.697450858262938, + "tokens_seen": 945029120 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005766900702106318, + "loss": 1.3299, + "theoretical_loss": 3.697426391667226, + "tokens_seen": 945094656 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005766740220661986, + "loss": 1.4185, + "theoretical_loss": 3.6974019272430594, + "tokens_seen": 945160192 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005766579739217652, + "loss": 1.4125, + "theoretical_loss": 3.697377464990095, + "tokens_seen": 945225728 + }, + { + "epoch": 0.29, + "learning_rate": 0.000576641925777332, + "loss": 1.418, + "theoretical_loss": 3.697353004907989, + "tokens_seen": 945291264 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005766258776328986, + "loss": 1.4882, + "theoretical_loss": 3.697328546996398, + "tokens_seen": 945356800 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005766098294884654, + "loss": 1.3721, + "theoretical_loss": 3.69730409125498, + "tokens_seen": 945422336 + }, + { + "epoch": 0.29, + "learning_rate": 0.000576593781344032, + "loss": 1.324, + "theoretical_loss": 3.6972796376833914, + "tokens_seen": 945487872 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005765777331995988, + "loss": 1.4404, + "theoretical_loss": 3.6972551862812892, + "tokens_seen": 945553408 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005765616850551655, + "loss": 1.3923, + "theoretical_loss": 3.697230737048331, + "tokens_seen": 945618944 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005765456369107322, + "loss": 1.3904, + "theoretical_loss": 3.697206289984174, + "tokens_seen": 945684480 + }, + { + "epoch": 0.29, + "learning_rate": 0.000576529588766299, + "loss": 1.3198, + "theoretical_loss": 3.6971818450884752, + "tokens_seen": 945750016 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005765135406218656, + "loss": 1.4616, + "theoretical_loss": 3.6971574023608924, + "tokens_seen": 945815552 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005764974924774323, + "loss": 1.3897, + "theoretical_loss": 3.6971329618010826, + "tokens_seen": 945881088 + }, + { + "epoch": 0.29, + "learning_rate": 0.000576481444332999, + "loss": 1.3639, + "theoretical_loss": 3.697108523408704, + "tokens_seen": 945946624 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005764653961885658, + "loss": 1.3866, + "theoretical_loss": 3.6970840871834136, + "tokens_seen": 946012160 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005764493480441324, + "loss": 1.2911, + "theoretical_loss": 3.697059653124869, + "tokens_seen": 946077696 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005764332998996992, + "loss": 1.5011, + "theoretical_loss": 3.697035221232729, + "tokens_seen": 946143232 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005764172517552658, + "loss": 1.4955, + "theoretical_loss": 3.697010791506651, + "tokens_seen": 946208768 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005764012036108326, + "loss": 1.4105, + "theoretical_loss": 3.6969863639462925, + "tokens_seen": 946274304 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005763851554663992, + "loss": 1.5014, + "theoretical_loss": 3.696961938551312, + "tokens_seen": 946339840 + }, + { + "epoch": 0.29, + "learning_rate": 0.000576369107321966, + "loss": 1.3319, + "theoretical_loss": 3.696937515321368, + "tokens_seen": 946405376 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005763530591775326, + "loss": 1.4702, + "theoretical_loss": 3.696913094256117, + "tokens_seen": 946470912 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005763370110330994, + "loss": 1.4524, + "theoretical_loss": 3.6968886753552193, + "tokens_seen": 946536448 + }, + { + "epoch": 0.29, + "learning_rate": 0.000576320962888666, + "loss": 1.5089, + "theoretical_loss": 3.6968642586183327, + "tokens_seen": 946601984 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005763049147442328, + "loss": 1.3824, + "theoretical_loss": 3.6968398440451145, + "tokens_seen": 946667520 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005762888665997994, + "loss": 1.4077, + "theoretical_loss": 3.696815431635225, + "tokens_seen": 946733056 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005762728184553662, + "loss": 1.4896, + "theoretical_loss": 3.696791021388321, + "tokens_seen": 946798592 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005762567703109328, + "loss": 1.4269, + "theoretical_loss": 3.6967666133040624, + "tokens_seen": 946864128 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005762407221664996, + "loss": 1.5544, + "theoretical_loss": 3.6967422073821075, + "tokens_seen": 946929664 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.9896473288536072, + "objective/train/avg_token_score": 0.010352644138038158, + "objective/train/docs_used": 348186, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4258931875228882, + "objective/train/original_loss": 1.441110610961914, + "objective/train/theoretical_loss": 3.696717803622115, + "objective/train/tokens_used": 967455200, + "objective/train/unlikelihood_loss": 0.024396885186433792, + "theoretical_loss": 3.696717803622115, + "tokens_seen": 946995200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005762246740220662, + "loss": 1.4232, + "theoretical_loss": 3.696717803622115, + "tokens_seen": 946995200 + }, + { + "epoch": 0.29, + "learning_rate": 0.000576208625877633, + "loss": 1.3301, + "theoretical_loss": 3.696693402023744, + "tokens_seen": 947060736 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005761925777331996, + "loss": 1.3477, + "theoretical_loss": 3.696669002586653, + "tokens_seen": 947126272 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005761765295887664, + "loss": 1.3812, + "theoretical_loss": 3.696644605310502, + "tokens_seen": 947191808 + }, + { + "epoch": 0.29, + "learning_rate": 0.000576160481444333, + "loss": 1.4374, + "theoretical_loss": 3.696620210194949, + "tokens_seen": 947257344 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005761444332998998, + "loss": 1.5031, + "theoretical_loss": 3.696595817239654, + "tokens_seen": 947322880 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005761283851554664, + "loss": 1.5189, + "theoretical_loss": 3.6965714264442764, + "tokens_seen": 947388416 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005761123370110332, + "loss": 1.4565, + "theoretical_loss": 3.6965470378084744, + "tokens_seen": 947453952 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005760962888665998, + "loss": 1.4941, + "theoretical_loss": 3.696522651331909, + "tokens_seen": 947519488 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005760802407221666, + "loss": 1.3616, + "theoretical_loss": 3.6964982670142383, + "tokens_seen": 947585024 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005760641925777332, + "loss": 1.5488, + "theoretical_loss": 3.6964738848551226, + "tokens_seen": 947650560 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005760481444333, + "loss": 1.3719, + "theoretical_loss": 3.696449504854222, + "tokens_seen": 947716096 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005760320962888666, + "loss": 1.4631, + "theoretical_loss": 3.696425127011195, + "tokens_seen": 947781632 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005760160481444334, + "loss": 1.3576, + "theoretical_loss": 3.6964007513257022, + "tokens_seen": 947847168 + }, + { + "epoch": 0.29, + "learning_rate": 0.000576, + "loss": 1.5063, + "theoretical_loss": 3.696376377797404, + "tokens_seen": 947912704 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005759839518555668, + "loss": 1.3107, + "theoretical_loss": 3.696352006425959, + "tokens_seen": 947978240 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005759679037111334, + "loss": 1.4519, + "theoretical_loss": 3.696327637211028, + "tokens_seen": 948043776 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005759518555667002, + "loss": 1.2816, + "theoretical_loss": 3.6963032701522716, + "tokens_seen": 948109312 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005759358074222668, + "loss": 1.4231, + "theoretical_loss": 3.6962789052493497, + "tokens_seen": 948174848 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005759197592778336, + "loss": 1.3763, + "theoretical_loss": 3.696254542501922, + "tokens_seen": 948240384 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005759037111334002, + "loss": 1.3898, + "theoretical_loss": 3.6962301819096495, + "tokens_seen": 948305920 + }, + { + "epoch": 0.29, + "learning_rate": 0.000575887662988967, + "loss": 1.5039, + "theoretical_loss": 3.696205823472192, + "tokens_seen": 948371456 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005758716148445336, + "loss": 1.4207, + "theoretical_loss": 3.696181467189211, + "tokens_seen": 948436992 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005758555667001004, + "loss": 1.4113, + "theoretical_loss": 3.6961571130603663, + "tokens_seen": 948502528 + }, + { + "epoch": 0.29, + "learning_rate": 0.000575839518555667, + "loss": 1.6238, + "theoretical_loss": 3.6961327610853187, + "tokens_seen": 948568064 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005758234704112338, + "loss": 1.3595, + "theoretical_loss": 3.6961084112637295, + "tokens_seen": 948633600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005758074222668004, + "loss": 1.4878, + "theoretical_loss": 3.6960840635952583, + "tokens_seen": 948699136 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005757913741223672, + "loss": 1.4258, + "theoretical_loss": 3.696059718079567, + "tokens_seen": 948764672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005757753259779338, + "loss": 1.5191, + "theoretical_loss": 3.696035374716316, + "tokens_seen": 948830208 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005757592778335006, + "loss": 1.4355, + "theoretical_loss": 3.696011033505167, + "tokens_seen": 948895744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005757432296890672, + "loss": 1.4967, + "theoretical_loss": 3.6959866944457813, + "tokens_seen": 948961280 + }, + { + "epoch": 0.29, + "learning_rate": 0.000575727181544634, + "loss": 1.4289, + "theoretical_loss": 3.6959623575378187, + "tokens_seen": 949026816 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005757111334002006, + "loss": 1.459, + "theoretical_loss": 3.6959380227809415, + "tokens_seen": 949092352 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005756950852557674, + "loss": 1.4545, + "theoretical_loss": 3.6959136901748115, + "tokens_seen": 949157888 + }, + { + "epoch": 0.29, + "learning_rate": 0.000575679037111334, + "loss": 1.4198, + "theoretical_loss": 3.695889359719089, + "tokens_seen": 949223424 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005756629889669008, + "loss": 1.4497, + "theoretical_loss": 3.695865031413436, + "tokens_seen": 949288960 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005756469408224674, + "loss": 1.4557, + "theoretical_loss": 3.695840705257514, + "tokens_seen": 949354496 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005756308926780342, + "loss": 1.4518, + "theoretical_loss": 3.6958163812509857, + "tokens_seen": 949420032 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005756148445336008, + "loss": 1.5259, + "theoretical_loss": 3.695792059393511, + "tokens_seen": 949485568 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005755987963891676, + "loss": 1.5057, + "theoretical_loss": 3.695767739684753, + "tokens_seen": 949551104 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005755827482447342, + "loss": 1.4224, + "theoretical_loss": 3.695743422124373, + "tokens_seen": 949616640 + }, + { + "epoch": 0.29, + "learning_rate": 0.000575566700100301, + "loss": 1.4185, + "theoretical_loss": 3.6957191067120334, + "tokens_seen": 949682176 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005755506519558676, + "loss": 1.4251, + "theoretical_loss": 3.6956947934473963, + "tokens_seen": 949747712 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005755346038114344, + "loss": 1.5348, + "theoretical_loss": 3.695670482330123, + "tokens_seen": 949813248 + }, + { + "epoch": 0.29, + "learning_rate": 0.000575518555667001, + "loss": 1.3905, + "theoretical_loss": 3.6956461733598767, + "tokens_seen": 949878784 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005755025075225678, + "loss": 1.4245, + "theoretical_loss": 3.695621866536319, + "tokens_seen": 949944320 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005754864593781344, + "loss": 1.3924, + "theoretical_loss": 3.6955975618591133, + "tokens_seen": 950009856 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005754704112337011, + "loss": 1.4646, + "theoretical_loss": 3.6955732593279205, + "tokens_seen": 950075392 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005754543630892678, + "loss": 1.4878, + "theoretical_loss": 3.6955489589424033, + "tokens_seen": 950140928 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005754383149448345, + "loss": 1.409, + "theoretical_loss": 3.6955246607022256, + "tokens_seen": 950206464 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.9935845136642456, + "objective/train/avg_token_score": 0.0064155180007219315, + "objective/train/docs_used": 349336, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.3599454164505005, + "objective/train/original_loss": 1.3695380687713623, + "objective/train/theoretical_loss": 3.695500364607049, + "objective/train/tokens_used": 970732000, + "objective/train/unlikelihood_loss": 0.015294418670237064, + "theoretical_loss": 3.695500364607049, + "tokens_seen": 950272000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005754222668004012, + "loss": 1.3767, + "theoretical_loss": 3.695500364607049, + "tokens_seen": 950272000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005754062186559679, + "loss": 1.3545, + "theoretical_loss": 3.695476070656537, + "tokens_seen": 950337536 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005753901705115346, + "loss": 1.561, + "theoretical_loss": 3.6954517788503507, + "tokens_seen": 950403072 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005753741223671013, + "loss": 1.2912, + "theoretical_loss": 3.695427489188155, + "tokens_seen": 950468608 + }, + { + "epoch": 0.29, + "learning_rate": 0.000575358074222668, + "loss": 1.3592, + "theoretical_loss": 3.6954032016696123, + "tokens_seen": 950534144 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005753420260782347, + "loss": 1.4115, + "theoretical_loss": 3.6953789162943846, + "tokens_seen": 950599680 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005753259779338014, + "loss": 1.5678, + "theoretical_loss": 3.695354633062136, + "tokens_seen": 950665216 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005753099297893681, + "loss": 1.3724, + "theoretical_loss": 3.6953303519725296, + "tokens_seen": 950730752 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005752938816449348, + "loss": 1.3845, + "theoretical_loss": 3.695306073025229, + "tokens_seen": 950796288 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005752778335005015, + "loss": 1.488, + "theoretical_loss": 3.6952817962198967, + "tokens_seen": 950861824 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005752617853560682, + "loss": 1.4743, + "theoretical_loss": 3.6952575215561962, + "tokens_seen": 950927360 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005752457372116349, + "loss": 1.3673, + "theoretical_loss": 3.695233249033792, + "tokens_seen": 950992896 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005752296890672016, + "loss": 1.3989, + "theoretical_loss": 3.6952089786523468, + "tokens_seen": 951058432 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005752136409227683, + "loss": 1.3851, + "theoretical_loss": 3.6951847104115236, + "tokens_seen": 951123968 + }, + { + "epoch": 0.29, + "learning_rate": 0.000575197592778335, + "loss": 1.4676, + "theoretical_loss": 3.6951604443109876, + "tokens_seen": 951189504 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005751815446339017, + "loss": 1.4801, + "theoretical_loss": 3.695136180350402, + "tokens_seen": 951255040 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005751654964894684, + "loss": 1.4607, + "theoretical_loss": 3.69511191852943, + "tokens_seen": 951320576 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005751494483450351, + "loss": 1.4973, + "theoretical_loss": 3.695087658847737, + "tokens_seen": 951386112 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005751334002006018, + "loss": 1.3961, + "theoretical_loss": 3.695063401304986, + "tokens_seen": 951451648 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005751173520561685, + "loss": 1.4728, + "theoretical_loss": 3.695039145900841, + "tokens_seen": 951517184 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005751013039117353, + "loss": 1.3311, + "theoretical_loss": 3.6950148926349664, + "tokens_seen": 951582720 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005750852557673019, + "loss": 1.4994, + "theoretical_loss": 3.694990641507027, + "tokens_seen": 951648256 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005750692076228687, + "loss": 1.4416, + "theoretical_loss": 3.694966392516686, + "tokens_seen": 951713792 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005750531594784353, + "loss": 1.4119, + "theoretical_loss": 3.6949421456636085, + "tokens_seen": 951779328 + }, + { + "epoch": 0.29, + "learning_rate": 0.000575037111334002, + "loss": 1.447, + "theoretical_loss": 3.694917900947459, + "tokens_seen": 951844864 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005750210631895687, + "loss": 1.3885, + "theoretical_loss": 3.6948936583679024, + "tokens_seen": 951910400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005750050150451355, + "loss": 1.2902, + "theoretical_loss": 3.694869417924603, + "tokens_seen": 951975936 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005749889669007021, + "loss": 1.3496, + "theoretical_loss": 3.694845179617225, + "tokens_seen": 952041472 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005749729187562689, + "loss": 1.5265, + "theoretical_loss": 3.6948209434454338, + "tokens_seen": 952107008 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005749568706118355, + "loss": 1.3638, + "theoretical_loss": 3.6947967094088936, + "tokens_seen": 952172544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005749408224674023, + "loss": 1.5204, + "theoretical_loss": 3.6947724775072706, + "tokens_seen": 952238080 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005749247743229689, + "loss": 1.3978, + "theoretical_loss": 3.6947482477402285, + "tokens_seen": 952303616 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005749087261785357, + "loss": 1.381, + "theoretical_loss": 3.694724020107433, + "tokens_seen": 952369152 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005748926780341023, + "loss": 1.5038, + "theoretical_loss": 3.6946997946085487, + "tokens_seen": 952434688 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005748766298896691, + "loss": 1.4775, + "theoretical_loss": 3.694675571243242, + "tokens_seen": 952500224 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005748605817452357, + "loss": 1.3749, + "theoretical_loss": 3.6946513500111777, + "tokens_seen": 952565760 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005748445336008025, + "loss": 1.4181, + "theoretical_loss": 3.6946271309120204, + "tokens_seen": 952631296 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005748284854563691, + "loss": 1.4325, + "theoretical_loss": 3.6946029139454364, + "tokens_seen": 952696832 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005748124373119359, + "loss": 1.3367, + "theoretical_loss": 3.6945786991110907, + "tokens_seen": 952762368 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005747963891675025, + "loss": 1.4236, + "theoretical_loss": 3.6945544864086495, + "tokens_seen": 952827904 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005747803410230693, + "loss": 1.4346, + "theoretical_loss": 3.6945302758377783, + "tokens_seen": 952893440 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005747642928786359, + "loss": 1.426, + "theoretical_loss": 3.6945060673981427, + "tokens_seen": 952958976 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005747482447342027, + "loss": 1.3744, + "theoretical_loss": 3.6944818610894083, + "tokens_seen": 953024512 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005747321965897693, + "loss": 1.3727, + "theoretical_loss": 3.6944576569112417, + "tokens_seen": 953090048 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005747161484453361, + "loss": 1.4121, + "theoretical_loss": 3.694433454863309, + "tokens_seen": 953155584 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005747001003009027, + "loss": 1.4975, + "theoretical_loss": 3.694409254945275, + "tokens_seen": 953221120 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005746840521564695, + "loss": 1.5149, + "theoretical_loss": 3.694385057156807, + "tokens_seen": 953286656 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005746680040120361, + "loss": 1.4433, + "theoretical_loss": 3.6943608614975707, + "tokens_seen": 953352192 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005746519558676029, + "loss": 1.5269, + "theoretical_loss": 3.6943366679672325, + "tokens_seen": 953417728 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005746359077231695, + "loss": 1.2469, + "theoretical_loss": 3.6943124765654587, + "tokens_seen": 953483264 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.9955537915229797, + "objective/train/avg_token_score": 0.004446200095117092, + "objective/train/docs_used": 350619, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.3078944683074951, + "objective/train/original_loss": 1.313474178314209, + "objective/train/theoretical_loss": 3.694288287291916, + "objective/train/tokens_used": 974008800, + "objective/train/unlikelihood_loss": 0.01200417522341013, + "theoretical_loss": 3.694288287291916, + "tokens_seen": 953548800 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005746198595787363, + "loss": 1.3653, + "theoretical_loss": 3.694288287291916, + "tokens_seen": 953548800 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005746038114343029, + "loss": 1.3701, + "theoretical_loss": 3.6942641001462713, + "tokens_seen": 953614336 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005745877632898697, + "loss": 1.4399, + "theoretical_loss": 3.6942399151281897, + "tokens_seen": 953679872 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005745717151454363, + "loss": 1.3627, + "theoretical_loss": 3.6942157322373395, + "tokens_seen": 953745408 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005745556670010031, + "loss": 1.5767, + "theoretical_loss": 3.694191551473387, + "tokens_seen": 953810944 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005745396188565697, + "loss": 1.4588, + "theoretical_loss": 3.694167372835998, + "tokens_seen": 953876480 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005745235707121365, + "loss": 1.445, + "theoretical_loss": 3.6941431963248403, + "tokens_seen": 953942016 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005745075225677031, + "loss": 1.4082, + "theoretical_loss": 3.6941190219395814, + "tokens_seen": 954007552 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005744914744232699, + "loss": 1.4458, + "theoretical_loss": 3.6940948496798875, + "tokens_seen": 954073088 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005744754262788365, + "loss": 1.3973, + "theoretical_loss": 3.694070679545426, + "tokens_seen": 954138624 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005744593781344033, + "loss": 1.3156, + "theoretical_loss": 3.6940465115358636, + "tokens_seen": 954204160 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005744433299899699, + "loss": 1.4195, + "theoretical_loss": 3.6940223456508683, + "tokens_seen": 954269696 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005744272818455367, + "loss": 1.4808, + "theoretical_loss": 3.693998181890107, + "tokens_seen": 954335232 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005744112337011033, + "loss": 1.3856, + "theoretical_loss": 3.693974020253248, + "tokens_seen": 954400768 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005743951855566701, + "loss": 1.3556, + "theoretical_loss": 3.6939498607399566, + "tokens_seen": 954466304 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005743791374122367, + "loss": 1.3716, + "theoretical_loss": 3.693925703349903, + "tokens_seen": 954531840 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005743630892678035, + "loss": 1.4167, + "theoretical_loss": 3.6939015480827533, + "tokens_seen": 954597376 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005743470411233701, + "loss": 1.4155, + "theoretical_loss": 3.6938773949381756, + "tokens_seen": 954662912 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005743309929789369, + "loss": 1.4503, + "theoretical_loss": 3.6938532439158376, + "tokens_seen": 954728448 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005743149448345035, + "loss": 1.3863, + "theoretical_loss": 3.693829095015407, + "tokens_seen": 954793984 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005742988966900703, + "loss": 1.4527, + "theoretical_loss": 3.6938049482365525, + "tokens_seen": 954859520 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005742828485456369, + "loss": 1.4405, + "theoretical_loss": 3.6937808035789415, + "tokens_seen": 954925056 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005742668004012037, + "loss": 1.3595, + "theoretical_loss": 3.6937566610422423, + "tokens_seen": 954990592 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005742507522567703, + "loss": 1.464, + "theoretical_loss": 3.6937325206261225, + "tokens_seen": 955056128 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005742347041123371, + "loss": 1.2907, + "theoretical_loss": 3.693708382330251, + "tokens_seen": 955121664 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005742186559679037, + "loss": 1.4946, + "theoretical_loss": 3.6936842461542962, + "tokens_seen": 955187200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005742026078234705, + "loss": 1.4911, + "theoretical_loss": 3.693660112097926, + "tokens_seen": 955252736 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005741865596790371, + "loss": 1.3662, + "theoretical_loss": 3.693635980160809, + "tokens_seen": 955318272 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005741705115346039, + "loss": 1.4219, + "theoretical_loss": 3.693611850342614, + "tokens_seen": 955383808 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005741544633901705, + "loss": 1.4727, + "theoretical_loss": 3.6935877226430094, + "tokens_seen": 955449344 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005741384152457373, + "loss": 1.384, + "theoretical_loss": 3.6935635970616643, + "tokens_seen": 955514880 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005741223671013039, + "loss": 1.3805, + "theoretical_loss": 3.6935394735982463, + "tokens_seen": 955580416 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005741063189568707, + "loss": 1.4131, + "theoretical_loss": 3.6935153522524256, + "tokens_seen": 955645952 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005740902708124373, + "loss": 1.4335, + "theoretical_loss": 3.6934912330238703, + "tokens_seen": 955711488 + }, + { + "epoch": 0.29, + "learning_rate": 0.000574074222668004, + "loss": 1.4622, + "theoretical_loss": 3.6934671159122496, + "tokens_seen": 955777024 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005740581745235707, + "loss": 1.4667, + "theoretical_loss": 3.6934430009172328, + "tokens_seen": 955842560 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005740421263791374, + "loss": 1.4533, + "theoretical_loss": 3.693418888038489, + "tokens_seen": 955908096 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005740260782347041, + "loss": 1.3146, + "theoretical_loss": 3.693394777275687, + "tokens_seen": 955973632 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005740100300902708, + "loss": 1.3475, + "theoretical_loss": 3.693370668628496, + "tokens_seen": 956039168 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005739939819458375, + "loss": 1.4853, + "theoretical_loss": 3.693346562096586, + "tokens_seen": 956104704 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005739779338014042, + "loss": 1.4507, + "theoretical_loss": 3.693322457679626, + "tokens_seen": 956170240 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005739618856569709, + "loss": 1.5377, + "theoretical_loss": 3.693298355377286, + "tokens_seen": 956235776 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005739458375125376, + "loss": 1.37, + "theoretical_loss": 3.693274255189235, + "tokens_seen": 956301312 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005739297893681043, + "loss": 1.4139, + "theoretical_loss": 3.693250157115142, + "tokens_seen": 956366848 + }, + { + "epoch": 0.29, + "learning_rate": 0.000573913741223671, + "loss": 1.4663, + "theoretical_loss": 3.6932260611546788, + "tokens_seen": 956432384 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005738976930792377, + "loss": 1.4641, + "theoretical_loss": 3.693201967307514, + "tokens_seen": 956497920 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005738816449348044, + "loss": 1.5955, + "theoretical_loss": 3.6931778755733164, + "tokens_seen": 956563456 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005738655967903711, + "loss": 1.4554, + "theoretical_loss": 3.6931537859517585, + "tokens_seen": 956628992 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005738495486459378, + "loss": 1.4768, + "theoretical_loss": 3.6931296984425077, + "tokens_seen": 956694528 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005738335005015045, + "loss": 1.3869, + "theoretical_loss": 3.693105613045236, + "tokens_seen": 956760064 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.9956148862838745, + "objective/train/avg_token_score": 0.004385089967399836, + "objective/train/docs_used": 351696, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4385031461715698, + "objective/train/original_loss": 1.44512939453125, + "objective/train/theoretical_loss": 3.693081529759612, + "objective/train/tokens_used": 977285600, + "objective/train/unlikelihood_loss": 0.010908097960054874, + "theoretical_loss": 3.693081529759612, + "tokens_seen": 956825600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005738174523570712, + "loss": 1.4134, + "theoretical_loss": 3.693081529759612, + "tokens_seen": 956825600 + }, + { + "epoch": 0.29, + "learning_rate": 0.000573801404212638, + "loss": 1.4756, + "theoretical_loss": 3.6930574485853076, + "tokens_seen": 956891136 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005737853560682046, + "loss": 1.5349, + "theoretical_loss": 3.6930333695219915, + "tokens_seen": 956956672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005737693079237713, + "loss": 1.4134, + "theoretical_loss": 3.693009292569336, + "tokens_seen": 957022208 + }, + { + "epoch": 0.29, + "learning_rate": 0.000573753259779338, + "loss": 1.4277, + "theoretical_loss": 3.69298521772701, + "tokens_seen": 957087744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005737372116349047, + "loss": 1.4548, + "theoretical_loss": 3.692961144994684, + "tokens_seen": 957153280 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005737211634904714, + "loss": 1.5394, + "theoretical_loss": 3.6929370743720296, + "tokens_seen": 957218816 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005737051153460382, + "loss": 1.5405, + "theoretical_loss": 3.692913005858717, + "tokens_seen": 957284352 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005736890672016048, + "loss": 1.4158, + "theoretical_loss": 3.6928889394544173, + "tokens_seen": 957349888 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005736730190571716, + "loss": 1.4048, + "theoretical_loss": 3.6928648751588007, + "tokens_seen": 957415424 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005736569709127382, + "loss": 1.4764, + "theoretical_loss": 3.6928408129715384, + "tokens_seen": 957480960 + }, + { + "epoch": 0.29, + "learning_rate": 0.000573640922768305, + "loss": 1.452, + "theoretical_loss": 3.692816752892302, + "tokens_seen": 957546496 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005736248746238716, + "loss": 1.5052, + "theoretical_loss": 3.6927926949207617, + "tokens_seen": 957612032 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005736088264794384, + "loss": 1.4359, + "theoretical_loss": 3.692768639056589, + "tokens_seen": 957677568 + }, + { + "epoch": 0.29, + "learning_rate": 0.000573592778335005, + "loss": 1.3242, + "theoretical_loss": 3.692744585299455, + "tokens_seen": 957743104 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005735767301905718, + "loss": 1.4153, + "theoretical_loss": 3.692720533649031, + "tokens_seen": 957808640 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005735606820461384, + "loss": 1.3685, + "theoretical_loss": 3.6926964841049887, + "tokens_seen": 957874176 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005735446339017052, + "loss": 1.5126, + "theoretical_loss": 3.692672436666999, + "tokens_seen": 957939712 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005735285857572718, + "loss": 1.3824, + "theoretical_loss": 3.6926483913347337, + "tokens_seen": 958005248 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005735125376128386, + "loss": 1.4377, + "theoretical_loss": 3.692624348107864, + "tokens_seen": 958070784 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005734964894684052, + "loss": 1.4915, + "theoretical_loss": 3.692600306986063, + "tokens_seen": 958136320 + }, + { + "epoch": 0.29, + "learning_rate": 0.000573480441323972, + "loss": 1.4666, + "theoretical_loss": 3.692576267969, + "tokens_seen": 958201856 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005734643931795386, + "loss": 1.486, + "theoretical_loss": 3.6925522310563488, + "tokens_seen": 958267392 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005734483450351054, + "loss": 1.4945, + "theoretical_loss": 3.6925281962477805, + "tokens_seen": 958332928 + }, + { + "epoch": 0.29, + "learning_rate": 0.000573432296890672, + "loss": 1.4544, + "theoretical_loss": 3.692504163542967, + "tokens_seen": 958398464 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005734162487462388, + "loss": 1.4201, + "theoretical_loss": 3.692480132941581, + "tokens_seen": 958464000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005734002006018054, + "loss": 1.4697, + "theoretical_loss": 3.6924561044432935, + "tokens_seen": 958529536 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005733841524573722, + "loss": 1.3711, + "theoretical_loss": 3.6924320780477773, + "tokens_seen": 958595072 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005733681043129388, + "loss": 1.6218, + "theoretical_loss": 3.692408053754705, + "tokens_seen": 958660608 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005733520561685056, + "loss": 1.4313, + "theoretical_loss": 3.6923840315637477, + "tokens_seen": 958726144 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005733360080240722, + "loss": 1.5039, + "theoretical_loss": 3.6923600114745794, + "tokens_seen": 958791680 + }, + { + "epoch": 0.29, + "learning_rate": 0.000573319959879639, + "loss": 1.427, + "theoretical_loss": 3.6923359934868714, + "tokens_seen": 958857216 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005733039117352056, + "loss": 1.4234, + "theoretical_loss": 3.692311977600297, + "tokens_seen": 958922752 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005732878635907724, + "loss": 1.4545, + "theoretical_loss": 3.6922879638145276, + "tokens_seen": 958988288 + }, + { + "epoch": 0.29, + "learning_rate": 0.000573271815446339, + "loss": 1.3986, + "theoretical_loss": 3.6922639521292373, + "tokens_seen": 959053824 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005732557673019058, + "loss": 1.39, + "theoretical_loss": 3.6922399425440977, + "tokens_seen": 959119360 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005732397191574724, + "loss": 1.4268, + "theoretical_loss": 3.6922159350587824, + "tokens_seen": 959184896 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005732236710130392, + "loss": 1.4274, + "theoretical_loss": 3.692191929672964, + "tokens_seen": 959250432 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005732076228686058, + "loss": 1.4673, + "theoretical_loss": 3.6921679263863156, + "tokens_seen": 959315968 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005731915747241726, + "loss": 1.2813, + "theoretical_loss": 3.6921439251985104, + "tokens_seen": 959381504 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005731755265797392, + "loss": 1.4619, + "theoretical_loss": 3.6921199261092212, + "tokens_seen": 959447040 + }, + { + "epoch": 0.29, + "learning_rate": 0.000573159478435306, + "loss": 1.3585, + "theoretical_loss": 3.6920959291181212, + "tokens_seen": 959512576 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005731434302908726, + "loss": 1.5522, + "theoretical_loss": 3.692071934224884, + "tokens_seen": 959578112 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005731273821464394, + "loss": 1.3934, + "theoretical_loss": 3.6920479414291822, + "tokens_seen": 959643648 + }, + { + "epoch": 0.29, + "learning_rate": 0.000573111334002006, + "loss": 1.4871, + "theoretical_loss": 3.6920239507306905, + "tokens_seen": 959709184 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005730952858575728, + "loss": 1.4969, + "theoretical_loss": 3.6919999621290813, + "tokens_seen": 959774720 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005730792377131394, + "loss": 1.5192, + "theoretical_loss": 3.691975975624028, + "tokens_seen": 959840256 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005730631895687062, + "loss": 1.5265, + "theoretical_loss": 3.6919519912152055, + "tokens_seen": 959905792 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005730471414242728, + "loss": 1.3958, + "theoretical_loss": 3.691928008902286, + "tokens_seen": 959971328 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005730310932798396, + "loss": 1.4958, + "theoretical_loss": 3.6919040286849443, + "tokens_seen": 960036864 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.9893192648887634, + "objective/train/avg_token_score": 0.010680736973881721, + "objective/train/docs_used": 352874, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.3688210248947144, + "objective/train/original_loss": 1.3803144693374634, + "objective/train/theoretical_loss": 3.6918800505628546, + "objective/train/tokens_used": 980562400, + "objective/train/unlikelihood_loss": 0.027725940570235252, + "theoretical_loss": 3.6918800505628546, + "tokens_seen": 960102400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005730150451354062, + "loss": 1.4025, + "theoretical_loss": 3.6918800505628546, + "tokens_seen": 960102400 + }, + { + "epoch": 0.29, + "learning_rate": 0.000572998996990973, + "loss": 1.4455, + "theoretical_loss": 3.691856074535689, + "tokens_seen": 960167936 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005729829488465396, + "loss": 1.4202, + "theoretical_loss": 3.691832100603124, + "tokens_seen": 960233472 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005729669007021064, + "loss": 1.5045, + "theoretical_loss": 3.691808128764831, + "tokens_seen": 960299008 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005729508525576731, + "loss": 1.4396, + "theoretical_loss": 3.6917841590204867, + "tokens_seen": 960364544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005729348044132398, + "loss": 1.3975, + "theoretical_loss": 3.6917601913697635, + "tokens_seen": 960430080 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005729187562688065, + "loss": 1.3568, + "theoretical_loss": 3.6917362258123365, + "tokens_seen": 960495616 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005729027081243732, + "loss": 1.3267, + "theoretical_loss": 3.69171226234788, + "tokens_seen": 960561152 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005728866599799399, + "loss": 1.3938, + "theoretical_loss": 3.691688300976068, + "tokens_seen": 960626688 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005728706118355066, + "loss": 1.4751, + "theoretical_loss": 3.6916643416965753, + "tokens_seen": 960692224 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005728545636910733, + "loss": 1.4316, + "theoretical_loss": 3.6916403845090766, + "tokens_seen": 960757760 + }, + { + "epoch": 0.29, + "learning_rate": 0.00057283851554664, + "loss": 1.3498, + "theoretical_loss": 3.691616429413247, + "tokens_seen": 960823296 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005728224674022067, + "loss": 1.4208, + "theoretical_loss": 3.6915924764087604, + "tokens_seen": 960888832 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005728064192577734, + "loss": 1.3993, + "theoretical_loss": 3.6915685254952915, + "tokens_seen": 960954368 + }, + { + "epoch": 0.29, + "learning_rate": 0.00057279037111334, + "loss": 1.4, + "theoretical_loss": 3.6915445766725163, + "tokens_seen": 961019904 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005727743229689068, + "loss": 1.4645, + "theoretical_loss": 3.691520629940108, + "tokens_seen": 961085440 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005727582748244734, + "loss": 1.4335, + "theoretical_loss": 3.6914966852977438, + "tokens_seen": 961150976 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005727422266800402, + "loss": 1.3799, + "theoretical_loss": 3.691472742745097, + "tokens_seen": 961216512 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005727261785356068, + "loss": 1.4858, + "theoretical_loss": 3.6914488022818435, + "tokens_seen": 961282048 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005727101303911736, + "loss": 1.4657, + "theoretical_loss": 3.691424863907659, + "tokens_seen": 961347584 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005726940822467402, + "loss": 1.3901, + "theoretical_loss": 3.691400927622217, + "tokens_seen": 961413120 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005726780341023069, + "loss": 1.3859, + "theoretical_loss": 3.6913769934251954, + "tokens_seen": 961478656 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005726619859578736, + "loss": 1.3051, + "theoretical_loss": 3.691353061316268, + "tokens_seen": 961544192 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005726459378134403, + "loss": 1.4242, + "theoretical_loss": 3.6913291312951104, + "tokens_seen": 961609728 + }, + { + "epoch": 0.29, + "learning_rate": 0.000572629889669007, + "loss": 1.4573, + "theoretical_loss": 3.6913052033613987, + "tokens_seen": 961675264 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005726138415245737, + "loss": 1.4067, + "theoretical_loss": 3.6912812775148085, + "tokens_seen": 961740800 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005725977933801404, + "loss": 1.4714, + "theoretical_loss": 3.691257353755015, + "tokens_seen": 961806336 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005725817452357071, + "loss": 1.4454, + "theoretical_loss": 3.691233432081695, + "tokens_seen": 961871872 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005725656970912738, + "loss": 1.4925, + "theoretical_loss": 3.691209512494524, + "tokens_seen": 961937408 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005725496489468405, + "loss": 1.5526, + "theoretical_loss": 3.6911855949931773, + "tokens_seen": 962002944 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005725336008024072, + "loss": 1.3869, + "theoretical_loss": 3.6911616795773314, + "tokens_seen": 962068480 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005725175526579739, + "loss": 1.4374, + "theoretical_loss": 3.691137766246663, + "tokens_seen": 962134016 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005725015045135406, + "loss": 1.4862, + "theoretical_loss": 3.6911138550008475, + "tokens_seen": 962199552 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005724854563691073, + "loss": 1.464, + "theoretical_loss": 3.691089945839561, + "tokens_seen": 962265088 + }, + { + "epoch": 0.29, + "learning_rate": 0.000572469408224674, + "loss": 1.4242, + "theoretical_loss": 3.6910660387624805, + "tokens_seen": 962330624 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005724533600802407, + "loss": 1.3607, + "theoretical_loss": 3.6910421337692823, + "tokens_seen": 962396160 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005724373119358074, + "loss": 1.4195, + "theoretical_loss": 3.6910182308596426, + "tokens_seen": 962461696 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005724212637913741, + "loss": 1.4729, + "theoretical_loss": 3.6909943300332384, + "tokens_seen": 962527232 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005724052156469408, + "loss": 1.4689, + "theoretical_loss": 3.6909704312897453, + "tokens_seen": 962592768 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005723891675025075, + "loss": 1.3402, + "theoretical_loss": 3.690946534628841, + "tokens_seen": 962658304 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005723731193580742, + "loss": 1.3278, + "theoretical_loss": 3.6909226400502018, + "tokens_seen": 962723840 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005723570712136409, + "loss": 1.3792, + "theoretical_loss": 3.6908987475535047, + "tokens_seen": 962789376 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005723410230692077, + "loss": 1.4572, + "theoretical_loss": 3.690874857138426, + "tokens_seen": 962854912 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005723249749247743, + "loss": 1.4617, + "theoretical_loss": 3.690850968804644, + "tokens_seen": 962920448 + }, + { + "epoch": 0.29, + "learning_rate": 0.000572308926780341, + "loss": 1.3252, + "theoretical_loss": 3.6908270825518343, + "tokens_seen": 962985984 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005722928786359077, + "loss": 1.4433, + "theoretical_loss": 3.690803198379675, + "tokens_seen": 963051520 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005722768304914745, + "loss": 1.3317, + "theoretical_loss": 3.690779316287843, + "tokens_seen": 963117056 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005722607823470411, + "loss": 1.3535, + "theoretical_loss": 3.6907554362760155, + "tokens_seen": 963182592 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005722447342026079, + "loss": 1.4024, + "theoretical_loss": 3.6907315583438693, + "tokens_seen": 963248128 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005722286860581745, + "loss": 1.4901, + "theoretical_loss": 3.690707682491083, + "tokens_seen": 963313664 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.9902940988540649, + "objective/train/avg_token_score": 0.009705900214612484, + "objective/train/docs_used": 353969, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.3660703897476196, + "objective/train/original_loss": 1.3796660900115967, + "objective/train/theoretical_loss": 3.690683808717333, + "objective/train/tokens_used": 983839200, + "objective/train/unlikelihood_loss": 0.028905106708407402, + "theoretical_loss": 3.690683808717333, + "tokens_seen": 963379200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005722126379137413, + "loss": 1.3302, + "theoretical_loss": 3.690683808717333, + "tokens_seen": 963379200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005721965897693079, + "loss": 1.4402, + "theoretical_loss": 3.6906599370222977, + "tokens_seen": 963444736 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005721805416248747, + "loss": 1.5801, + "theoretical_loss": 3.690636067405654, + "tokens_seen": 963510272 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005721644934804413, + "loss": 1.3855, + "theoretical_loss": 3.69061219986708, + "tokens_seen": 963575808 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005721484453360081, + "loss": 1.3888, + "theoretical_loss": 3.6905883344062533, + "tokens_seen": 963641344 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005721323971915747, + "loss": 1.3547, + "theoretical_loss": 3.690564471022852, + "tokens_seen": 963706880 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005721163490471415, + "loss": 1.3504, + "theoretical_loss": 3.690540609716554, + "tokens_seen": 963772416 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005721003009027081, + "loss": 1.3902, + "theoretical_loss": 3.6905167504870366, + "tokens_seen": 963837952 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005720842527582749, + "loss": 1.3952, + "theoretical_loss": 3.6904928933339787, + "tokens_seen": 963903488 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005720682046138415, + "loss": 1.5243, + "theoretical_loss": 3.690469038257058, + "tokens_seen": 963969024 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005720521564694083, + "loss": 1.3732, + "theoretical_loss": 3.6904451852559528, + "tokens_seen": 964034560 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005720361083249749, + "loss": 1.5121, + "theoretical_loss": 3.6904213343303414, + "tokens_seen": 964100096 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005720200601805417, + "loss": 1.3812, + "theoretical_loss": 3.690397485479902, + "tokens_seen": 964165632 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005720040120361083, + "loss": 1.4158, + "theoretical_loss": 3.690373638704313, + "tokens_seen": 964231168 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005719879638916751, + "loss": 1.4434, + "theoretical_loss": 3.6903497940032532, + "tokens_seen": 964296704 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005719719157472417, + "loss": 1.2873, + "theoretical_loss": 3.690325951376401, + "tokens_seen": 964362240 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005719558676028085, + "loss": 1.4523, + "theoretical_loss": 3.690302110823435, + "tokens_seen": 964427776 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005719398194583751, + "loss": 1.5352, + "theoretical_loss": 3.6902782723440337, + "tokens_seen": 964493312 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005719237713139419, + "loss": 1.4451, + "theoretical_loss": 3.6902544359378755, + "tokens_seen": 964558848 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005719077231695085, + "loss": 1.3727, + "theoretical_loss": 3.6902306016046404, + "tokens_seen": 964624384 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005718916750250753, + "loss": 1.5181, + "theoretical_loss": 3.690206769344007, + "tokens_seen": 964689920 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005718756268806419, + "loss": 1.3715, + "theoretical_loss": 3.6901829391556538, + "tokens_seen": 964755456 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005718595787362087, + "loss": 1.5675, + "theoretical_loss": 3.69015911103926, + "tokens_seen": 964820992 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005718435305917753, + "loss": 1.35, + "theoretical_loss": 3.690135284994504, + "tokens_seen": 964886528 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005718274824473421, + "loss": 1.4667, + "theoretical_loss": 3.6901114610210666, + "tokens_seen": 964952064 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005718114343029087, + "loss": 1.4026, + "theoretical_loss": 3.6900876391186266, + "tokens_seen": 965017600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005717953861584755, + "loss": 1.3757, + "theoretical_loss": 3.6900638192868627, + "tokens_seen": 965083136 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005717793380140421, + "loss": 1.3178, + "theoretical_loss": 3.690040001525454, + "tokens_seen": 965148672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005717632898696089, + "loss": 1.4806, + "theoretical_loss": 3.6900161858340814, + "tokens_seen": 965214208 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005717472417251756, + "loss": 1.3003, + "theoretical_loss": 3.689992372212423, + "tokens_seen": 965279744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005717311935807423, + "loss": 1.5376, + "theoretical_loss": 3.6899685606601595, + "tokens_seen": 965345280 + }, + { + "epoch": 0.29, + "learning_rate": 0.000571715145436309, + "loss": 1.6313, + "theoretical_loss": 3.68994475117697, + "tokens_seen": 965410816 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005716990972918757, + "loss": 1.4486, + "theoretical_loss": 3.6899209437625347, + "tokens_seen": 965476352 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005716830491474424, + "loss": 1.3788, + "theoretical_loss": 3.6898971384165327, + "tokens_seen": 965541888 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005716670010030091, + "loss": 1.3778, + "theoretical_loss": 3.6898733351386444, + "tokens_seen": 965607424 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005716509528585758, + "loss": 1.4388, + "theoretical_loss": 3.68984953392855, + "tokens_seen": 965672960 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005716509528585758, + "loss": 1.4595, + "theoretical_loss": 3.68982573478593, + "tokens_seen": 965738496 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005716349047141425, + "loss": 1.4148, + "theoretical_loss": 3.689801937710463, + "tokens_seen": 965804032 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005716188565697092, + "loss": 1.524, + "theoretical_loss": 3.6897781427018304, + "tokens_seen": 965869568 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005716028084252759, + "loss": 1.5289, + "theoretical_loss": 3.6897543497597125, + "tokens_seen": 965935104 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005715867602808426, + "loss": 1.4384, + "theoretical_loss": 3.689730558883789, + "tokens_seen": 966000640 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005715707121364093, + "loss": 1.4825, + "theoretical_loss": 3.68970677007374, + "tokens_seen": 966066176 + }, + { + "epoch": 0.29, + "learning_rate": 0.000571554663991976, + "loss": 1.3264, + "theoretical_loss": 3.6896829833292477, + "tokens_seen": 966131712 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005715386158475427, + "loss": 1.522, + "theoretical_loss": 3.6896591986499905, + "tokens_seen": 966197248 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005715386158475427, + "loss": 1.4641, + "theoretical_loss": 3.689635416035651, + "tokens_seen": 966262784 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005715225677031094, + "loss": 1.4753, + "theoretical_loss": 3.6896116354859085, + "tokens_seen": 966328320 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005715065195586761, + "loss": 1.6486, + "theoretical_loss": 3.6895878570004443, + "tokens_seen": 966393856 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005714904714142428, + "loss": 1.4709, + "theoretical_loss": 3.689564080578939, + "tokens_seen": 966459392 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005714744232698095, + "loss": 1.6646, + "theoretical_loss": 3.689540306221074, + "tokens_seen": 966524928 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005714583751253762, + "loss": 1.467, + "theoretical_loss": 3.6895165339265295, + "tokens_seen": 966590464 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.9932740330696106, + "objective/train/avg_token_score": 0.006726000923663378, + "objective/train/docs_used": 355190, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.6736036539077759, + "objective/train/original_loss": 1.6876091957092285, + "objective/train/theoretical_loss": 3.6894927636949877, + "objective/train/tokens_used": 987116000, + "objective/train/unlikelihood_loss": 0.01146997045725584, + "theoretical_loss": 3.6894927636949877, + "tokens_seen": 966656000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005714423269809429, + "loss": 1.6487, + "theoretical_loss": 3.6894927636949877, + "tokens_seen": 966656000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005714262788365096, + "loss": 1.5357, + "theoretical_loss": 3.689468995526128, + "tokens_seen": 966721536 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005714102306920763, + "loss": 1.6502, + "theoretical_loss": 3.6894452294196336, + "tokens_seen": 966787072 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005713941825476429, + "loss": 1.5221, + "theoretical_loss": 3.6894214653751845, + "tokens_seen": 966852608 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005713781344032097, + "loss": 1.491, + "theoretical_loss": 3.689397703392462, + "tokens_seen": 966918144 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005713620862587763, + "loss": 1.469, + "theoretical_loss": 3.689373943471148, + "tokens_seen": 966983680 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005713460381143431, + "loss": 1.394, + "theoretical_loss": 3.689350185610924, + "tokens_seen": 967049216 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005713299899699097, + "loss": 1.4074, + "theoretical_loss": 3.6893264298114716, + "tokens_seen": 967114752 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005713139418254765, + "loss": 1.4839, + "theoretical_loss": 3.689302676072472, + "tokens_seen": 967180288 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005712978936810431, + "loss": 1.5082, + "theoretical_loss": 3.689278924393607, + "tokens_seen": 967245824 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005712818455366099, + "loss": 1.4761, + "theoretical_loss": 3.689255174774558, + "tokens_seen": 967311360 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005712657973921765, + "loss": 1.4434, + "theoretical_loss": 3.6892314272150077, + "tokens_seen": 967376896 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005712497492477432, + "loss": 1.4077, + "theoretical_loss": 3.6892076817146373, + "tokens_seen": 967442432 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005712337011033099, + "loss": 1.44, + "theoretical_loss": 3.6891839382731297, + "tokens_seen": 967507968 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005712176529588766, + "loss": 1.5196, + "theoretical_loss": 3.6891601968901657, + "tokens_seen": 967573504 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005712016048144433, + "loss": 1.3847, + "theoretical_loss": 3.689136457565428, + "tokens_seen": 967639040 + }, + { + "epoch": 0.29, + "learning_rate": 0.00057118555667001, + "loss": 1.4756, + "theoretical_loss": 3.689112720298599, + "tokens_seen": 967704576 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005711695085255767, + "loss": 1.5459, + "theoretical_loss": 3.6890889850893602, + "tokens_seen": 967770112 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005711534603811434, + "loss": 1.5204, + "theoretical_loss": 3.689065251937395, + "tokens_seen": 967835648 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005711374122367101, + "loss": 1.4881, + "theoretical_loss": 3.6890415208423857, + "tokens_seen": 967901184 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005711213640922768, + "loss": 1.4051, + "theoretical_loss": 3.6890177918040132, + "tokens_seen": 967966720 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005711053159478435, + "loss": 1.472, + "theoretical_loss": 3.688994064821962, + "tokens_seen": 968032256 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005710892678034102, + "loss": 1.5286, + "theoretical_loss": 3.6889703398959135, + "tokens_seen": 968097792 + }, + { + "epoch": 0.29, + "learning_rate": 0.000571073219658977, + "loss": 1.4371, + "theoretical_loss": 3.6889466170255503, + "tokens_seen": 968163328 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005710571715145436, + "loss": 1.5646, + "theoretical_loss": 3.6889228962105562, + "tokens_seen": 968228864 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005710411233701103, + "loss": 1.4763, + "theoretical_loss": 3.6888991774506135, + "tokens_seen": 968294400 + }, + { + "epoch": 0.29, + "learning_rate": 0.000571025075225677, + "loss": 1.4468, + "theoretical_loss": 3.688875460745405, + "tokens_seen": 968359936 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005710090270812437, + "loss": 1.5162, + "theoretical_loss": 3.6888517460946133, + "tokens_seen": 968425472 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005709929789368104, + "loss": 1.3215, + "theoretical_loss": 3.6888280334979218, + "tokens_seen": 968491008 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005709769307923772, + "loss": 1.5102, + "theoretical_loss": 3.688804322955014, + "tokens_seen": 968556544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005709608826479438, + "loss": 1.4381, + "theoretical_loss": 3.688780614465572, + "tokens_seen": 968622080 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005709448345035106, + "loss": 1.4326, + "theoretical_loss": 3.68875690802928, + "tokens_seen": 968687616 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005709287863590772, + "loss": 1.359, + "theoretical_loss": 3.688733203645821, + "tokens_seen": 968753152 + }, + { + "epoch": 0.29, + "learning_rate": 0.000570912738214644, + "loss": 1.5337, + "theoretical_loss": 3.6887095013148787, + "tokens_seen": 968818688 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005708966900702106, + "loss": 1.4992, + "theoretical_loss": 3.6886858010361356, + "tokens_seen": 968884224 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005708806419257774, + "loss": 1.5562, + "theoretical_loss": 3.6886621028092765, + "tokens_seen": 968949760 + }, + { + "epoch": 0.29, + "learning_rate": 0.000570864593781344, + "loss": 1.488, + "theoretical_loss": 3.688638406633984, + "tokens_seen": 969015296 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005708485456369108, + "loss": 1.3361, + "theoretical_loss": 3.688614712509942, + "tokens_seen": 969080832 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005708324974924774, + "loss": 1.3367, + "theoretical_loss": 3.688591020436835, + "tokens_seen": 969146368 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005708164493480442, + "loss": 1.5297, + "theoretical_loss": 3.6885673304143456, + "tokens_seen": 969211904 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005708004012036108, + "loss": 1.4539, + "theoretical_loss": 3.6885436424421583, + "tokens_seen": 969277440 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005707843530591776, + "loss": 1.463, + "theoretical_loss": 3.688519956519957, + "tokens_seen": 969342976 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005707683049147442, + "loss": 1.4663, + "theoretical_loss": 3.6884962726474253, + "tokens_seen": 969408512 + }, + { + "epoch": 0.29, + "learning_rate": 0.000570752256770311, + "loss": 1.493, + "theoretical_loss": 3.688472590824248, + "tokens_seen": 969474048 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005707362086258776, + "loss": 1.4233, + "theoretical_loss": 3.6884489110501093, + "tokens_seen": 969539584 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005707201604814444, + "loss": 1.4019, + "theoretical_loss": 3.6884252333246925, + "tokens_seen": 969605120 + }, + { + "epoch": 0.29, + "learning_rate": 0.000570704112337011, + "loss": 1.4562, + "theoretical_loss": 3.6884015576476825, + "tokens_seen": 969670656 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005706880641925778, + "loss": 1.4397, + "theoretical_loss": 3.688377884018764, + "tokens_seen": 969736192 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005706720160481444, + "loss": 1.3999, + "theoretical_loss": 3.6883542124376207, + "tokens_seen": 969801728 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005706559679037112, + "loss": 1.5453, + "theoretical_loss": 3.6883305429039375, + "tokens_seen": 969867264 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.9915783405303955, + "objective/train/avg_token_score": 0.008421660400927067, + "objective/train/docs_used": 356486, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.3953741788864136, + "objective/train/original_loss": 1.4082467555999756, + "objective/train/theoretical_loss": 3.688306875417399, + "objective/train/tokens_used": 990392800, + "objective/train/unlikelihood_loss": 0.018663326278328896, + "theoretical_loss": 3.688306875417399, + "tokens_seen": 969932800 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005706399197592778, + "loss": 1.4246, + "theoretical_loss": 3.688306875417399, + "tokens_seen": 969932800 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005706238716148446, + "loss": 1.389, + "theoretical_loss": 3.688283209977689, + "tokens_seen": 969998336 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005706078234704112, + "loss": 1.3918, + "theoretical_loss": 3.6882595465844936, + "tokens_seen": 970063872 + }, + { + "epoch": 0.29, + "learning_rate": 0.000570591775325978, + "loss": 1.509, + "theoretical_loss": 3.6882358852374972, + "tokens_seen": 970129408 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005705757271815446, + "loss": 1.498, + "theoretical_loss": 3.688212225936384, + "tokens_seen": 970194944 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005705596790371114, + "loss": 1.3684, + "theoretical_loss": 3.6881885686808396, + "tokens_seen": 970260480 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005705436308926781, + "loss": 1.4738, + "theoretical_loss": 3.688164913470548, + "tokens_seen": 970326016 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005705275827482448, + "loss": 1.5314, + "theoretical_loss": 3.6881412603051955, + "tokens_seen": 970391552 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005705115346038115, + "loss": 1.3679, + "theoretical_loss": 3.6881176091844665, + "tokens_seen": 970457088 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005704954864593782, + "loss": 1.3937, + "theoretical_loss": 3.6880939601080467, + "tokens_seen": 970522624 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005704794383149449, + "loss": 1.4602, + "theoretical_loss": 3.688070313075621, + "tokens_seen": 970588160 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005704633901705116, + "loss": 1.3686, + "theoretical_loss": 3.6880466680868746, + "tokens_seen": 970653696 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005704473420260783, + "loss": 1.5775, + "theoretical_loss": 3.6880230251414936, + "tokens_seen": 970719232 + }, + { + "epoch": 0.29, + "learning_rate": 0.000570431293881645, + "loss": 1.477, + "theoretical_loss": 3.6879993842391627, + "tokens_seen": 970784768 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005704152457372117, + "loss": 1.4258, + "theoretical_loss": 3.6879757453795676, + "tokens_seen": 970850304 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005703991975927784, + "loss": 1.4381, + "theoretical_loss": 3.6879521085623943, + "tokens_seen": 970915840 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005703831494483451, + "loss": 1.479, + "theoretical_loss": 3.687928473787328, + "tokens_seen": 970981376 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005703671013039118, + "loss": 1.4483, + "theoretical_loss": 3.6879048410540554, + "tokens_seen": 971046912 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005703510531594785, + "loss": 1.3992, + "theoretical_loss": 3.687881210362261, + "tokens_seen": 971112448 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005703350050150452, + "loss": 1.5079, + "theoretical_loss": 3.687857581711631, + "tokens_seen": 971177984 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005703189568706119, + "loss": 1.4215, + "theoretical_loss": 3.687833955101852, + "tokens_seen": 971243520 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005703029087261786, + "loss": 1.5182, + "theoretical_loss": 3.68781033053261, + "tokens_seen": 971309056 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005702868605817453, + "loss": 1.3912, + "theoretical_loss": 3.687786708003591, + "tokens_seen": 971374592 + }, + { + "epoch": 0.29, + "learning_rate": 0.000570270812437312, + "loss": 1.5122, + "theoretical_loss": 3.6877630875144805, + "tokens_seen": 971440128 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005702547642928787, + "loss": 1.494, + "theoretical_loss": 3.687739469064966, + "tokens_seen": 971505664 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005702387161484454, + "loss": 1.4265, + "theoretical_loss": 3.6877158526547325, + "tokens_seen": 971571200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005702226680040121, + "loss": 1.4975, + "theoretical_loss": 3.6876922382834674, + "tokens_seen": 971636736 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005702066198595788, + "loss": 1.4108, + "theoretical_loss": 3.687668625950856, + "tokens_seen": 971702272 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005701905717151455, + "loss": 1.3909, + "theoretical_loss": 3.687645015656586, + "tokens_seen": 971767808 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005701745235707122, + "loss": 1.4718, + "theoretical_loss": 3.6876214074003437, + "tokens_seen": 971833344 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005701584754262789, + "loss": 1.4499, + "theoretical_loss": 3.6875978011818153, + "tokens_seen": 971898880 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005701424272818456, + "loss": 1.4062, + "theoretical_loss": 3.6875741970006883, + "tokens_seen": 971964416 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005701263791374123, + "loss": 1.4908, + "theoretical_loss": 3.6875505948566487, + "tokens_seen": 972029952 + }, + { + "epoch": 0.29, + "learning_rate": 0.000570110330992979, + "loss": 1.3973, + "theoretical_loss": 3.6875269947493834, + "tokens_seen": 972095488 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005700942828485457, + "loss": 1.5136, + "theoretical_loss": 3.68750339667858, + "tokens_seen": 972161024 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005700782347041124, + "loss": 1.4217, + "theoretical_loss": 3.687479800643925, + "tokens_seen": 972226560 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005700621865596791, + "loss": 1.3308, + "theoretical_loss": 3.687456206645106, + "tokens_seen": 972292096 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005700461384152458, + "loss": 1.414, + "theoretical_loss": 3.6874326146818097, + "tokens_seen": 972357632 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005700300902708125, + "loss": 1.4968, + "theoretical_loss": 3.687409024753723, + "tokens_seen": 972423168 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005700140421263792, + "loss": 1.3264, + "theoretical_loss": 3.6873854368605334, + "tokens_seen": 972488704 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005699979939819458, + "loss": 1.5074, + "theoretical_loss": 3.687361851001929, + "tokens_seen": 972554240 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005699819458375126, + "loss": 1.3868, + "theoretical_loss": 3.687338267177596, + "tokens_seen": 972619776 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005699658976930792, + "loss": 1.45, + "theoretical_loss": 3.687314685387223, + "tokens_seen": 972685312 + }, + { + "epoch": 0.29, + "learning_rate": 0.000569949849548646, + "loss": 1.5079, + "theoretical_loss": 3.6872911056304973, + "tokens_seen": 972750848 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005699338014042126, + "loss": 1.3822, + "theoretical_loss": 3.687267527907106, + "tokens_seen": 972816384 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005699177532597794, + "loss": 1.5361, + "theoretical_loss": 3.687243952216737, + "tokens_seen": 972881920 + }, + { + "epoch": 0.29, + "learning_rate": 0.000569901705115346, + "loss": 1.3043, + "theoretical_loss": 3.6872203785590782, + "tokens_seen": 972947456 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005698856569709128, + "loss": 1.415, + "theoretical_loss": 3.6871968069338177, + "tokens_seen": 973012992 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005698696088264794, + "loss": 1.5621, + "theoretical_loss": 3.6871732373406436, + "tokens_seen": 973078528 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005698535606820461, + "loss": 1.5566, + "theoretical_loss": 3.6871496697792425, + "tokens_seen": 973144064 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.9938650131225586, + "objective/train/avg_token_score": 0.006134972907602787, + "objective/train/docs_used": 357717, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.4987468719482422, + "objective/train/original_loss": 1.5090758800506592, + "objective/train/theoretical_loss": 3.6871261042493035, + "objective/train/tokens_used": 993669600, + "objective/train/unlikelihood_loss": 0.013454155065119267, + "theoretical_loss": 3.6871261042493035, + "tokens_seen": 973209600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005698375125376128, + "loss": 1.5417, + "theoretical_loss": 3.6871261042493035, + "tokens_seen": 973209600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005698214643931795, + "loss": 1.4672, + "theoretical_loss": 3.687102540750515, + "tokens_seen": 973275136 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005698054162487462, + "loss": 1.3618, + "theoretical_loss": 3.687078979282564, + "tokens_seen": 973340672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005697893681043129, + "loss": 1.3806, + "theoretical_loss": 3.6870554198451404, + "tokens_seen": 973406208 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005697733199598796, + "loss": 1.4736, + "theoretical_loss": 3.687031862437931, + "tokens_seen": 973471744 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005697572718154463, + "loss": 1.3825, + "theoretical_loss": 3.687008307060625, + "tokens_seen": 973537280 + }, + { + "epoch": 0.3, + "learning_rate": 0.000569741223671013, + "loss": 1.3502, + "theoretical_loss": 3.6869847537129106, + "tokens_seen": 973602816 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005697251755265797, + "loss": 1.4388, + "theoretical_loss": 3.6869612023944764, + "tokens_seen": 973668352 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005697091273821464, + "loss": 1.4591, + "theoretical_loss": 3.6869376531050113, + "tokens_seen": 973733888 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005696930792377131, + "loss": 1.441, + "theoretical_loss": 3.686914105844203, + "tokens_seen": 973799424 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005696770310932798, + "loss": 1.3351, + "theoretical_loss": 3.686890560611741, + "tokens_seen": 973864960 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005696609829488465, + "loss": 1.4353, + "theoretical_loss": 3.686867017407314, + "tokens_seen": 973930496 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005696449348044132, + "loss": 1.3649, + "theoretical_loss": 3.686843476230611, + "tokens_seen": 973996032 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005696288866599799, + "loss": 1.4939, + "theoretical_loss": 3.686819937081321, + "tokens_seen": 974061568 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005696128385155467, + "loss": 1.3103, + "theoretical_loss": 3.6867963999591327, + "tokens_seen": 974127104 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005695967903711133, + "loss": 1.3814, + "theoretical_loss": 3.686772864863735, + "tokens_seen": 974192640 + }, + { + "epoch": 0.3, + "learning_rate": 0.00056958074222668, + "loss": 1.4423, + "theoretical_loss": 3.6867493317948172, + "tokens_seen": 974258176 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005695646940822467, + "loss": 1.4695, + "theoretical_loss": 3.686725800752069, + "tokens_seen": 974323712 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005695486459378135, + "loss": 1.3159, + "theoretical_loss": 3.6867022717351787, + "tokens_seen": 974389248 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005695325977933801, + "loss": 1.5326, + "theoretical_loss": 3.686678744743837, + "tokens_seen": 974454784 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005695165496489469, + "loss": 1.4022, + "theoretical_loss": 3.686655219777732, + "tokens_seen": 974520320 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005695005015045135, + "loss": 1.4409, + "theoretical_loss": 3.686631696836554, + "tokens_seen": 974585856 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005694844533600803, + "loss": 1.5614, + "theoretical_loss": 3.686608175919992, + "tokens_seen": 974651392 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005694684052156469, + "loss": 1.3468, + "theoretical_loss": 3.6865846570277365, + "tokens_seen": 974716928 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005694523570712137, + "loss": 1.4458, + "theoretical_loss": 3.6865611401594762, + "tokens_seen": 974782464 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005694363089267803, + "loss": 1.4003, + "theoretical_loss": 3.6865376253149016, + "tokens_seen": 974848000 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005694202607823471, + "loss": 1.4127, + "theoretical_loss": 3.6865141124937013, + "tokens_seen": 974913536 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005694042126379137, + "loss": 1.4691, + "theoretical_loss": 3.686490601695567, + "tokens_seen": 974979072 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005693881644934805, + "loss": 1.4827, + "theoretical_loss": 3.6864670929201875, + "tokens_seen": 975044608 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005693721163490471, + "loss": 1.4547, + "theoretical_loss": 3.686443586167253, + "tokens_seen": 975110144 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005693560682046139, + "loss": 1.534, + "theoretical_loss": 3.686420081436454, + "tokens_seen": 975175680 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005693400200601806, + "loss": 1.4374, + "theoretical_loss": 3.6863965787274804, + "tokens_seen": 975241216 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005693239719157473, + "loss": 1.4739, + "theoretical_loss": 3.686373078040022, + "tokens_seen": 975306752 + }, + { + "epoch": 0.3, + "learning_rate": 0.000569307923771314, + "loss": 1.4327, + "theoretical_loss": 3.6863495793737697, + "tokens_seen": 975372288 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005692918756268807, + "loss": 1.5269, + "theoretical_loss": 3.6863260827284137, + "tokens_seen": 975437824 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005692758274824474, + "loss": 1.436, + "theoretical_loss": 3.6863025881036444, + "tokens_seen": 975503360 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005692597793380141, + "loss": 1.3267, + "theoretical_loss": 3.6862790954991524, + "tokens_seen": 975568896 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005692437311935808, + "loss": 1.3994, + "theoretical_loss": 3.686255604914628, + "tokens_seen": 975634432 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005692276830491475, + "loss": 1.5101, + "theoretical_loss": 3.6862321163497622, + "tokens_seen": 975699968 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005692116349047142, + "loss": 1.4824, + "theoretical_loss": 3.686208629804246, + "tokens_seen": 975765504 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005691955867602809, + "loss": 1.4758, + "theoretical_loss": 3.686185145277769, + "tokens_seen": 975831040 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005691795386158476, + "loss": 1.4411, + "theoretical_loss": 3.686161662770023, + "tokens_seen": 975896576 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005691634904714143, + "loss": 1.294, + "theoretical_loss": 3.686138182280699, + "tokens_seen": 975962112 + }, + { + "epoch": 0.3, + "learning_rate": 0.000569147442326981, + "loss": 1.4578, + "theoretical_loss": 3.686114703809488, + "tokens_seen": 976027648 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005691313941825477, + "loss": 1.4383, + "theoretical_loss": 3.68609122735608, + "tokens_seen": 976093184 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005691153460381144, + "loss": 1.3786, + "theoretical_loss": 3.6860677529201675, + "tokens_seen": 976158720 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005690992978936811, + "loss": 1.4184, + "theoretical_loss": 3.686044280501441, + "tokens_seen": 976224256 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005690832497492478, + "loss": 1.4199, + "theoretical_loss": 3.686020810099592, + "tokens_seen": 976289792 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005690672016048145, + "loss": 1.3941, + "theoretical_loss": 3.6859973417143115, + "tokens_seen": 976355328 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005690511534603812, + "loss": 1.4517, + "theoretical_loss": 3.685973875345291, + "tokens_seen": 976420864 + }, + { + "epoch": 0.3, + "objective/train/aligned_token_count": 0.9938191175460815, + "objective/train/avg_token_score": 0.006180894561111927, + "objective/train/docs_used": 358882, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.2661415338516235, + "objective/train/original_loss": 1.2749769687652588, + "objective/train/theoretical_loss": 3.6859504109922225, + "objective/train/tokens_used": 996946400, + "objective/train/unlikelihood_loss": 0.019851766526699066, + "theoretical_loss": 3.6859504109922225, + "tokens_seen": 976486400 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005690351053159479, + "loss": 1.4057, + "theoretical_loss": 3.6859504109922225, + "tokens_seen": 976486400 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005690190571715146, + "loss": 1.4662, + "theoretical_loss": 3.6859269486547968, + "tokens_seen": 976551936 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005690030090270813, + "loss": 1.3814, + "theoretical_loss": 3.6859034883327055, + "tokens_seen": 976617472 + }, + { + "epoch": 0.3, + "learning_rate": 0.000568986960882648, + "loss": 1.4003, + "theoretical_loss": 3.6858800300256407, + "tokens_seen": 976683008 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005689709127382147, + "loss": 1.3922, + "theoretical_loss": 3.685856573733294, + "tokens_seen": 976748544 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005689548645937814, + "loss": 1.4364, + "theoretical_loss": 3.6858331194553573, + "tokens_seen": 976814080 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005689388164493481, + "loss": 1.4374, + "theoretical_loss": 3.6858096671915224, + "tokens_seen": 976879616 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005689227683049148, + "loss": 1.4661, + "theoretical_loss": 3.685786216941481, + "tokens_seen": 976945152 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005689067201604815, + "loss": 1.4032, + "theoretical_loss": 3.6857627687049255, + "tokens_seen": 977010688 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005688906720160482, + "loss": 1.5781, + "theoretical_loss": 3.6857393224815476, + "tokens_seen": 977076224 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005688746238716149, + "loss": 1.5131, + "theoretical_loss": 3.68571587827104, + "tokens_seen": 977141760 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005688585757271816, + "loss": 1.3923, + "theoretical_loss": 3.6856924360730936, + "tokens_seen": 977207296 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005688425275827483, + "loss": 1.461, + "theoretical_loss": 3.6856689958874025, + "tokens_seen": 977272832 + }, + { + "epoch": 0.3, + "learning_rate": 0.000568826479438315, + "loss": 1.5105, + "theoretical_loss": 3.6856455577136584, + "tokens_seen": 977338368 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005688104312938817, + "loss": 1.5151, + "theoretical_loss": 3.6856221215515528, + "tokens_seen": 977403904 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005687943831494484, + "loss": 1.3509, + "theoretical_loss": 3.685598687400779, + "tokens_seen": 977469440 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005687783350050151, + "loss": 1.5018, + "theoretical_loss": 3.6855752552610292, + "tokens_seen": 977534976 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005687622868605818, + "loss": 1.457, + "theoretical_loss": 3.6855518251319968, + "tokens_seen": 977600512 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005687462387161485, + "loss": 1.3741, + "theoretical_loss": 3.6855283970133734, + "tokens_seen": 977666048 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005687301905717152, + "loss": 1.3654, + "theoretical_loss": 3.6855049709048524, + "tokens_seen": 977731584 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005687141424272819, + "loss": 1.3712, + "theoretical_loss": 3.685481546806127, + "tokens_seen": 977797120 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005686980942828486, + "loss": 1.3857, + "theoretical_loss": 3.685458124716889, + "tokens_seen": 977862656 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005686820461384153, + "loss": 1.3628, + "theoretical_loss": 3.6854347046368314, + "tokens_seen": 977928192 + }, + { + "epoch": 0.3, + "learning_rate": 0.000568665997993982, + "loss": 1.5215, + "theoretical_loss": 3.6854112865656488, + "tokens_seen": 977993728 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005686499498495487, + "loss": 1.5793, + "theoretical_loss": 3.6853878705030327, + "tokens_seen": 978059264 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005686339017051154, + "loss": 1.4535, + "theoretical_loss": 3.685364456448677, + "tokens_seen": 978124800 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005686178535606821, + "loss": 1.3533, + "theoretical_loss": 3.685341044402274, + "tokens_seen": 978190336 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005686018054162487, + "loss": 1.4474, + "theoretical_loss": 3.6853176343635186, + "tokens_seen": 978255872 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005685857572718155, + "loss": 1.4687, + "theoretical_loss": 3.6852942263321022, + "tokens_seen": 978321408 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005685697091273821, + "loss": 1.3903, + "theoretical_loss": 3.6852708203077205, + "tokens_seen": 978386944 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005685536609829489, + "loss": 1.3809, + "theoretical_loss": 3.685247416290065, + "tokens_seen": 978452480 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005685376128385155, + "loss": 1.397, + "theoretical_loss": 3.6852240142788304, + "tokens_seen": 978518016 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005685215646940823, + "loss": 1.4152, + "theoretical_loss": 3.6852006142737093, + "tokens_seen": 978583552 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005685055165496489, + "loss": 1.4551, + "theoretical_loss": 3.685177216274397, + "tokens_seen": 978649088 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005684894684052157, + "loss": 1.4151, + "theoretical_loss": 3.6851538202805854, + "tokens_seen": 978714624 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005684734202607823, + "loss": 1.3598, + "theoretical_loss": 3.6851304262919697, + "tokens_seen": 978780160 + }, + { + "epoch": 0.3, + "learning_rate": 0.000568457372116349, + "loss": 1.5423, + "theoretical_loss": 3.6851070343082433, + "tokens_seen": 978845696 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005684413239719157, + "loss": 1.4245, + "theoretical_loss": 3.6850836443291004, + "tokens_seen": 978911232 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005684252758274824, + "loss": 1.2581, + "theoretical_loss": 3.6850602563542347, + "tokens_seen": 978976768 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005684092276830491, + "loss": 1.4666, + "theoretical_loss": 3.6850368703833403, + "tokens_seen": 979042304 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005683931795386158, + "loss": 1.3929, + "theoretical_loss": 3.6850134864161115, + "tokens_seen": 979107840 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005683771313941825, + "loss": 1.3876, + "theoretical_loss": 3.684990104452243, + "tokens_seen": 979173376 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005683610832497492, + "loss": 1.3209, + "theoretical_loss": 3.684966724491428, + "tokens_seen": 979238912 + }, + { + "epoch": 0.3, + "learning_rate": 0.000568345035105316, + "loss": 1.3353, + "theoretical_loss": 3.6849433465333616, + "tokens_seen": 979304448 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005683289869608826, + "loss": 1.4093, + "theoretical_loss": 3.6849199705777385, + "tokens_seen": 979369984 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005683129388164493, + "loss": 1.3961, + "theoretical_loss": 3.6848965966242524, + "tokens_seen": 979435520 + }, + { + "epoch": 0.3, + "learning_rate": 0.000568296890672016, + "loss": 1.4732, + "theoretical_loss": 3.684873224672599, + "tokens_seen": 979501056 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005682808425275827, + "loss": 1.5119, + "theoretical_loss": 3.6848498547224717, + "tokens_seen": 979566592 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005682647943831494, + "loss": 1.4973, + "theoretical_loss": 3.6848264867735656, + "tokens_seen": 979632128 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005682487462387162, + "loss": 1.5465, + "theoretical_loss": 3.684803120825576, + "tokens_seen": 979697664 + }, + { + "epoch": 0.3, + "objective/train/aligned_token_count": 0.9935358762741089, + "objective/train/avg_token_score": 0.0064641498029232025, + "objective/train/docs_used": 360128, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.5129897594451904, + "objective/train/original_loss": 1.5225951671600342, + "objective/train/theoretical_loss": 3.6847797568781973, + "objective/train/tokens_used": 1000223200, + "objective/train/unlikelihood_loss": 0.016053931787610054, + "theoretical_loss": 3.6847797568781973, + "tokens_seen": 979763200 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005682326980942828, + "loss": 1.4484, + "theoretical_loss": 3.6847797568781973, + "tokens_seen": 979763200 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005682166499498496, + "loss": 1.4361, + "theoretical_loss": 3.6847563949311244, + "tokens_seen": 979828736 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005682006018054162, + "loss": 1.3907, + "theoretical_loss": 3.684733034984052, + "tokens_seen": 979894272 + }, + { + "epoch": 0.3, + "learning_rate": 0.000568184553660983, + "loss": 1.3958, + "theoretical_loss": 3.684709677036676, + "tokens_seen": 979959808 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005681685055165496, + "loss": 1.4613, + "theoretical_loss": 3.6846863210886904, + "tokens_seen": 980025344 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005681524573721164, + "loss": 1.4618, + "theoretical_loss": 3.6846629671397912, + "tokens_seen": 980090880 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005681364092276831, + "loss": 1.4278, + "theoretical_loss": 3.684639615189673, + "tokens_seen": 980156416 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005681203610832498, + "loss": 1.3807, + "theoretical_loss": 3.6846162652380325, + "tokens_seen": 980221952 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005681043129388165, + "loss": 1.4534, + "theoretical_loss": 3.6845929172845633, + "tokens_seen": 980287488 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005680882647943832, + "loss": 1.3934, + "theoretical_loss": 3.6845695713289617, + "tokens_seen": 980353024 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005680722166499499, + "loss": 1.458, + "theoretical_loss": 3.684546227370923, + "tokens_seen": 980418560 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005680561685055166, + "loss": 1.374, + "theoretical_loss": 3.684522885410143, + "tokens_seen": 980484096 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005680401203610833, + "loss": 1.3946, + "theoretical_loss": 3.684499545446317, + "tokens_seen": 980549632 + }, + { + "epoch": 0.3, + "learning_rate": 0.00056802407221665, + "loss": 1.3392, + "theoretical_loss": 3.684476207479141, + "tokens_seen": 980615168 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005680080240722167, + "loss": 1.4511, + "theoretical_loss": 3.6844528715083107, + "tokens_seen": 980680704 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005679919759277834, + "loss": 1.4393, + "theoretical_loss": 3.6844295375335223, + "tokens_seen": 980746240 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005679759277833501, + "loss": 1.4091, + "theoretical_loss": 3.684406205554471, + "tokens_seen": 980811776 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005679598796389168, + "loss": 1.4029, + "theoretical_loss": 3.6843828755708525, + "tokens_seen": 980877312 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005679438314944835, + "loss": 1.3852, + "theoretical_loss": 3.684359547582364, + "tokens_seen": 980942848 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005679277833500502, + "loss": 1.4556, + "theoretical_loss": 3.684336221588701, + "tokens_seen": 981008384 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005679117352056169, + "loss": 1.4376, + "theoretical_loss": 3.6843128975895603, + "tokens_seen": 981073920 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005678956870611836, + "loss": 1.4882, + "theoretical_loss": 3.6842895755846365, + "tokens_seen": 981139456 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005678796389167503, + "loss": 1.4206, + "theoretical_loss": 3.684266255573627, + "tokens_seen": 981204992 + }, + { + "epoch": 0.3, + "learning_rate": 0.000567863590772317, + "loss": 1.4523, + "theoretical_loss": 3.684242937556229, + "tokens_seen": 981270528 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005678475426278837, + "loss": 1.452, + "theoretical_loss": 3.6842196215321374, + "tokens_seen": 981336064 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005678314944834504, + "loss": 1.5247, + "theoretical_loss": 3.6841963075010495, + "tokens_seen": 981401600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005678154463390171, + "loss": 1.3568, + "theoretical_loss": 3.684172995462662, + "tokens_seen": 981467136 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005677993981945838, + "loss": 1.3818, + "theoretical_loss": 3.6841496854166706, + "tokens_seen": 981532672 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005677833500501505, + "loss": 1.5367, + "theoretical_loss": 3.6841263773627735, + "tokens_seen": 981598208 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005677673019057172, + "loss": 1.4593, + "theoretical_loss": 3.684103071300666, + "tokens_seen": 981663744 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005677512537612839, + "loss": 1.4196, + "theoretical_loss": 3.6840797672300463, + "tokens_seen": 981729280 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005677352056168506, + "loss": 1.4439, + "theoretical_loss": 3.6840564651506096, + "tokens_seen": 981794816 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005677191574724173, + "loss": 1.3565, + "theoretical_loss": 3.684033165062055, + "tokens_seen": 981860352 + }, + { + "epoch": 0.3, + "learning_rate": 0.000567703109327984, + "loss": 1.4185, + "theoretical_loss": 3.6840098669640775, + "tokens_seen": 981925888 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005676870611835507, + "loss": 1.4438, + "theoretical_loss": 3.6839865708563755, + "tokens_seen": 981991424 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005676710130391174, + "loss": 1.4193, + "theoretical_loss": 3.6839632767386457, + "tokens_seen": 982056960 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005676549648946841, + "loss": 1.506, + "theoretical_loss": 3.6839399846105847, + "tokens_seen": 982122496 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005676389167502508, + "loss": 1.4396, + "theoretical_loss": 3.6839166944718915, + "tokens_seen": 982188032 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005676228686058175, + "loss": 1.4829, + "theoretical_loss": 3.683893406322262, + "tokens_seen": 982253568 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005676068204613842, + "loss": 1.4054, + "theoretical_loss": 3.6838701201613944, + "tokens_seen": 982319104 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005675907723169509, + "loss": 1.3923, + "theoretical_loss": 3.6838468359889855, + "tokens_seen": 982384640 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005675747241725176, + "loss": 1.4957, + "theoretical_loss": 3.683823553804733, + "tokens_seen": 982450176 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005675586760280843, + "loss": 1.4818, + "theoretical_loss": 3.6838002736083353, + "tokens_seen": 982515712 + }, + { + "epoch": 0.3, + "learning_rate": 0.000567542627883651, + "loss": 1.4862, + "theoretical_loss": 3.6837769953994894, + "tokens_seen": 982581248 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005675265797392177, + "loss": 1.5121, + "theoretical_loss": 3.6837537191778935, + "tokens_seen": 982646784 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005675105315947844, + "loss": 1.4705, + "theoretical_loss": 3.6837304449432446, + "tokens_seen": 982712320 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005674944834503511, + "loss": 1.4731, + "theoretical_loss": 3.683707172695242, + "tokens_seen": 982777856 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005674784353059178, + "loss": 1.4674, + "theoretical_loss": 3.6836839024335823, + "tokens_seen": 982843392 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005674623871614845, + "loss": 1.4049, + "theoretical_loss": 3.683660634157964, + "tokens_seen": 982908928 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005674463390170512, + "loss": 1.3658, + "theoretical_loss": 3.6836373678680854, + "tokens_seen": 982974464 + }, + { + "debugging/Compilability": 1.0, + "debugging/distinct-1-grams": 0.7488293247094114, + "debugging/entropy-1-grams": 5.349564433264487, + "debugging/length": 462.46153846153845, + "debugging/num_segments": 13, + "debugging/score": 0.0073770168052465215, + "debugging/score_std": 0.0054551366535427, + "epoch": 0.3, + "objective/train/aligned_token_count": 0.9902040362358093, + "objective/train/avg_token_score": 0.009795984253287315, + "objective/train/docs_used": 361383, + "objective/train/instantaneous_batch_size": 32, + "objective/train/instantaneous_microbatch_size": 32768, + "objective/train/likelihood_loss": 1.3389241695404053, + "objective/train/original_loss": 1.351909875869751, + "objective/train/theoretical_loss": 3.6836141035636447, + "objective/train/tokens_used": 1003500000, + "objective/train/unlikelihood_loss": 0.02754289098083973, + "theoretical_loss": 3.6836141035636447, + "tokens_seen": 983040000 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005674302908726179, + "loss": 1.4242, + "theoretical_loss": 3.6836141035636447, + "tokens_seen": 983040000 + } + ], + "max_steps": 50354, + "num_train_epochs": 9223372036854775807, + "total_flos": 5.0168070144e+17, + "trial_name": null, + "trial_params": null +}