| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.0, | |
| "global_step": 42724, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 1.1682242990654204e-06, | |
| "loss": 10.8706, | |
| "theoretical_loss": 20.812814784551147, | |
| "tokens_seen": 65536 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 5.841121495327103e-05, | |
| "loss": 9.0299, | |
| "theoretical_loss": 8.563479647615063, | |
| "tokens_seen": 3276800 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00011682242990654206, | |
| "loss": 6.9487, | |
| "theoretical_loss": 7.4777557010520255, | |
| "tokens_seen": 6553600 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00017523364485981307, | |
| "loss": 6.1047, | |
| "theoretical_loss": 6.933751471898896, | |
| "tokens_seen": 9830400 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00023364485981308412, | |
| "loss": 5.7281, | |
| "theoretical_loss": 6.583563211430409, | |
| "tokens_seen": 13107200 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00029205607476635517, | |
| "loss": 5.4625, | |
| "theoretical_loss": 6.330710548120079, | |
| "tokens_seen": 16384000 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00035046728971962614, | |
| "loss": 5.2473, | |
| "theoretical_loss": 6.135526214944321, | |
| "tokens_seen": 19660800 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004088785046728972, | |
| "loss": 5.1303, | |
| "theoretical_loss": 5.978098566873603, | |
| "tokens_seen": 22937600 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00046728971962616824, | |
| "loss": 4.9972, | |
| "theoretical_loss": 5.847114309269919, | |
| "tokens_seen": 26214400 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000499739928125591, | |
| "loss": 4.8435, | |
| "theoretical_loss": 5.73557379888612, | |
| "tokens_seen": 29491200 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004991488556837526, | |
| "loss": 4.7351, | |
| "theoretical_loss": 5.638867127075349, | |
| "tokens_seen": 32768000 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004985577832419141, | |
| "loss": 4.6269, | |
| "theoretical_loss": 5.553809364848902, | |
| "tokens_seen": 36044800 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004979667108000757, | |
| "loss": 4.5409, | |
| "theoretical_loss": 5.478115063560433, | |
| "tokens_seen": 39321600 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004973756383582371, | |
| "loss": 4.428, | |
| "theoretical_loss": 5.410092942583357, | |
| "tokens_seen": 42598400 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004967845659163987, | |
| "loss": 4.4082, | |
| "theoretical_loss": 5.34845906673983, | |
| "tokens_seen": 45875200 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004961934934745603, | |
| "loss": 4.3024, | |
| "theoretical_loss": 5.292217549941562, | |
| "tokens_seen": 49152000 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004956024210327218, | |
| "loss": 4.2468, | |
| "theoretical_loss": 5.240581608773973, | |
| "tokens_seen": 52428800 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004950113485908833, | |
| "loss": 4.161, | |
| "theoretical_loss": 5.192919707529784, | |
| "tokens_seen": 55705600 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004944202761490448, | |
| "loss": 4.0719, | |
| "theoretical_loss": 5.1487178463604355, | |
| "tokens_seen": 58982400 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004938292037072064, | |
| "loss": 3.9664, | |
| "theoretical_loss": 5.107552545409097, | |
| "tokens_seen": 62259200 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004932381312653678, | |
| "loss": 3.8768, | |
| "theoretical_loss": 5.069071100147241, | |
| "tokens_seen": 65536000 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004926470588235294, | |
| "loss": 3.7989, | |
| "theoretical_loss": 5.032976892842002, | |
| "tokens_seen": 68812800 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.000492055986381691, | |
| "loss": 3.7069, | |
| "theoretical_loss": 4.999018291228659, | |
| "tokens_seen": 72089600 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004914649139398525, | |
| "loss": 3.7358, | |
| "theoretical_loss": 4.966980138355957, | |
| "tokens_seen": 75366400 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.000490873841498014, | |
| "loss": 3.6399, | |
| "theoretical_loss": 4.936677144629131, | |
| "tokens_seen": 78643200 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004902827690561755, | |
| "loss": 3.5978, | |
| "theoretical_loss": 4.907948696834077, | |
| "tokens_seen": 81920000 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004896916966143371, | |
| "loss": 3.5751, | |
| "theoretical_loss": 4.880654736816922, | |
| "tokens_seen": 85196800 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004891006241724985, | |
| "loss": 3.5139, | |
| "theoretical_loss": 4.8546724574857745, | |
| "tokens_seen": 88473600 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004885095517306601, | |
| "loss": 3.504, | |
| "theoretical_loss": 4.829893630312808, | |
| "tokens_seen": 91750400 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004879184792888217, | |
| "loss": 3.425, | |
| "theoretical_loss": 4.806222425781916, | |
| "tokens_seen": 95027200 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004873274068469832, | |
| "loss": 3.4333, | |
| "theoretical_loss": 4.783573622280253, | |
| "tokens_seen": 98304000 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.00048673633440514467, | |
| "loss": 3.4695, | |
| "theoretical_loss": 4.7618711237764035, | |
| "tokens_seen": 101580800 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004861452619633062, | |
| "loss": 3.4454, | |
| "theoretical_loss": 4.741046724966468, | |
| "tokens_seen": 104857600 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004855541895214677, | |
| "loss": 3.4016, | |
| "theoretical_loss": 4.721039076253046, | |
| "tokens_seen": 108134400 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004849631170796293, | |
| "loss": 3.4363, | |
| "theoretical_loss": 4.701792811235861, | |
| "tokens_seen": 111411200 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004843720446377908, | |
| "loss": 3.4284, | |
| "theoretical_loss": 4.683257807239925, | |
| "tokens_seen": 114688000 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.00048378097219595233, | |
| "loss": 3.4046, | |
| "theoretical_loss": 4.665388555430277, | |
| "tokens_seen": 117964800 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.00048318989975411385, | |
| "loss": 3.3862, | |
| "theoretical_loss": 4.648143621723734, | |
| "tokens_seen": 121241600 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.00048259882731227537, | |
| "loss": 3.3841, | |
| "theoretical_loss": 4.631485183343639, | |
| "tokens_seen": 124518400 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0004820077548704369, | |
| "loss": 3.3571, | |
| "theoretical_loss": 4.6153786287197125, | |
| "tokens_seen": 127795200 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00048141668242859847, | |
| "loss": 3.3434, | |
| "theoretical_loss": 4.5997922106945, | |
| "tokens_seen": 131072000 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00048082560998676, | |
| "loss": 3.3525, | |
| "theoretical_loss": 4.58469674479667, | |
| "tokens_seen": 134348800 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0004802345375449215, | |
| "loss": 3.3541, | |
| "theoretical_loss": 4.570065345782512, | |
| "tokens_seen": 137625600 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00047964346510308303, | |
| "loss": 3.2821, | |
| "theoretical_loss": 4.555873196808033, | |
| "tokens_seen": 140902400 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00047905239266124455, | |
| "loss": 3.3014, | |
| "theoretical_loss": 4.542097346534794, | |
| "tokens_seen": 144179200 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00047846132021940607, | |
| "loss": 3.2487, | |
| "theoretical_loss": 4.528716530238812, | |
| "tokens_seen": 147456000 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0004778702477775676, | |
| "loss": 3.2912, | |
| "theoretical_loss": 4.515711011618992, | |
| "tokens_seen": 150732800 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047727917533572917, | |
| "loss": 3.2372, | |
| "theoretical_loss": 4.503062442517334, | |
| "tokens_seen": 154009600 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0004766881028938907, | |
| "loss": 3.2413, | |
| "theoretical_loss": 4.4907537381892615, | |
| "tokens_seen": 157286400 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0004760970304520522, | |
| "loss": 3.2631, | |
| "theoretical_loss": 4.478768966115963, | |
| "tokens_seen": 160563200 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "objective/train/docs_used": 101104, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 3.195765972137451, | |
| "objective/train/theoretical_loss": 4.467093246645215, | |
| "objective/train/tokens_used": 184300000, | |
| "theoretical_loss": 4.467093246645215, | |
| "tokens_seen": 163840000 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047550595801021373, | |
| "loss": 3.2345, | |
| "theoretical_loss": 4.467093246645215, | |
| "tokens_seen": 163840000 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047491488556837525, | |
| "loss": 3.2289, | |
| "theoretical_loss": 4.455712663993541, | |
| "tokens_seen": 167116800 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047432381312653677, | |
| "loss": 3.1906, | |
| "theoretical_loss": 4.444614186349425, | |
| "tokens_seen": 170393600 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047373274068469835, | |
| "loss": 3.1652, | |
| "theoretical_loss": 4.433785593991642, | |
| "tokens_seen": 173670400 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047314166824285987, | |
| "loss": 3.1589, | |
| "theoretical_loss": 4.423215414484155, | |
| "tokens_seen": 176947200 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0004725505958010214, | |
| "loss": 3.1271, | |
| "theoretical_loss": 4.412892864134137, | |
| "tokens_seen": 180224000 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0004719595233591829, | |
| "loss": 3.1363, | |
| "theoretical_loss": 4.402807795006074, | |
| "tokens_seen": 183500800 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00047136845091734443, | |
| "loss": 3.1272, | |
| "theoretical_loss": 4.392950646875857, | |
| "tokens_seen": 186777600 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00047077737847550595, | |
| "loss": 3.1839, | |
| "theoretical_loss": 4.383312403586528, | |
| "tokens_seen": 190054400 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0004701863060336675, | |
| "loss": 3.152, | |
| "theoretical_loss": 4.373884553334271, | |
| "tokens_seen": 193331200 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00046959523359182905, | |
| "loss": 3.162, | |
| "theoretical_loss": 4.364659052470699, | |
| "tokens_seen": 196608000 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00046900416114999057, | |
| "loss": 3.1289, | |
| "theoretical_loss": 4.355628292457278, | |
| "tokens_seen": 199884800 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0004684130887081521, | |
| "loss": 3.1229, | |
| "theoretical_loss": 4.346785069650667, | |
| "tokens_seen": 203161600 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0004678220162663136, | |
| "loss": 3.1095, | |
| "theoretical_loss": 4.338122557635156, | |
| "tokens_seen": 206438400 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00046723094382447513, | |
| "loss": 3.1342, | |
| "theoretical_loss": 4.329634281850807, | |
| "tokens_seen": 209715200 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.00046663987138263665, | |
| "loss": 3.143, | |
| "theoretical_loss": 4.321314096294248, | |
| "tokens_seen": 212992000 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0004660487989407982, | |
| "loss": 3.0744, | |
| "theoretical_loss": 4.3131561620937875, | |
| "tokens_seen": 216268800 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.00046545772649895975, | |
| "loss": 3.0913, | |
| "theoretical_loss": 4.305154927782223, | |
| "tokens_seen": 219545600 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.00046486665405712127, | |
| "loss": 3.0461, | |
| "theoretical_loss": 4.297305111109683, | |
| "tokens_seen": 222822400 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0004642755816152828, | |
| "loss": 3.1219, | |
| "theoretical_loss": 4.2896016822555945, | |
| "tokens_seen": 226099200 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0004636845091734443, | |
| "loss": 3.1287, | |
| "theoretical_loss": 4.282039848313611, | |
| "tokens_seen": 229376000 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.00046309343673160583, | |
| "loss": 3.1571, | |
| "theoretical_loss": 4.274615038936293, | |
| "tokens_seen": 232652800 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0004625023642897674, | |
| "loss": 3.1364, | |
| "theoretical_loss": 4.267322893037893, | |
| "tokens_seen": 235929600 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004619112918479289, | |
| "loss": 3.1272, | |
| "theoretical_loss": 4.26015924646374, | |
| "tokens_seen": 239206400 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.00046132021940609044, | |
| "loss": 3.0918, | |
| "theoretical_loss": 4.253120120543809, | |
| "tokens_seen": 242483200 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.00046072914696425197, | |
| "loss": 3.0417, | |
| "theoretical_loss": 4.24620171145605, | |
| "tokens_seen": 245760000 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004601380745224135, | |
| "loss": 3.0699, | |
| "theoretical_loss": 4.239400380332256, | |
| "tokens_seen": 249036800 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.000459547002080575, | |
| "loss": 3.0506, | |
| "theoretical_loss": 4.232712644045627, | |
| "tokens_seen": 252313600 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004589559296387366, | |
| "loss": 3.063, | |
| "theoretical_loss": 4.226135166624862, | |
| "tokens_seen": 255590400 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004583648571968981, | |
| "loss": 3.0794, | |
| "theoretical_loss": 4.21966475124477, | |
| "tokens_seen": 258867200 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004577737847550596, | |
| "loss": 3.0363, | |
| "theoretical_loss": 4.21329833274792, | |
| "tokens_seen": 262144000 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004571827123132211, | |
| "loss": 3.0212, | |
| "theoretical_loss": 4.207032970655965, | |
| "tokens_seen": 265420800 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0004565916398713826, | |
| "loss": 2.9725, | |
| "theoretical_loss": 4.2008658426329974, | |
| "tokens_seen": 268697600 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.00045600056742954413, | |
| "loss": 2.9869, | |
| "theoretical_loss": 4.194794238366544, | |
| "tokens_seen": 271974400 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0004554094949877057, | |
| "loss": 2.9599, | |
| "theoretical_loss": 4.188815553834879, | |
| "tokens_seen": 275251200 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.00045481842254586723, | |
| "loss": 2.921, | |
| "theoretical_loss": 4.182927285931959, | |
| "tokens_seen": 278528000 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.00045422735010402875, | |
| "loss": 2.9792, | |
| "theoretical_loss": 4.177127027423776, | |
| "tokens_seen": 281804800 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.00045363627766219027, | |
| "loss": 3.0045, | |
| "theoretical_loss": 4.171412462212087, | |
| "tokens_seen": 285081600 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0004530452052203518, | |
| "loss": 2.9735, | |
| "theoretical_loss": 4.165781360883513, | |
| "tokens_seen": 288358400 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0004524541327785133, | |
| "loss": 2.9912, | |
| "theoretical_loss": 4.160231576523763, | |
| "tokens_seen": 291635200 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.00045186306033667483, | |
| "loss": 3.0026, | |
| "theoretical_loss": 4.1547610407784346, | |
| "tokens_seen": 294912000 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0004512719878948364, | |
| "loss": 2.9891, | |
| "theoretical_loss": 4.1493677601432815, | |
| "tokens_seen": 298188800 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.00045068091545299793, | |
| "loss": 3.0014, | |
| "theoretical_loss": 4.1440498124682446, | |
| "tokens_seen": 301465600 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.00045008984301115945, | |
| "loss": 3.0057, | |
| "theoretical_loss": 4.138805343660737, | |
| "tokens_seen": 304742400 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.00044949877056932097, | |
| "loss": 2.9975, | |
| "theoretical_loss": 4.133632564574831, | |
| "tokens_seen": 308019200 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0004489076981274825, | |
| "loss": 2.9608, | |
| "theoretical_loss": 4.128529748073999, | |
| "tokens_seen": 311296000 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.000448316625685644, | |
| "loss": 2.9428, | |
| "theoretical_loss": 4.123495226256027, | |
| "tokens_seen": 314572800 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0004477255532438056, | |
| "loss": 2.9603, | |
| "theoretical_loss": 4.118527387829552, | |
| "tokens_seen": 317849600 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0004471344808019671, | |
| "loss": 2.935, | |
| "theoretical_loss": 4.11362467563246, | |
| "tokens_seen": 321126400 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.00044654340836012863, | |
| "loss": 2.8866, | |
| "theoretical_loss": 4.108785584283144, | |
| "tokens_seen": 324403200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.6334409154007015, | |
| "debugging/distinct-1-grams": 0.8097649436454978, | |
| "debugging/distinct-2-grams": 0.9690801766993341, | |
| "debugging/entropy-1-grams": 6.418125228974367, | |
| "debugging/entropy-2-grams": 7.390714834687655, | |
| "debugging/length": 533.0, | |
| "debugging/num_segments": 25, | |
| "epoch": 0.12, | |
| "objective/train/docs_used": 191946, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 3.1231026649475098, | |
| "objective/train/theoretical_loss": 4.104008657956216, | |
| "objective/train/tokens_used": 348140000, | |
| "theoretical_loss": 4.104008657956216, | |
| "tokens_seen": 327680000 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.00044595233591829015, | |
| "loss": 2.9231, | |
| "theoretical_loss": 4.104008657956216, | |
| "tokens_seen": 327680000 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.00044536126347645167, | |
| "loss": 2.9282, | |
| "theoretical_loss": 4.099292488274917, | |
| "tokens_seen": 330956800 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0004447701910346132, | |
| "loss": 2.9623, | |
| "theoretical_loss": 4.094635712313026, | |
| "tokens_seen": 334233600 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.00044417911859277476, | |
| "loss": 2.9152, | |
| "theoretical_loss": 4.090037010699552, | |
| "tokens_seen": 337510400 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0004435880461509363, | |
| "loss": 2.9675, | |
| "theoretical_loss": 4.085495105819987, | |
| "tokens_seen": 340787200 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0004429969737090978, | |
| "loss": 2.9016, | |
| "theoretical_loss": 4.081008760108329, | |
| "tokens_seen": 344064000 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0004424059012672593, | |
| "loss": 2.9007, | |
| "theoretical_loss": 4.076576774424465, | |
| "tokens_seen": 347340800 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00044181482882542085, | |
| "loss": 2.9414, | |
| "theoretical_loss": 4.072197986511911, | |
| "tokens_seen": 350617600 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00044122375638358237, | |
| "loss": 2.8938, | |
| "theoretical_loss": 4.067871269531193, | |
| "tokens_seen": 353894400 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0004406326839417439, | |
| "loss": 2.8917, | |
| "theoretical_loss": 4.063595530664515, | |
| "tokens_seen": 357171200 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00044004161149990546, | |
| "loss": 2.9466, | |
| "theoretical_loss": 4.059369709787625, | |
| "tokens_seen": 360448000 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.000439450539058067, | |
| "loss": 2.9499, | |
| "theoretical_loss": 4.055192778205064, | |
| "tokens_seen": 363724800 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0004388594666162285, | |
| "loss": 2.956, | |
| "theoretical_loss": 4.051063737445231, | |
| "tokens_seen": 367001600 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00043826839417439, | |
| "loss": 2.8953, | |
| "theoretical_loss": 4.046981618111936, | |
| "tokens_seen": 370278400 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00043767732173255155, | |
| "loss": 2.9374, | |
| "theoretical_loss": 4.042945478789308, | |
| "tokens_seen": 373555200 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00043708624929071307, | |
| "loss": 2.9325, | |
| "theoretical_loss": 4.038954404997148, | |
| "tokens_seen": 376832000 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.00043649517684887464, | |
| "loss": 2.8987, | |
| "theoretical_loss": 4.035007508193978, | |
| "tokens_seen": 380108800 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.00043590410440703616, | |
| "loss": 2.9152, | |
| "theoretical_loss": 4.031103924825214, | |
| "tokens_seen": 383385600 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0004353130319651977, | |
| "loss": 2.8626, | |
| "theoretical_loss": 4.027242815414074, | |
| "tokens_seen": 386662400 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0004347219595233592, | |
| "loss": 2.8536, | |
| "theoretical_loss": 4.023423363692939, | |
| "tokens_seen": 389939200 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0004341308870815207, | |
| "loss": 2.9131, | |
| "theoretical_loss": 4.019644775773043, | |
| "tokens_seen": 393216000 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.00043353981463968225, | |
| "loss": 2.9274, | |
| "theoretical_loss": 4.015906279350517, | |
| "tokens_seen": 396492800 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0004329487421978438, | |
| "loss": 2.8901, | |
| "theoretical_loss": 4.01220712294689, | |
| "tokens_seen": 399769600 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.00043235766975600534, | |
| "loss": 2.903, | |
| "theoretical_loss": 4.008546575182286, | |
| "tokens_seen": 403046400 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.00043176659731416686, | |
| "loss": 2.901, | |
| "theoretical_loss": 4.0049239240796695, | |
| "tokens_seen": 406323200 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004311755248723284, | |
| "loss": 2.8966, | |
| "theoretical_loss": 4.001338476398553, | |
| "tokens_seen": 409600000 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004305844524304899, | |
| "loss": 2.8991, | |
| "theoretical_loss": 3.9977895569967217, | |
| "tokens_seen": 412876800 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004299933799886514, | |
| "loss": 2.9085, | |
| "theoretical_loss": 3.99427650821855, | |
| "tokens_seen": 416153600 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.00042940230754681295, | |
| "loss": 2.9123, | |
| "theoretical_loss": 3.990798689308642, | |
| "tokens_seen": 419430400 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004288112351049745, | |
| "loss": 2.9372, | |
| "theoretical_loss": 3.987355475849527, | |
| "tokens_seen": 422707200 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.000428220162663136, | |
| "loss": 2.9329, | |
| "theoretical_loss": 3.98394625922226, | |
| "tokens_seen": 425984000 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004276290902212975, | |
| "loss": 2.922, | |
| "theoretical_loss": 3.9805704460888256, | |
| "tokens_seen": 429260800 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.00042703801777945903, | |
| "loss": 2.9447, | |
| "theoretical_loss": 3.9772274578953013, | |
| "tokens_seen": 432537600 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042644694533762055, | |
| "loss": 2.9067, | |
| "theoretical_loss": 3.973916730394796, | |
| "tokens_seen": 435814400 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042585587289578207, | |
| "loss": 2.9158, | |
| "theoretical_loss": 3.970637713189244, | |
| "tokens_seen": 439091200 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042526480045394365, | |
| "loss": 2.9318, | |
| "theoretical_loss": 3.967389869289161, | |
| "tokens_seen": 442368000 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042467372801210517, | |
| "loss": 2.9163, | |
| "theoretical_loss": 3.964172674690542, | |
| "tokens_seen": 445644800 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0004240826555702667, | |
| "loss": 2.911, | |
| "theoretical_loss": 3.9609856179681078, | |
| "tokens_seen": 448921600 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0004234915831284282, | |
| "loss": 2.8926, | |
| "theoretical_loss": 3.957828199884155, | |
| "tokens_seen": 452198400 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042290051068658973, | |
| "loss": 2.8582, | |
| "theoretical_loss": 3.9546999330123036, | |
| "tokens_seen": 455475200 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042230943824475125, | |
| "loss": 2.8693, | |
| "theoretical_loss": 3.951600341375469, | |
| "tokens_seen": 458752000 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0004217183658029128, | |
| "loss": 2.8617, | |
| "theoretical_loss": 3.9485289600974305, | |
| "tokens_seen": 462028800 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.00042112729336107435, | |
| "loss": 2.8166, | |
| "theoretical_loss": 3.945485335067386, | |
| "tokens_seen": 465305600 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.00042053622091923587, | |
| "loss": 2.8552, | |
| "theoretical_loss": 3.9424690226169212, | |
| "tokens_seen": 468582400 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0004199451484773974, | |
| "loss": 2.874, | |
| "theoretical_loss": 3.9394795892088585, | |
| "tokens_seen": 471859200 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0004193540760355589, | |
| "loss": 2.8287, | |
| "theoretical_loss": 3.9365166111374608, | |
| "tokens_seen": 475136000 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.00041876300359372043, | |
| "loss": 2.8014, | |
| "theoretical_loss": 3.933579674239507, | |
| "tokens_seen": 478412800 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.00041817193115188195, | |
| "loss": 2.8463, | |
| "theoretical_loss": 3.930668373615765, | |
| "tokens_seen": 481689600 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0004175808587100435, | |
| "loss": 2.7963, | |
| "theoretical_loss": 3.9277823133624366, | |
| "tokens_seen": 484966400 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.00041698978626820505, | |
| "loss": 2.8429, | |
| "theoretical_loss": 3.9249211063121283, | |
| "tokens_seen": 488243200 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "objective/train/docs_used": 280313, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.8110482692718506, | |
| "objective/train/theoretical_loss": 3.9220843737839752, | |
| "objective/train/tokens_used": 511980000, | |
| "theoretical_loss": 3.9220843737839752, | |
| "tokens_seen": 491520000 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.00041639871382636657, | |
| "loss": 2.8682, | |
| "theoretical_loss": 3.9220843737839752, | |
| "tokens_seen": 491520000 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0004158076413845281, | |
| "loss": 2.8998, | |
| "theoretical_loss": 3.919271745342514, | |
| "tokens_seen": 494796800 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0004152165689426896, | |
| "loss": 2.8376, | |
| "theoretical_loss": 3.916482858564957, | |
| "tokens_seen": 498073600 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.00041462549650085113, | |
| "loss": 2.8084, | |
| "theoretical_loss": 3.9137173588165135, | |
| "tokens_seen": 501350400 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0004140344240590127, | |
| "loss": 2.8155, | |
| "theoretical_loss": 3.9109748990334348, | |
| "tokens_seen": 504627200 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0004134433516171742, | |
| "loss": 2.8228, | |
| "theoretical_loss": 3.908255139513467, | |
| "tokens_seen": 507904000 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.00041285227917533575, | |
| "loss": 2.8322, | |
| "theoretical_loss": 3.905557747713412, | |
| "tokens_seen": 511180800 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.000412273028182334, | |
| "loss": 2.8328, | |
| "theoretical_loss": 3.90288239805351, | |
| "tokens_seen": 514457600 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.00041168195574049554, | |
| "loss": 2.8241, | |
| "theoretical_loss": 3.9002287717283783, | |
| "tokens_seen": 517734400 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.00041109088329865706, | |
| "loss": 2.7933, | |
| "theoretical_loss": 3.8975965565242423, | |
| "tokens_seen": 521011200 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.00041049981085681863, | |
| "loss": 2.7839, | |
| "theoretical_loss": 3.894985446642206, | |
| "tokens_seen": 524288000 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.00040990873841498016, | |
| "loss": 2.8036, | |
| "theoretical_loss": 3.8923951425273398, | |
| "tokens_seen": 527564800 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0004093176659731417, | |
| "loss": 2.7935, | |
| "theoretical_loss": 3.889825350703344, | |
| "tokens_seen": 530841600 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0004087265935313032, | |
| "loss": 2.8123, | |
| "theoretical_loss": 3.8872757836125844, | |
| "tokens_seen": 534118400 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0004081355210894647, | |
| "loss": 2.8314, | |
| "theoretical_loss": 3.8847461594612884, | |
| "tokens_seen": 537395200 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.00040754444864762624, | |
| "loss": 2.8355, | |
| "theoretical_loss": 3.882236202069703, | |
| "tokens_seen": 540672000 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0004069533762057878, | |
| "loss": 2.8371, | |
| "theoretical_loss": 3.8797456407270343, | |
| "tokens_seen": 543948800 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.00040636230376394933, | |
| "loss": 2.7971, | |
| "theoretical_loss": 3.8772742100509774, | |
| "tokens_seen": 547225600 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.00040577123132211085, | |
| "loss": 2.8083, | |
| "theoretical_loss": 3.874821649851678, | |
| "tokens_seen": 550502400 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004051801588802724, | |
| "loss": 2.8806, | |
| "theoretical_loss": 3.8723877049999444, | |
| "tokens_seen": 553779200 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004045890864384339, | |
| "loss": 2.8402, | |
| "theoretical_loss": 3.869972125299568, | |
| "tokens_seen": 557056000 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004039980139965954, | |
| "loss": 2.8138, | |
| "theoretical_loss": 3.867574665363595, | |
| "tokens_seen": 560332800 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.00040340694155475694, | |
| "loss": 2.8393, | |
| "theoretical_loss": 3.865195084494398, | |
| "tokens_seen": 563609600 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004028158691129185, | |
| "loss": 2.8361, | |
| "theoretical_loss": 3.8628331465674224, | |
| "tokens_seen": 566886400 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.00040222479667108003, | |
| "loss": 2.8911, | |
| "theoretical_loss": 3.860488619918462, | |
| "tokens_seen": 570163200 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.00040163372422924155, | |
| "loss": 2.8633, | |
| "theoretical_loss": 3.858161277234349, | |
| "tokens_seen": 573440000 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0004010426517874031, | |
| "loss": 2.8655, | |
| "theoretical_loss": 3.855850895446925, | |
| "tokens_seen": 576716800 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0004004515793455646, | |
| "loss": 2.8538, | |
| "theoretical_loss": 3.8535572556301823, | |
| "tokens_seen": 579993600 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0003998605069037261, | |
| "loss": 2.8293, | |
| "theoretical_loss": 3.851280142900463, | |
| "tokens_seen": 583270400 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0003992694344618877, | |
| "loss": 2.8696, | |
| "theoretical_loss": 3.8490193463196127, | |
| "tokens_seen": 586547200 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0003986783620200492, | |
| "loss": 2.8401, | |
| "theoretical_loss": 3.8467746588009692, | |
| "tokens_seen": 589824000 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.00039808728957821073, | |
| "loss": 2.8243, | |
| "theoretical_loss": 3.8445458770181116, | |
| "tokens_seen": 593100800 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.00039749621713637225, | |
| "loss": 2.8062, | |
| "theoretical_loss": 3.842332801316254, | |
| "tokens_seen": 596377600 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0003969051446945338, | |
| "loss": 2.7718, | |
| "theoretical_loss": 3.840135235626204, | |
| "tokens_seen": 599654400 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0003963140722526953, | |
| "loss": 2.7977, | |
| "theoretical_loss": 3.8379529873807945, | |
| "tokens_seen": 602931200 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.00039572299981085687, | |
| "loss": 2.7847, | |
| "theoretical_loss": 3.835785867433705, | |
| "tokens_seen": 606208000 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0003951319273690184, | |
| "loss": 2.771, | |
| "theoretical_loss": 3.8336336899805916, | |
| "tokens_seen": 609484800 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0003945408549271799, | |
| "loss": 2.8282, | |
| "theoretical_loss": 3.8314962724824495, | |
| "tokens_seen": 612761600 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.00039394978248534143, | |
| "loss": 2.801, | |
| "theoretical_loss": 3.8293734355911297, | |
| "tokens_seen": 616038400 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.00039335871004350295, | |
| "loss": 2.7742, | |
| "theoretical_loss": 3.8272650030769433, | |
| "tokens_seen": 619315200 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0003927676376016645, | |
| "loss": 2.8178, | |
| "theoretical_loss": 3.8251708017582793, | |
| "tokens_seen": 622592000 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.000392176565159826, | |
| "loss": 2.7556, | |
| "theoretical_loss": 3.8230906614331737, | |
| "tokens_seen": 625868800 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.00039158549271798757, | |
| "loss": 2.7756, | |
| "theoretical_loss": 3.821024414812761, | |
| "tokens_seen": 629145600 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0003909944202761491, | |
| "loss": 2.77, | |
| "theoretical_loss": 3.818971897456552, | |
| "tokens_seen": 632422400 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.00039040334783431056, | |
| "loss": 2.8367, | |
| "theoretical_loss": 3.8169329477094784, | |
| "tokens_seen": 635699200 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0003898122753924721, | |
| "loss": 2.8257, | |
| "theoretical_loss": 3.814907406640639, | |
| "tokens_seen": 638976000 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0003892212029506336, | |
| "loss": 2.8574, | |
| "theoretical_loss": 3.8128951179837056, | |
| "tokens_seen": 642252800 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0003886301305087951, | |
| "loss": 2.8296, | |
| "theoretical_loss": 3.8108959280789243, | |
| "tokens_seen": 645529600 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0003880390580669567, | |
| "loss": 2.8344, | |
| "theoretical_loss": 3.8089096858166718, | |
| "tokens_seen": 648806400 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0003874479856251182, | |
| "loss": 2.8004, | |
| "theoretical_loss": 3.8069362425825037, | |
| "tokens_seen": 652083200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.5268657803535504, | |
| "debugging/distinct-1-grams": 0.7733134705474689, | |
| "debugging/distinct-2-grams": 0.9620239068787645, | |
| "debugging/entropy-1-grams": 5.925348960854147, | |
| "debugging/entropy-2-grams": 6.698235169198012, | |
| "debugging/length": 567.4545454545455, | |
| "debugging/num_segments": 11, | |
| "epoch": 0.23, | |
| "objective/train/docs_used": 369218, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.6233978271484375, | |
| "objective/train/theoretical_loss": 3.8049754522036645, | |
| "objective/train/tokens_used": 675820000, | |
| "theoretical_loss": 3.8049754522036645, | |
| "tokens_seen": 655360000 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.00038685691318327974, | |
| "loss": 2.7853, | |
| "theoretical_loss": 3.8049754522036645, | |
| "tokens_seen": 655360000 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.00038626584074144126, | |
| "loss": 2.7931, | |
| "theoretical_loss": 3.8030271708970003, | |
| "tokens_seen": 658636800 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003856747682996028, | |
| "loss": 2.7683, | |
| "theoretical_loss": 3.801091257218237, | |
| "tokens_seen": 661913600 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003850836958577643, | |
| "loss": 2.7793, | |
| "theoretical_loss": 3.79916757201258, | |
| "tokens_seen": 665190400 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.00038451626631359943, | |
| "loss": 2.8162, | |
| "theoretical_loss": 3.7972559783665965, | |
| "tokens_seen": 668467200 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.00038392519387176095, | |
| "loss": 2.787, | |
| "theoretical_loss": 3.7953563415613325, | |
| "tokens_seen": 671744000 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.00038333412142992247, | |
| "loss": 2.8157, | |
| "theoretical_loss": 3.7934685290266454, | |
| "tokens_seen": 675020800 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.000382743048988084, | |
| "loss": 2.8311, | |
| "theoretical_loss": 3.7915924102966914, | |
| "tokens_seen": 678297600 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003821519765462455, | |
| "loss": 2.8514, | |
| "theoretical_loss": 3.789727856966552, | |
| "tokens_seen": 681574400 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.00038156090410440703, | |
| "loss": 2.7928, | |
| "theoretical_loss": 3.787874742649958, | |
| "tokens_seen": 684851200 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0003809698316625686, | |
| "loss": 2.8251, | |
| "theoretical_loss": 3.786032942938073, | |
| "tokens_seen": 688128000 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.00038037875922073013, | |
| "loss": 2.8142, | |
| "theoretical_loss": 3.784202335359316, | |
| "tokens_seen": 691404800 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.00037978768677889165, | |
| "loss": 2.809, | |
| "theoretical_loss": 3.7823827993401844, | |
| "tokens_seen": 694681600 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.00037919661433705317, | |
| "loss": 2.7972, | |
| "theoretical_loss": 3.7805742161670466, | |
| "tokens_seen": 697958400 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0003786055418952147, | |
| "loss": 2.7828, | |
| "theoretical_loss": 3.7787764689488847, | |
| "tokens_seen": 701235200 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0003780144694533762, | |
| "loss": 2.7532, | |
| "theoretical_loss": 3.77698944258095, | |
| "tokens_seen": 704512000 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.00037742339701153773, | |
| "loss": 2.7904, | |
| "theoretical_loss": 3.7752130237093064, | |
| "tokens_seen": 707788800 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0003768323245696993, | |
| "loss": 2.7898, | |
| "theoretical_loss": 3.773447100696245, | |
| "tokens_seen": 711065600 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.00037624125212786083, | |
| "loss": 2.8074, | |
| "theoretical_loss": 3.771691563586529, | |
| "tokens_seen": 714342400 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.00037565017968602235, | |
| "loss": 2.7717, | |
| "theoretical_loss": 3.7699463040744616, | |
| "tokens_seen": 717619200 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.00037505910724418387, | |
| "loss": 2.7412, | |
| "theoretical_loss": 3.768211215471741, | |
| "tokens_seen": 720896000 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0003744680348023454, | |
| "loss": 2.7547, | |
| "theoretical_loss": 3.766486192676084, | |
| "tokens_seen": 724172800 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0003738769623605069, | |
| "loss": 2.7029, | |
| "theoretical_loss": 3.764771132140602, | |
| "tokens_seen": 727449600 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0003732858899186685, | |
| "loss": 2.7481, | |
| "theoretical_loss": 3.763065931843898, | |
| "tokens_seen": 730726400 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.00037269481747683, | |
| "loss": 2.7378, | |
| "theoretical_loss": 3.7613704912608723, | |
| "tokens_seen": 734003200 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.00037210374503499153, | |
| "loss": 2.7671, | |
| "theoretical_loss": 3.759684711334213, | |
| "tokens_seen": 737280000 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.000371512672593153, | |
| "loss": 2.7614, | |
| "theoretical_loss": 3.7580084944465555, | |
| "tokens_seen": 740556800 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0003709216001513145, | |
| "loss": 2.7923, | |
| "theoretical_loss": 3.7563417443932905, | |
| "tokens_seen": 743833600 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.00037033052770947604, | |
| "loss": 2.7829, | |
| "theoretical_loss": 3.754684366355999, | |
| "tokens_seen": 747110400 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0003697394552676376, | |
| "loss": 2.7462, | |
| "theoretical_loss": 3.753036266876505, | |
| "tokens_seen": 750387200 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.00036914838282579913, | |
| "loss": 2.7509, | |
| "theoretical_loss": 3.751397353831524, | |
| "tokens_seen": 753664000 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.00036855731038396065, | |
| "loss": 2.7093, | |
| "theoretical_loss": 3.749767536407891, | |
| "tokens_seen": 756940800 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0003679662379421222, | |
| "loss": 2.7551, | |
| "theoretical_loss": 3.7481467250783504, | |
| "tokens_seen": 760217600 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0003673751655002837, | |
| "loss": 2.7483, | |
| "theoretical_loss": 3.746534831577904, | |
| "tokens_seen": 763494400 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0003667840930584452, | |
| "loss": 2.7002, | |
| "theoretical_loss": 3.744931768880681, | |
| "tokens_seen": 766771200 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.00036619302061660674, | |
| "loss": 2.7493, | |
| "theoretical_loss": 3.743337451177343, | |
| "tokens_seen": 770048000 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0003656019481747683, | |
| "loss": 2.7132, | |
| "theoretical_loss": 3.74175179385298, | |
| "tokens_seen": 773324800 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.00036501087573292983, | |
| "loss": 2.7777, | |
| "theoretical_loss": 3.740174713465512, | |
| "tokens_seen": 776601600 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.00036441980329109135, | |
| "loss": 2.7334, | |
| "theoretical_loss": 3.7386061277245655, | |
| "tokens_seen": 779878400 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0003638287308492529, | |
| "loss": 2.7612, | |
| "theoretical_loss": 3.7370459554708146, | |
| "tokens_seen": 783155200 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0003632376584074144, | |
| "loss": 2.7913, | |
| "theoretical_loss": 3.735494116655784, | |
| "tokens_seen": 786432000 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0003626465859655759, | |
| "loss": 2.7684, | |
| "theoretical_loss": 3.733950532322087, | |
| "tokens_seen": 789708800 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0003620555135237375, | |
| "loss": 2.806, | |
| "theoretical_loss": 3.7324151245841044, | |
| "tokens_seen": 792985600 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.000361464441081899, | |
| "loss": 2.7815, | |
| "theoretical_loss": 3.730887816609077, | |
| "tokens_seen": 796262400 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.00036087336864006053, | |
| "loss": 2.7677, | |
| "theoretical_loss": 3.729368532598609, | |
| "tokens_seen": 799539200 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.00036028229619822205, | |
| "loss": 2.7659, | |
| "theoretical_loss": 3.7278571977705734, | |
| "tokens_seen": 802816000 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0003596912237563836, | |
| "loss": 2.8014, | |
| "theoretical_loss": 3.7263537383414023, | |
| "tokens_seen": 806092800 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0003591001513145451, | |
| "loss": 2.7815, | |
| "theoretical_loss": 3.7248580815087626, | |
| "tokens_seen": 809369600 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.00035850907887270667, | |
| "loss": 2.804, | |
| "theoretical_loss": 3.7233701554345924, | |
| "tokens_seen": 812646400 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0003579180064308682, | |
| "loss": 2.7522, | |
| "theoretical_loss": 3.721889889228506, | |
| "tokens_seen": 815923200 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "objective/train/docs_used": 458204, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 3.0099945068359375, | |
| "objective/train/theoretical_loss": 3.720417212931543, | |
| "objective/train/tokens_used": 839660000, | |
| "theoretical_loss": 3.720417212931543, | |
| "tokens_seen": 819200000 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0003573269339890297, | |
| "loss": 2.7797, | |
| "theoretical_loss": 3.720417212931543, | |
| "tokens_seen": 819200000 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.00035673586154719123, | |
| "loss": 2.7814, | |
| "theoretical_loss": 3.7189520575002666, | |
| "tokens_seen": 822476800 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.00035614478910535275, | |
| "loss": 2.7295, | |
| "theoretical_loss": 3.717494354791188, | |
| "tokens_seen": 825753600 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0003555537166635143, | |
| "loss": 2.7757, | |
| "theoretical_loss": 3.716044037545523, | |
| "tokens_seen": 829030400 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0003549626442216758, | |
| "loss": 2.7389, | |
| "theoretical_loss": 3.714601039374263, | |
| "tokens_seen": 832307200 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.00035437157177983737, | |
| "loss": 2.7296, | |
| "theoretical_loss": 3.7131652947435536, | |
| "tokens_seen": 835584000 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0003537804993379989, | |
| "loss": 2.7188, | |
| "theoretical_loss": 3.7117367389603793, | |
| "tokens_seen": 838860800 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0003531894268961604, | |
| "loss": 2.6896, | |
| "theoretical_loss": 3.710315308158541, | |
| "tokens_seen": 842137600 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.00035259835445432193, | |
| "loss": 2.7325, | |
| "theoretical_loss": 3.7089009392849173, | |
| "tokens_seen": 845414400 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.00035200728201248345, | |
| "loss": 2.6881, | |
| "theoretical_loss": 3.7074935700860143, | |
| "tokens_seen": 848691200 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.000351416209570645, | |
| "loss": 2.7437, | |
| "theoretical_loss": 3.706093139094781, | |
| "tokens_seen": 851968000 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00035082513712880655, | |
| "loss": 2.7186, | |
| "theoretical_loss": 3.7046995856176954, | |
| "tokens_seen": 855244800 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00035023406468696807, | |
| "loss": 2.7244, | |
| "theoretical_loss": 3.703312849722111, | |
| "tokens_seen": 858521600 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.0003496429922451296, | |
| "loss": 2.728, | |
| "theoretical_loss": 3.701932872223858, | |
| "tokens_seen": 861798400 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.0003490519198032911, | |
| "loss": 2.6658, | |
| "theoretical_loss": 3.7005595946750924, | |
| "tokens_seen": 865075200 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00034846084736145263, | |
| "loss": 2.6899, | |
| "theoretical_loss": 3.699192959352386, | |
| "tokens_seen": 868352000 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00034786977491961415, | |
| "loss": 2.6979, | |
| "theoretical_loss": 3.6978329092450557, | |
| "tokens_seen": 871628800 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00034727870247777573, | |
| "loss": 2.7318, | |
| "theoretical_loss": 3.6964793880437226, | |
| "tokens_seen": 874905600 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00034668763003593725, | |
| "loss": 2.7345, | |
| "theoretical_loss": 3.6951323401290974, | |
| "tokens_seen": 878182400 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00034609655759409877, | |
| "loss": 2.6997, | |
| "theoretical_loss": 3.6937917105609834, | |
| "tokens_seen": 881459200 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0003455054851522603, | |
| "loss": 2.742, | |
| "theoretical_loss": 3.692457445067501, | |
| "tokens_seen": 884736000 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0003449144127104218, | |
| "loss": 2.7555, | |
| "theoretical_loss": 3.6911294900345166, | |
| "tokens_seen": 888012800 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.00034432334026858333, | |
| "loss": 2.6965, | |
| "theoretical_loss": 3.6898077924952775, | |
| "tokens_seen": 891289600 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.00034373226782674485, | |
| "loss": 2.6908, | |
| "theoretical_loss": 3.6884923001202505, | |
| "tokens_seen": 894566400 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.00034314119538490643, | |
| "loss": 2.7095, | |
| "theoretical_loss": 3.6871829612071583, | |
| "tokens_seen": 897843200 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0003425501229430679, | |
| "loss": 2.7095, | |
| "theoretical_loss": 3.6858797246711976, | |
| "tokens_seen": 901120000 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0003419590505012294, | |
| "loss": 2.6875, | |
| "theoretical_loss": 3.684582540035456, | |
| "tokens_seen": 904396800 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.00034136797805939094, | |
| "loss": 2.6855, | |
| "theoretical_loss": 3.683291357421508, | |
| "tokens_seen": 907673600 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00034077690561755246, | |
| "loss": 2.7326, | |
| "theoretical_loss": 3.682006127540184, | |
| "tokens_seen": 910950400 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.000340185833175714, | |
| "loss": 2.7151, | |
| "theoretical_loss": 3.680726801682522, | |
| "tokens_seen": 914227200 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00033959476073387555, | |
| "loss": 2.7191, | |
| "theoretical_loss": 3.679453331710889, | |
| "tokens_seen": 917504000 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0003390036882920371, | |
| "loss": 2.7294, | |
| "theoretical_loss": 3.6781856700502646, | |
| "tokens_seen": 920780800 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0003384126158501986, | |
| "loss": 2.7229, | |
| "theoretical_loss": 3.6769237696796933, | |
| "tokens_seen": 924057600 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0003378215434083601, | |
| "loss": 2.7169, | |
| "theoretical_loss": 3.6756675841238913, | |
| "tokens_seen": 927334400 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00033723047096652164, | |
| "loss": 2.7367, | |
| "theoretical_loss": 3.6744170674450176, | |
| "tokens_seen": 930611200 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00033663939852468316, | |
| "loss": 2.7266, | |
| "theoretical_loss": 3.673172174234587, | |
| "tokens_seen": 933888000 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00033604832608284473, | |
| "loss": 2.7491, | |
| "theoretical_loss": 3.6719328596055423, | |
| "tokens_seen": 937164800 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00033545725364100625, | |
| "loss": 2.6973, | |
| "theoretical_loss": 3.670699079184467, | |
| "tokens_seen": 940441600 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0003348661811991678, | |
| "loss": 2.7381, | |
| "theoretical_loss": 3.669470789103942, | |
| "tokens_seen": 943718400 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0003342751087573293, | |
| "loss": 2.7048, | |
| "theoretical_loss": 3.6682479459950446, | |
| "tokens_seen": 946995200 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0003336840363154908, | |
| "loss": 2.6816, | |
| "theoretical_loss": 3.6670305069799785, | |
| "tokens_seen": 950272000 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00033309296387365234, | |
| "loss": 2.6775, | |
| "theoretical_loss": 3.6658184296648457, | |
| "tokens_seen": 953548800 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00033250189143181386, | |
| "loss": 2.6529, | |
| "theoretical_loss": 3.6646116721325415, | |
| "tokens_seen": 956825600 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00033191081898997543, | |
| "loss": 2.631, | |
| "theoretical_loss": 3.6634101929357836, | |
| "tokens_seen": 960102400 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00033131974654813695, | |
| "loss": 2.6154, | |
| "theoretical_loss": 3.6622139510902625, | |
| "tokens_seen": 963379200 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003307404955551352, | |
| "loss": 2.6605, | |
| "theoretical_loss": 3.6610229060679167, | |
| "tokens_seen": 966656000 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00033014942311329675, | |
| "loss": 2.7005, | |
| "theoretical_loss": 3.659837017790328, | |
| "tokens_seen": 969932800 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00032955835067145827, | |
| "loss": 2.7149, | |
| "theoretical_loss": 3.658656246622233, | |
| "tokens_seen": 973209600 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003289672782296198, | |
| "loss": 2.7079, | |
| "theoretical_loss": 3.6574805533651515, | |
| "tokens_seen": 976486400 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00032837620578778136, | |
| "loss": 2.6308, | |
| "theoretical_loss": 3.6563098992511267, | |
| "tokens_seen": 979763200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.36988534170376464, | |
| "debugging/distinct-1-grams": 0.7978170447064457, | |
| "debugging/distinct-2-grams": 0.9699762510402439, | |
| "debugging/entropy-1-grams": 5.166066569251399, | |
| "debugging/entropy-2-grams": 5.640626124943832, | |
| "debugging/length": 444.0, | |
| "debugging/num_segments": 5, | |
| "epoch": 0.35, | |
| "objective/train/docs_used": 548757, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.5548319816589355, | |
| "objective/train/theoretical_loss": 3.655144245936574, | |
| "objective/train/tokens_used": 1003500000, | |
| "theoretical_loss": 3.655144245936574, | |
| "tokens_seen": 983040000 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003277851333459429, | |
| "loss": 2.695, | |
| "theoretical_loss": 3.655144245936574, | |
| "tokens_seen": 983040000 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003271940609041044, | |
| "loss": 2.6569, | |
| "theoretical_loss": 3.653983555496242, | |
| "tokens_seen": 986316800 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003266029884622659, | |
| "loss": 2.6833, | |
| "theoretical_loss": 3.6528277904172755, | |
| "tokens_seen": 989593600 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00032601191602042745, | |
| "loss": 2.6997, | |
| "theoretical_loss": 3.6516769135933815, | |
| "tokens_seen": 992870400 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.00032542084357858897, | |
| "loss": 2.7903, | |
| "theoretical_loss": 3.650530888319103, | |
| "tokens_seen": 996147200 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.00032482977113675054, | |
| "loss": 2.7817, | |
| "theoretical_loss": 3.649389678284182, | |
| "tokens_seen": 999424000 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.00032423869869491206, | |
| "loss": 2.8282, | |
| "theoretical_loss": 3.6482532475680287, | |
| "tokens_seen": 1002700800 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0003236476262530736, | |
| "loss": 2.7855, | |
| "theoretical_loss": 3.6471215606342833, | |
| "tokens_seen": 1005977600 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0003230565538112351, | |
| "loss": 2.7825, | |
| "theoretical_loss": 3.645994582325468, | |
| "tokens_seen": 1009254400 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0003224654813693966, | |
| "loss": 2.7798, | |
| "theoretical_loss": 3.6448722778577327, | |
| "tokens_seen": 1012531200 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.00032187440892755814, | |
| "loss": 2.8039, | |
| "theoretical_loss": 3.6437546128156946, | |
| "tokens_seen": 1015808000 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0003212833364857197, | |
| "loss": 2.7713, | |
| "theoretical_loss": 3.6426415531473566, | |
| "tokens_seen": 1019084800 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00032069226404388124, | |
| "loss": 2.7523, | |
| "theoretical_loss": 3.641533065159118, | |
| "tokens_seen": 1022361600 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00032010119160204276, | |
| "loss": 2.7315, | |
| "theoretical_loss": 3.6404291155108712, | |
| "tokens_seen": 1025638400 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0003195101191602043, | |
| "loss": 2.7432, | |
| "theoretical_loss": 3.639329671211173, | |
| "tokens_seen": 1028915200 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0003189190467183658, | |
| "loss": 2.7447, | |
| "theoretical_loss": 3.6382346996125055, | |
| "tokens_seen": 1032192000 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0003183279742765273, | |
| "loss": 2.7196, | |
| "theoretical_loss": 3.6371441684066097, | |
| "tokens_seen": 1035468800 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00031773690183468884, | |
| "loss": 2.7461, | |
| "theoretical_loss": 3.6360580456199036, | |
| "tokens_seen": 1038745600 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0003171458293928504, | |
| "loss": 2.7746, | |
| "theoretical_loss": 3.6349762996089683, | |
| "tokens_seen": 1042022400 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00031655475695101194, | |
| "loss": 2.7022, | |
| "theoretical_loss": 3.633898899056115, | |
| "tokens_seen": 1045299200 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00031596368450917346, | |
| "loss": 2.7864, | |
| "theoretical_loss": 3.6328258129650246, | |
| "tokens_seen": 1048576000 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.000315372612067335, | |
| "loss": 2.7388, | |
| "theoretical_loss": 3.6317570106564565, | |
| "tokens_seen": 1051852800 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0003147815396254965, | |
| "loss": 2.7535, | |
| "theoretical_loss": 3.6306924617640295, | |
| "tokens_seen": 1055129600 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.000314190467183658, | |
| "loss": 2.7354, | |
| "theoretical_loss": 3.6296321362300716, | |
| "tokens_seen": 1058406400 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0003135993947418196, | |
| "loss": 2.7542, | |
| "theoretical_loss": 3.6285760043015385, | |
| "tokens_seen": 1061683200 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0003130083222999811, | |
| "loss": 2.754, | |
| "theoretical_loss": 3.6275240365259958, | |
| "tokens_seen": 1064960000 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.00031241724985814264, | |
| "loss": 2.6926, | |
| "theoretical_loss": 3.6264762037476683, | |
| "tokens_seen": 1068236800 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.00031182617741630416, | |
| "loss": 2.6943, | |
| "theoretical_loss": 3.625432477103554, | |
| "tokens_seen": 1071513600 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0003112351049744657, | |
| "loss": 2.7087, | |
| "theoretical_loss": 3.6243928280195976, | |
| "tokens_seen": 1074790400 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003106440325326272, | |
| "loss": 2.6966, | |
| "theoretical_loss": 3.62335722820693, | |
| "tokens_seen": 1078067200 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003100529600907888, | |
| "loss": 2.7153, | |
| "theoretical_loss": 3.6223256496581637, | |
| "tokens_seen": 1081344000 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003094618876489503, | |
| "loss": 2.7069, | |
| "theoretical_loss": 3.6212980646437485, | |
| "tokens_seen": 1084620800 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003088708152071118, | |
| "loss": 2.6799, | |
| "theoretical_loss": 3.6202744457083877, | |
| "tokens_seen": 1087897600 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00030827974276527334, | |
| "loss": 2.7278, | |
| "theoretical_loss": 3.6192547656675083, | |
| "tokens_seen": 1091174400 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00030768867032343486, | |
| "loss": 2.7174, | |
| "theoretical_loss": 3.618238997603788, | |
| "tokens_seen": 1094451200 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003070975978815964, | |
| "loss": 2.693, | |
| "theoretical_loss": 3.617227114863738, | |
| "tokens_seen": 1097728000 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003065065254397579, | |
| "loss": 2.686, | |
| "theoretical_loss": 3.6162190910543366, | |
| "tokens_seen": 1101004800 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003059154529979195, | |
| "loss": 2.6809, | |
| "theoretical_loss": 3.615214900039721, | |
| "tokens_seen": 1104281600 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.000305324380556081, | |
| "loss": 2.6561, | |
| "theoretical_loss": 3.614214515937924, | |
| "tokens_seen": 1107558400 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00030473330811424246, | |
| "loss": 2.6753, | |
| "theoretical_loss": 3.613217913117667, | |
| "tokens_seen": 1110835200 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.000304142235672404, | |
| "loss": 2.6834, | |
| "theoretical_loss": 3.612225066195201, | |
| "tokens_seen": 1114112000 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0003035511632305655, | |
| "loss": 2.716, | |
| "theoretical_loss": 3.611235950031194, | |
| "tokens_seen": 1117388800 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.000302960090788727, | |
| "loss": 2.6889, | |
| "theoretical_loss": 3.6102505397276743, | |
| "tokens_seen": 1120665600 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0003023690183468886, | |
| "loss": 2.6946, | |
| "theoretical_loss": 3.60926881062501, | |
| "tokens_seen": 1123942400 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0003017779459050501, | |
| "loss": 2.653, | |
| "theoretical_loss": 3.608290738298942, | |
| "tokens_seen": 1127219200 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00030118687346321164, | |
| "loss": 2.6779, | |
| "theoretical_loss": 3.6073162985576643, | |
| "tokens_seen": 1130496000 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00030059580102137316, | |
| "loss": 2.659, | |
| "theoretical_loss": 3.606345467438941, | |
| "tokens_seen": 1133772800 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0003000047285795347, | |
| "loss": 2.6885, | |
| "theoretical_loss": 3.6053782212072747, | |
| "tokens_seen": 1137049600 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0002994136561376962, | |
| "loss": 2.6944, | |
| "theoretical_loss": 3.604414536351113, | |
| "tokens_seen": 1140326400 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0002988225836958578, | |
| "loss": 2.678, | |
| "theoretical_loss": 3.6034543895801017, | |
| "tokens_seen": 1143603200 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "objective/train/docs_used": 634175, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.4743967056274414, | |
| "objective/train/theoretical_loss": 3.6024977578223742, | |
| "objective/train/tokens_used": 1167340000, | |
| "theoretical_loss": 3.6024977578223742, | |
| "tokens_seen": 1146880000 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0002982315112540193, | |
| "loss": 2.6462, | |
| "theoretical_loss": 3.6024977578223742, | |
| "tokens_seen": 1146880000 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0002976404388121808, | |
| "loss": 2.7348, | |
| "theoretical_loss": 3.6015446182218875, | |
| "tokens_seen": 1150156800 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00029704936637034234, | |
| "loss": 2.7499, | |
| "theoretical_loss": 3.600594948135793, | |
| "tokens_seen": 1153433600 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00029645829392850386, | |
| "loss": 2.7004, | |
| "theoretical_loss": 3.59964872513185, | |
| "tokens_seen": 1156710400 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0002958672214866654, | |
| "loss": 2.7267, | |
| "theoretical_loss": 3.5987059269858763, | |
| "tokens_seen": 1159987200 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0002952761490448269, | |
| "loss": 2.7081, | |
| "theoretical_loss": 3.5977665316792375, | |
| "tokens_seen": 1163264000 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0002946850766029885, | |
| "loss": 2.7148, | |
| "theoretical_loss": 3.5968305173963744, | |
| "tokens_seen": 1166540800 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00029409400416115, | |
| "loss": 2.6749, | |
| "theoretical_loss": 3.5958978625223628, | |
| "tokens_seen": 1169817600 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0002935029317193115, | |
| "loss": 2.6642, | |
| "theoretical_loss": 3.5949685456405165, | |
| "tokens_seen": 1173094400 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00029291185927747304, | |
| "loss": 2.6922, | |
| "theoretical_loss": 3.5940425455300176, | |
| "tokens_seen": 1176371200 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00029232078683563456, | |
| "loss": 2.6919, | |
| "theoretical_loss": 3.593119841163589, | |
| "tokens_seen": 1179648000 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0002917297143937961, | |
| "loss": 2.6804, | |
| "theoretical_loss": 3.5922004117051944, | |
| "tokens_seen": 1182924800 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00029113864195195766, | |
| "loss": 2.6932, | |
| "theoretical_loss": 3.5912842365077777, | |
| "tokens_seen": 1186201600 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0002905475695101192, | |
| "loss": 2.7037, | |
| "theoretical_loss": 3.5903712951110305, | |
| "tokens_seen": 1189478400 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0002899564970682807, | |
| "loss": 2.676, | |
| "theoretical_loss": 3.5894615672391947, | |
| "tokens_seen": 1192755200 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0002893654246264422, | |
| "loss": 2.7106, | |
| "theoretical_loss": 3.5885550327988973, | |
| "tokens_seen": 1196032000 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.00028877435218460374, | |
| "loss": 2.6759, | |
| "theoretical_loss": 3.587651671877014, | |
| "tokens_seen": 1199308800 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.00028818327974276526, | |
| "loss": 2.7006, | |
| "theoretical_loss": 3.5867514647385663, | |
| "tokens_seen": 1202585600 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.00028759220730092684, | |
| "loss": 2.7011, | |
| "theoretical_loss": 3.585854391824647, | |
| "tokens_seen": 1205862400 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.00028700113485908836, | |
| "loss": 2.6564, | |
| "theoretical_loss": 3.584960433750375, | |
| "tokens_seen": 1209139200 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0002864100624172499, | |
| "loss": 2.6935, | |
| "theoretical_loss": 3.5840695713028827, | |
| "tokens_seen": 1212416000 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0002858189899754114, | |
| "loss": 2.6789, | |
| "theoretical_loss": 3.5831817854393266, | |
| "tokens_seen": 1215692800 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002852279175335729, | |
| "loss": 2.7045, | |
| "theoretical_loss": 3.582297057284933, | |
| "tokens_seen": 1218969600 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00028463684509173444, | |
| "loss": 2.7214, | |
| "theoretical_loss": 3.5814153681310623, | |
| "tokens_seen": 1222246400 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00028404577264989596, | |
| "loss": 2.7105, | |
| "theoretical_loss": 3.5805366994333125, | |
| "tokens_seen": 1225523200 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00028345470020805754, | |
| "loss": 2.6844, | |
| "theoretical_loss": 3.5796610328096365, | |
| "tokens_seen": 1228800000 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00028286362776621906, | |
| "loss": 2.7018, | |
| "theoretical_loss": 3.578788350038497, | |
| "tokens_seen": 1232076800 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002822725553243806, | |
| "loss": 2.6849, | |
| "theoretical_loss": 3.5779186330570405, | |
| "tokens_seen": 1235353600 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002816814828825421, | |
| "loss": 2.6658, | |
| "theoretical_loss": 3.5770518639592983, | |
| "tokens_seen": 1238630400 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002810904104407036, | |
| "loss": 2.7339, | |
| "theoretical_loss": 3.5761880249944147, | |
| "tokens_seen": 1241907200 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00028049933799886514, | |
| "loss": 2.7244, | |
| "theoretical_loss": 3.5753270985648973, | |
| "tokens_seen": 1245184000 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.0002799082655570267, | |
| "loss": 2.7507, | |
| "theoretical_loss": 3.574469067224892, | |
| "tokens_seen": 1248460800 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00027931719311518824, | |
| "loss": 2.744, | |
| "theoretical_loss": 3.573613913678484, | |
| "tokens_seen": 1251737600 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00027872612067334976, | |
| "loss": 2.7416, | |
| "theoretical_loss": 3.57276162077802, | |
| "tokens_seen": 1255014400 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.0002781350482315113, | |
| "loss": 2.7218, | |
| "theoretical_loss": 3.5719121715224524, | |
| "tokens_seen": 1258291200 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.0002775439757896728, | |
| "loss": 2.6696, | |
| "theoretical_loss": 3.571065549055712, | |
| "tokens_seen": 1261568000 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.0002769529033478343, | |
| "loss": 2.7177, | |
| "theoretical_loss": 3.5702217366650935, | |
| "tokens_seen": 1264844800 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.0002763618309059959, | |
| "loss": 2.6499, | |
| "theoretical_loss": 3.5693807177796737, | |
| "tokens_seen": 1268121600 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00027577075846415736, | |
| "loss": 2.6388, | |
| "theoretical_loss": 3.5685424759687434, | |
| "tokens_seen": 1271398400 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0002751796860223189, | |
| "loss": 2.6672, | |
| "theoretical_loss": 3.567706994940263, | |
| "tokens_seen": 1274675200 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0002745886135804804, | |
| "loss": 2.6905, | |
| "theoretical_loss": 3.5668742585393405, | |
| "tokens_seen": 1277952000 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0002739975411386419, | |
| "loss": 2.6476, | |
| "theoretical_loss": 3.566044250746728, | |
| "tokens_seen": 1281228800 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00027340646869680345, | |
| "loss": 2.6753, | |
| "theoretical_loss": 3.5652169556773403, | |
| "tokens_seen": 1284505600 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00027281539625496497, | |
| "loss": 2.6262, | |
| "theoretical_loss": 3.5643923575787912, | |
| "tokens_seen": 1287782400 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00027222432381312654, | |
| "loss": 2.642, | |
| "theoretical_loss": 3.563570440829951, | |
| "tokens_seen": 1291059200 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00027163325137128806, | |
| "loss": 2.6258, | |
| "theoretical_loss": 3.562751189939524, | |
| "tokens_seen": 1294336000 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0002710421789294496, | |
| "loss": 2.666, | |
| "theoretical_loss": 3.5619345895446424, | |
| "tokens_seen": 1297612800 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0002704511064876111, | |
| "loss": 2.6471, | |
| "theoretical_loss": 3.561120624409482, | |
| "tokens_seen": 1300889600 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0002698600340457726, | |
| "loss": 2.6465, | |
| "theoretical_loss": 3.560309279423894, | |
| "tokens_seen": 1304166400 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00026926896160393415, | |
| "loss": 2.6289, | |
| "theoretical_loss": 3.5595005396020554, | |
| "tokens_seen": 1307443200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.528396129459468, | |
| "debugging/distinct-1-grams": 0.7342587134430482, | |
| "debugging/distinct-2-grams": 0.9617049578129524, | |
| "debugging/entropy-1-grams": 5.512289049191654, | |
| "debugging/entropy-2-grams": 6.384961594108999, | |
| "debugging/length": 612.8571428571429, | |
| "debugging/num_segments": 7, | |
| "epoch": 0.47, | |
| "objective/train/docs_used": 722031, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.723985195159912, | |
| "objective/train/theoretical_loss": 3.558694390081137, | |
| "objective/train/tokens_used": 1331180000, | |
| "theoretical_loss": 3.558694390081137, | |
| "tokens_seen": 1310720000 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0002686778891620957, | |
| "loss": 2.6182, | |
| "theoretical_loss": 3.558694390081137, | |
| "tokens_seen": 1310720000 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00026808681672025724, | |
| "loss": 2.6074, | |
| "theoretical_loss": 3.5578908161199934, | |
| "tokens_seen": 1313996800 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00026749574427841876, | |
| "loss": 2.6128, | |
| "theoretical_loss": 3.5570898030978584, | |
| "tokens_seen": 1317273600 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0002669046718365803, | |
| "loss": 2.6063, | |
| "theoretical_loss": 3.556291336513074, | |
| "tokens_seen": 1320550400 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0002663135993947418, | |
| "loss": 2.6225, | |
| "theoretical_loss": 3.5554954019818235, | |
| "tokens_seen": 1323827200 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0002657225269529033, | |
| "loss": 2.6084, | |
| "theoretical_loss": 3.554701985236883, | |
| "tokens_seen": 1327104000 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.0002651314545110649, | |
| "loss": 2.6295, | |
| "theoretical_loss": 3.553911072126394, | |
| "tokens_seen": 1330380800 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.0002645403820692264, | |
| "loss": 2.6257, | |
| "theoretical_loss": 3.5531226486126504, | |
| "tokens_seen": 1333657600 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026394930962738794, | |
| "loss": 2.6525, | |
| "theoretical_loss": 3.552336700770896, | |
| "tokens_seen": 1336934400 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026335823718554946, | |
| "loss": 2.6374, | |
| "theoretical_loss": 3.5515532147881443, | |
| "tokens_seen": 1340211200 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.000262767164743711, | |
| "loss": 2.6727, | |
| "theoretical_loss": 3.5507721769620098, | |
| "tokens_seen": 1343488000 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.0002621760923018725, | |
| "loss": 2.662, | |
| "theoretical_loss": 3.549993573699556, | |
| "tokens_seen": 1346764800 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.000261585019860034, | |
| "loss": 2.6595, | |
| "theoretical_loss": 3.5492173915161565, | |
| "tokens_seen": 1350041600 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.0002609939474181956, | |
| "loss": 2.6198, | |
| "theoretical_loss": 3.548443617034371, | |
| "tokens_seen": 1353318400 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.0002604028749763571, | |
| "loss": 2.6581, | |
| "theoretical_loss": 3.547672236982839, | |
| "tokens_seen": 1356595200 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00025981180253451864, | |
| "loss": 2.6674, | |
| "theoretical_loss": 3.5469032381951804, | |
| "tokens_seen": 1359872000 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00025922073009268016, | |
| "loss": 2.6643, | |
| "theoretical_loss": 3.5461366076089202, | |
| "tokens_seen": 1363148800 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.0002586296576508417, | |
| "loss": 2.6494, | |
| "theoretical_loss": 3.5453723322644146, | |
| "tokens_seen": 1366425600 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.0002580385852090032, | |
| "loss": 2.6457, | |
| "theoretical_loss": 3.544610399303803, | |
| "tokens_seen": 1369702400 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.0002574475127671648, | |
| "loss": 2.6389, | |
| "theoretical_loss": 3.5438507959699637, | |
| "tokens_seen": 1372979200 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.0002568564403253263, | |
| "loss": 2.6765, | |
| "theoretical_loss": 3.5430935096054883, | |
| "tokens_seen": 1376256000 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.0002562653678834878, | |
| "loss": 2.6015, | |
| "theoretical_loss": 3.5423385276516663, | |
| "tokens_seen": 1379532800 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00025567429544164934, | |
| "loss": 2.6291, | |
| "theoretical_loss": 3.5415858376474825, | |
| "tokens_seen": 1382809600 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.00025508322299981086, | |
| "loss": 2.5961, | |
| "theoretical_loss": 3.5408354272286298, | |
| "tokens_seen": 1386086400 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.0002544921505579724, | |
| "loss": 2.6271, | |
| "theoretical_loss": 3.540087284126531, | |
| "tokens_seen": 1389363200 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.00025390107811613396, | |
| "loss": 2.6286, | |
| "theoretical_loss": 3.539341396167372, | |
| "tokens_seen": 1392640000 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.0002533100056742955, | |
| "loss": 2.6664, | |
| "theoretical_loss": 3.538597751271153, | |
| "tokens_seen": 1395916800 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.000252718933232457, | |
| "loss": 2.6892, | |
| "theoretical_loss": 3.5378563374507443, | |
| "tokens_seen": 1399193600 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.0002521278607906185, | |
| "loss": 2.6479, | |
| "theoretical_loss": 3.5371171428109576, | |
| "tokens_seen": 1402470400 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.00025153678834878004, | |
| "loss": 2.6632, | |
| "theoretical_loss": 3.5363801555476275, | |
| "tokens_seen": 1405747200 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.00025094571590694156, | |
| "loss": 2.6645, | |
| "theoretical_loss": 3.5356453639467063, | |
| "tokens_seen": 1409024000 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.00025035464346510314, | |
| "loss": 2.6674, | |
| "theoretical_loss": 3.534912756383366, | |
| "tokens_seen": 1412300800 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.0002497635710232646, | |
| "loss": 2.6497, | |
| "theoretical_loss": 3.534182321321116, | |
| "tokens_seen": 1415577600 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.0002491724985814262, | |
| "loss": 2.6566, | |
| "theoretical_loss": 3.5334540473109275, | |
| "tokens_seen": 1418854400 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.0002485814261395877, | |
| "loss": 2.6031, | |
| "theoretical_loss": 3.532727922990372, | |
| "tokens_seen": 1422131200 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.0002479903536977492, | |
| "loss": 2.6189, | |
| "theoretical_loss": 3.532003937082766, | |
| "tokens_seen": 1425408000 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.00024739928125591074, | |
| "loss": 2.6496, | |
| "theoretical_loss": 3.5312820783963303, | |
| "tokens_seen": 1428684800 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.00024680820881407226, | |
| "loss": 2.6855, | |
| "theoretical_loss": 3.5305623358233578, | |
| "tokens_seen": 1431961600 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.0002462171363722338, | |
| "loss": 2.6596, | |
| "theoretical_loss": 3.529844698339391, | |
| "tokens_seen": 1435238400 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.0002456260639303953, | |
| "loss": 2.6482, | |
| "theoretical_loss": 3.5291291550024093, | |
| "tokens_seen": 1438515200 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.0002450349914885568, | |
| "loss": 2.6302, | |
| "theoretical_loss": 3.5284156949520264, | |
| "tokens_seen": 1441792000 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 0.00024444391904671834, | |
| "loss": 2.6396, | |
| "theoretical_loss": 3.5277043074086993, | |
| "tokens_seen": 1445068800 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 0.0002438528466048799, | |
| "loss": 2.5942, | |
| "theoretical_loss": 3.5269949816729422, | |
| "tokens_seen": 1448345600 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 0.00024326177416304144, | |
| "loss": 2.6265, | |
| "theoretical_loss": 3.5262877071245526, | |
| "tokens_seen": 1451622400 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 0.00024267070172120296, | |
| "loss": 2.6406, | |
| "theoretical_loss": 3.525582473221849, | |
| "tokens_seen": 1454899200 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 0.00024207962927936448, | |
| "loss": 2.5939, | |
| "theoretical_loss": 3.5248792695009126, | |
| "tokens_seen": 1458176000 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 0.00024150037828636278, | |
| "loss": 2.6213, | |
| "theoretical_loss": 3.5241780855748415, | |
| "tokens_seen": 1461452800 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 0.0002409093058445243, | |
| "loss": 2.6209, | |
| "theoretical_loss": 3.523478911133012, | |
| "tokens_seen": 1464729600 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 0.00024031823340268582, | |
| "loss": 2.6134, | |
| "theoretical_loss": 3.5227817359403506, | |
| "tokens_seen": 1468006400 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.00023972716096084737, | |
| "loss": 2.6302, | |
| "theoretical_loss": 3.5220865498366107, | |
| "tokens_seen": 1471283200 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "objective/train/docs_used": 814232, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 1.971326470375061, | |
| "objective/train/theoretical_loss": 3.5213933427356636, | |
| "objective/train/tokens_used": 1495020000, | |
| "theoretical_loss": 3.5213933427356636, | |
| "tokens_seen": 1474560000 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.0002391360885190089, | |
| "loss": 2.5587, | |
| "theoretical_loss": 3.5213933427356636, | |
| "tokens_seen": 1474560000 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.0002385450160771704, | |
| "loss": 2.5759, | |
| "theoretical_loss": 3.5207021046247924, | |
| "tokens_seen": 1477836800 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.00023795394363533196, | |
| "loss": 2.5641, | |
| "theoretical_loss": 3.520012825563997, | |
| "tokens_seen": 1481113600 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.00023736287119349348, | |
| "loss": 2.5564, | |
| "theoretical_loss": 3.519325495685306, | |
| "tokens_seen": 1484390400 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.000236771798751655, | |
| "loss": 2.5884, | |
| "theoretical_loss": 3.5186401051920977, | |
| "tokens_seen": 1487667200 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.00023618072630981655, | |
| "loss": 2.5732, | |
| "theoretical_loss": 3.517956644358428, | |
| "tokens_seen": 1490944000 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.00023558965386797807, | |
| "loss": 2.5841, | |
| "theoretical_loss": 3.517275103528368, | |
| "tokens_seen": 1494220800 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.00023502222432381312, | |
| "loss": 2.5713, | |
| "theoretical_loss": 3.516595473115344, | |
| "tokens_seen": 1497497600 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 0.00023443115188197467, | |
| "loss": 2.6242, | |
| "theoretical_loss": 3.5159177436014932, | |
| "tokens_seen": 1500774400 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 0.0002338400794401362, | |
| "loss": 2.6055, | |
| "theoretical_loss": 3.5152419055370205, | |
| "tokens_seen": 1504051200 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 0.0002332490069982977, | |
| "loss": 2.5976, | |
| "theoretical_loss": 3.5145679495395643, | |
| "tokens_seen": 1507328000 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 0.00023265793455645926, | |
| "loss": 2.6197, | |
| "theoretical_loss": 3.513895866293571, | |
| "tokens_seen": 1510604800 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 0.00023206686211462078, | |
| "loss": 2.6415, | |
| "theoretical_loss": 3.5132256465496736, | |
| "tokens_seen": 1513881600 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 0.0002314757896727823, | |
| "loss": 2.5839, | |
| "theoretical_loss": 3.5125572811240833, | |
| "tokens_seen": 1517158400 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 0.00023088471723094385, | |
| "loss": 2.6045, | |
| "theoretical_loss": 3.5118907608979786, | |
| "tokens_seen": 1520435200 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 0.00023029364478910537, | |
| "loss": 2.6161, | |
| "theoretical_loss": 3.5112260768169135, | |
| "tokens_seen": 1523712000 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.0002297025723472669, | |
| "loss": 2.6499, | |
| "theoretical_loss": 3.510563219890219, | |
| "tokens_seen": 1526988800 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.00022911149990542844, | |
| "loss": 2.648, | |
| "theoretical_loss": 3.509902181190423, | |
| "tokens_seen": 1530265600 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.00022852042746358996, | |
| "loss": 2.6739, | |
| "theoretical_loss": 3.50924295185267, | |
| "tokens_seen": 1533542400 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.00022792935502175148, | |
| "loss": 2.6783, | |
| "theoretical_loss": 3.508585523074149, | |
| "tokens_seen": 1536819200 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.000227338282579913, | |
| "loss": 2.636, | |
| "theoretical_loss": 3.5079298861135295, | |
| "tokens_seen": 1540096000 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.00022674721013807455, | |
| "loss": 2.5976, | |
| "theoretical_loss": 3.5072760322904006, | |
| "tokens_seen": 1543372800 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.00022615613769623604, | |
| "loss": 2.6265, | |
| "theoretical_loss": 3.50662395298472, | |
| "tokens_seen": 1546649600 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.00022556506525439756, | |
| "loss": 2.661, | |
| "theoretical_loss": 3.505973639636264, | |
| "tokens_seen": 1549926400 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.0002249739928125591, | |
| "loss": 2.6665, | |
| "theoretical_loss": 3.5053250837440926, | |
| "tokens_seen": 1553203200 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 0.00022438292037072063, | |
| "loss": 2.6314, | |
| "theoretical_loss": 3.5046782768660085, | |
| "tokens_seen": 1556480000 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 0.00022379184792888215, | |
| "loss": 2.619, | |
| "theoretical_loss": 3.5040332106180347, | |
| "tokens_seen": 1559756800 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 0.0002232007754870437, | |
| "loss": 2.6229, | |
| "theoretical_loss": 3.503389876673886, | |
| "tokens_seen": 1563033600 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 0.00022260970304520522, | |
| "loss": 2.6183, | |
| "theoretical_loss": 3.5027482667644554, | |
| "tokens_seen": 1566310400 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 0.00022201863060336674, | |
| "loss": 2.592, | |
| "theoretical_loss": 3.502108372677303, | |
| "tokens_seen": 1569587200 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 0.0002214275581615283, | |
| "loss": 2.6128, | |
| "theoretical_loss": 3.5014701862561486, | |
| "tokens_seen": 1572864000 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 0.0002208364857196898, | |
| "loss": 2.5797, | |
| "theoretical_loss": 3.5008336994003724, | |
| "tokens_seen": 1576140800 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 0.00022024541327785133, | |
| "loss": 2.5656, | |
| "theoretical_loss": 3.500198904064521, | |
| "tokens_seen": 1579417600 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.00021965434083601288, | |
| "loss": 2.6077, | |
| "theoretical_loss": 3.499565792257817, | |
| "tokens_seen": 1582694400 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.0002190632683941744, | |
| "loss": 2.6472, | |
| "theoretical_loss": 3.4989343560436756, | |
| "tokens_seen": 1585971200 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.00021847219595233592, | |
| "loss": 2.6389, | |
| "theoretical_loss": 3.498304587539227, | |
| "tokens_seen": 1589248000 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.00021788112351049747, | |
| "loss": 2.6387, | |
| "theoretical_loss": 3.4976764789148422, | |
| "tokens_seen": 1592524800 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.000217290051068659, | |
| "loss": 2.6073, | |
| "theoretical_loss": 3.4970500223936627, | |
| "tokens_seen": 1595801600 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.0002166989786268205, | |
| "loss": 2.6221, | |
| "theoretical_loss": 3.496425210251142, | |
| "tokens_seen": 1599078400 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.00021610790618498203, | |
| "loss": 2.6116, | |
| "theoretical_loss": 3.4958020348145835, | |
| "tokens_seen": 1602355200 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.00021551683374314358, | |
| "loss": 2.6123, | |
| "theoretical_loss": 3.4951804884626885, | |
| "tokens_seen": 1605632000 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.0002149257613013051, | |
| "loss": 2.5363, | |
| "theoretical_loss": 3.494560563625107, | |
| "tokens_seen": 1608908800 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 0.00021433468885946662, | |
| "loss": 2.6202, | |
| "theoretical_loss": 3.4939422527819968, | |
| "tokens_seen": 1612185600 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 0.00021374361641762817, | |
| "loss": 2.6406, | |
| "theoretical_loss": 3.4933255484635826, | |
| "tokens_seen": 1615462400 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 0.0002131525439757897, | |
| "loss": 2.6459, | |
| "theoretical_loss": 3.4927104432497202, | |
| "tokens_seen": 1618739200 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 0.0002125614715339512, | |
| "loss": 2.6453, | |
| "theoretical_loss": 3.4920969297694713, | |
| "tokens_seen": 1622016000 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 0.00021197039909211273, | |
| "loss": 2.6063, | |
| "theoretical_loss": 3.491485000700675, | |
| "tokens_seen": 1625292800 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 0.00021137932665027425, | |
| "loss": 2.5588, | |
| "theoretical_loss": 3.4908746487695304, | |
| "tokens_seen": 1628569600 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 0.00021078825420843577, | |
| "loss": 2.5973, | |
| "theoretical_loss": 3.4902658667501774, | |
| "tokens_seen": 1631846400 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 0.00021019718176659732, | |
| "loss": 2.568, | |
| "theoretical_loss": 3.489658647464288, | |
| "tokens_seen": 1635123200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.5149630825070948, | |
| "debugging/distinct-1-grams": 0.7834320051091582, | |
| "debugging/distinct-2-grams": 0.9512996152642781, | |
| "debugging/entropy-1-grams": 5.8715522169488885, | |
| "debugging/entropy-2-grams": 6.774781074452074, | |
| "debugging/length": 520.8461538461538, | |
| "debugging/num_segments": 13, | |
| "epoch": 0.59, | |
| "objective/train/docs_used": 904193, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.604003667831421, | |
| "objective/train/theoretical_loss": 3.489052983780656, | |
| "objective/train/tokens_used": 1658860000, | |
| "theoretical_loss": 3.489052983780656, | |
| "tokens_seen": 1638400000 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.00020960610932475884, | |
| "loss": 2.6082, | |
| "theoretical_loss": 3.489052983780656, | |
| "tokens_seen": 1638400000 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.00020901503688292036, | |
| "loss": 2.6249, | |
| "theoretical_loss": 3.488448868614799, | |
| "tokens_seen": 1641676800 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.0002084239644410819, | |
| "loss": 2.5977, | |
| "theoretical_loss": 3.4878462949285556, | |
| "tokens_seen": 1644953600 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.00020783289199924343, | |
| "loss": 2.5804, | |
| "theoretical_loss": 3.4872452557296922, | |
| "tokens_seen": 1648230400 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.00020724181955740495, | |
| "loss": 2.5911, | |
| "theoretical_loss": 3.486645744071512, | |
| "tokens_seen": 1651507200 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.0002066507471155665, | |
| "loss": 2.6117, | |
| "theoretical_loss": 3.486047753052471, | |
| "tokens_seen": 1654784000 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.00020605967467372802, | |
| "loss": 2.5941, | |
| "theoretical_loss": 3.4854512758157945, | |
| "tokens_seen": 1658060800 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.00020546860223188954, | |
| "loss": 2.5892, | |
| "theoretical_loss": 3.484856305549097, | |
| "tokens_seen": 1661337600 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.00020487752979005106, | |
| "loss": 2.6073, | |
| "theoretical_loss": 3.484262835484011, | |
| "tokens_seen": 1664614400 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "learning_rate": 0.0002042864573482126, | |
| "loss": 2.6096, | |
| "theoretical_loss": 3.483670858895815, | |
| "tokens_seen": 1667891200 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "learning_rate": 0.00020369538490637413, | |
| "loss": 2.5759, | |
| "theoretical_loss": 3.4830803691030683, | |
| "tokens_seen": 1671168000 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "learning_rate": 0.00020310431246453565, | |
| "loss": 2.5433, | |
| "theoretical_loss": 3.4824913594672458, | |
| "tokens_seen": 1674444800 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "learning_rate": 0.0002025132400226972, | |
| "loss": 2.5501, | |
| "theoretical_loss": 3.481903823392381, | |
| "tokens_seen": 1677721600 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "learning_rate": 0.00020192216758085872, | |
| "loss": 2.5882, | |
| "theoretical_loss": 3.4813177543247096, | |
| "tokens_seen": 1680998400 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "learning_rate": 0.00020133109513902024, | |
| "loss": 2.5707, | |
| "theoretical_loss": 3.480733145752319, | |
| "tokens_seen": 1684275200 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "learning_rate": 0.00020074002269718179, | |
| "loss": 2.5565, | |
| "theoretical_loss": 3.4801499912048, | |
| "tokens_seen": 1687552000 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "learning_rate": 0.0002001489502553433, | |
| "loss": 2.623, | |
| "theoretical_loss": 3.4795682842529, | |
| "tokens_seen": 1690828800 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.00019955787781350483, | |
| "loss": 2.57, | |
| "theoretical_loss": 3.4789880185081863, | |
| "tokens_seen": 1694105600 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.00019896680537166638, | |
| "loss": 2.5926, | |
| "theoretical_loss": 3.4784091876227046, | |
| "tokens_seen": 1697382400 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.0001983757329298279, | |
| "loss": 2.5451, | |
| "theoretical_loss": 3.4778317852886476, | |
| "tokens_seen": 1700659200 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.00019778466048798942, | |
| "loss": 2.5513, | |
| "theoretical_loss": 3.4772558052380216, | |
| "tokens_seen": 1703936000 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.00019719358804615094, | |
| "loss": 2.5281, | |
| "theoretical_loss": 3.476681241242323, | |
| "tokens_seen": 1707212800 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.00019660251560431246, | |
| "loss": 2.5711, | |
| "theoretical_loss": 3.4761080871122116, | |
| "tokens_seen": 1710489600 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.00019601144316247398, | |
| "loss": 2.5609, | |
| "theoretical_loss": 3.4755363366971896, | |
| "tokens_seen": 1713766400 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.00019542037072063553, | |
| "loss": 2.6015, | |
| "theoretical_loss": 3.4749659838852858, | |
| "tokens_seen": 1717043200 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.00019482929827879705, | |
| "loss": 2.592, | |
| "theoretical_loss": 3.4743970226027403, | |
| "tokens_seen": 1720320000 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.00019423822583695857, | |
| "loss": 2.5793, | |
| "theoretical_loss": 3.4738294468136934, | |
| "tokens_seen": 1723596800 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.00019364715339512012, | |
| "loss": 2.589, | |
| "theoretical_loss": 3.473263250519878, | |
| "tokens_seen": 1726873600 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.00019305608095328164, | |
| "loss": 2.5858, | |
| "theoretical_loss": 3.4726984277603146, | |
| "tokens_seen": 1730150400 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.00019246500851144316, | |
| "loss": 2.6317, | |
| "theoretical_loss": 3.472134972611009, | |
| "tokens_seen": 1733427200 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.00019187393606960468, | |
| "loss": 2.6081, | |
| "theoretical_loss": 3.471572879184653, | |
| "tokens_seen": 1736704000 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.00019128286362776623, | |
| "loss": 2.5788, | |
| "theoretical_loss": 3.4710121416303306, | |
| "tokens_seen": 1739980800 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.00019069179118592775, | |
| "loss": 2.5795, | |
| "theoretical_loss": 3.4704527541332224, | |
| "tokens_seen": 1743257600 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.00019010071874408927, | |
| "loss": 2.6276, | |
| "theoretical_loss": 3.469894710914317, | |
| "tokens_seen": 1746534400 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.00018950964630225082, | |
| "loss": 2.616, | |
| "theoretical_loss": 3.469338006230123, | |
| "tokens_seen": 1749811200 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 0.00018891857386041234, | |
| "loss": 2.6005, | |
| "theoretical_loss": 3.468782634372387, | |
| "tokens_seen": 1753088000 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 0.00018832750141857386, | |
| "loss": 2.5743, | |
| "theoretical_loss": 3.4682285896678087, | |
| "tokens_seen": 1756364800 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 0.0001877364289767354, | |
| "loss": 2.5893, | |
| "theoretical_loss": 3.467675866477763, | |
| "tokens_seen": 1759641600 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 0.00018714535653489693, | |
| "loss": 2.6116, | |
| "theoretical_loss": 3.4671244591980264, | |
| "tokens_seen": 1762918400 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 0.00018655428409305845, | |
| "loss": 2.5787, | |
| "theoretical_loss": 3.466574362258501, | |
| "tokens_seen": 1766195200 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 0.00018596321165122, | |
| "loss": 2.5956, | |
| "theoretical_loss": 3.4660255701229445, | |
| "tokens_seen": 1769472000 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 0.00018537213920938152, | |
| "loss": 2.576, | |
| "theoretical_loss": 3.4654780772887026, | |
| "tokens_seen": 1772748800 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 0.00018478106676754304, | |
| "loss": 2.6302, | |
| "theoretical_loss": 3.464931878286443, | |
| "tokens_seen": 1776025600 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.00018418999432570459, | |
| "loss": 2.6376, | |
| "theoretical_loss": 3.4643869676798937, | |
| "tokens_seen": 1779302400 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.0001835989218838661, | |
| "loss": 2.6387, | |
| "theoretical_loss": 3.463843340065581, | |
| "tokens_seen": 1782579200 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.0001830078494420276, | |
| "loss": 2.6107, | |
| "theoretical_loss": 3.4633009900725735, | |
| "tokens_seen": 1785856000 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.00018241677700018915, | |
| "loss": 2.6135, | |
| "theoretical_loss": 3.4627599123622255, | |
| "tokens_seen": 1789132800 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.00018182570455835067, | |
| "loss": 2.6239, | |
| "theoretical_loss": 3.462220101627925, | |
| "tokens_seen": 1792409600 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.0001812346321165122, | |
| "loss": 2.6567, | |
| "theoretical_loss": 3.4616815525948432, | |
| "tokens_seen": 1795686400 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.0001806435596746737, | |
| "loss": 2.6415, | |
| "theoretical_loss": 3.4611442600196867, | |
| "tokens_seen": 1798963200 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "objective/train/docs_used": 992685, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.6905806064605713, | |
| "objective/train/theoretical_loss": 3.460608218690451, | |
| "objective/train/tokens_used": 1822700000, | |
| "theoretical_loss": 3.460608218690451, | |
| "tokens_seen": 1802240000 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.00018005248723283526, | |
| "loss": 2.6327, | |
| "theoretical_loss": 3.460608218690451, | |
| "tokens_seen": 1802240000 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.00017946141479099678, | |
| "loss": 2.6035, | |
| "theoretical_loss": 3.460073423426179, | |
| "tokens_seen": 1805516800 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 0.0001788703423491583, | |
| "loss": 2.6307, | |
| "theoretical_loss": 3.4595398690767176, | |
| "tokens_seen": 1808793600 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 0.00017827926990731985, | |
| "loss": 2.6058, | |
| "theoretical_loss": 3.459007550522482, | |
| "tokens_seen": 1812070400 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 0.00017768819746548137, | |
| "loss": 2.6003, | |
| "theoretical_loss": 3.4584764626742173, | |
| "tokens_seen": 1815347200 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 0.0001770971250236429, | |
| "loss": 2.621, | |
| "theoretical_loss": 3.4579466004727664, | |
| "tokens_seen": 1818624000 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 0.00017650605258180444, | |
| "loss": 2.5904, | |
| "theoretical_loss": 3.4574179588888345, | |
| "tokens_seen": 1821900800 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 0.00017591498013996596, | |
| "loss": 2.5858, | |
| "theoretical_loss": 3.456890532922765, | |
| "tokens_seen": 1825177600 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 0.00017532390769812748, | |
| "loss": 2.5873, | |
| "theoretical_loss": 3.456364317604306, | |
| "tokens_seen": 1828454400 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 0.00017473283525628903, | |
| "loss": 2.6097, | |
| "theoretical_loss": 3.455839307992391, | |
| "tokens_seen": 1831731200 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.00017414176281445055, | |
| "loss": 2.563, | |
| "theoretical_loss": 3.45531549917491, | |
| "tokens_seen": 1835008000 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.00017355069037261207, | |
| "loss": 2.5792, | |
| "theoretical_loss": 3.454792886268492, | |
| "tokens_seen": 1838284800 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.00017295961793077362, | |
| "loss": 2.5772, | |
| "theoretical_loss": 3.454271464418285, | |
| "tokens_seen": 1841561600 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.00017236854548893514, | |
| "loss": 2.5527, | |
| "theoretical_loss": 3.4537512287977403, | |
| "tokens_seen": 1844838400 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.00017177747304709666, | |
| "loss": 2.5652, | |
| "theoretical_loss": 3.4532321746083956, | |
| "tokens_seen": 1848115200 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.0001711864006052582, | |
| "loss": 2.6013, | |
| "theoretical_loss": 3.4527142970796625, | |
| "tokens_seen": 1851392000 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.00017059532816341973, | |
| "loss": 2.5559, | |
| "theoretical_loss": 3.4521975914686163, | |
| "tokens_seen": 1854668800 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.00017000425572158125, | |
| "loss": 2.5467, | |
| "theoretical_loss": 3.4516820530597867, | |
| "tokens_seen": 1857945600 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.00016941318327974277, | |
| "loss": 2.5256, | |
| "theoretical_loss": 3.4511676771649524, | |
| "tokens_seen": 1861222400 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 0.00016882211083790432, | |
| "loss": 2.5289, | |
| "theoretical_loss": 3.450654459122932, | |
| "tokens_seen": 1864499200 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 0.0001682310383960658, | |
| "loss": 2.5144, | |
| "theoretical_loss": 3.4501423942993843, | |
| "tokens_seen": 1867776000 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 0.00016763996595422733, | |
| "loss": 2.5567, | |
| "theoretical_loss": 3.449631478086607, | |
| "tokens_seen": 1871052800 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 0.00016704889351238888, | |
| "loss": 2.6078, | |
| "theoretical_loss": 3.449121705903335, | |
| "tokens_seen": 1874329600 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 0.0001664578210705504, | |
| "loss": 2.5454, | |
| "theoretical_loss": 3.448613073194544, | |
| "tokens_seen": 1877606400 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 0.00016586674862871192, | |
| "loss": 2.5857, | |
| "theoretical_loss": 3.4481055754312564, | |
| "tokens_seen": 1880883200 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 0.00016527567618687347, | |
| "loss": 2.5579, | |
| "theoretical_loss": 3.4475992081103435, | |
| "tokens_seen": 1884160000 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 0.000164684603745035, | |
| "loss": 2.5673, | |
| "theoretical_loss": 3.4470939667543377, | |
| "tokens_seen": 1887436800 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.0001640935313031965, | |
| "loss": 2.5587, | |
| "theoretical_loss": 3.4465898469112366, | |
| "tokens_seen": 1890713600 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.00016350245886135806, | |
| "loss": 2.6082, | |
| "theoretical_loss": 3.4460868441543218, | |
| "tokens_seen": 1893990400 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.00016291138641951958, | |
| "loss": 2.6246, | |
| "theoretical_loss": 3.445584954081964, | |
| "tokens_seen": 1897267200 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.0001623203139776811, | |
| "loss": 2.6121, | |
| "theoretical_loss": 3.4450841723174426, | |
| "tokens_seen": 1900544000 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.00016172924153584265, | |
| "loss": 2.654, | |
| "theoretical_loss": 3.444584494508761, | |
| "tokens_seen": 1903820800 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.00016113816909400417, | |
| "loss": 2.6132, | |
| "theoretical_loss": 3.444085916328465, | |
| "tokens_seen": 1907097600 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.0001605470966521657, | |
| "loss": 2.6425, | |
| "theoretical_loss": 3.443588433473461, | |
| "tokens_seen": 1910374400 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.00015995602421032724, | |
| "loss": 2.6226, | |
| "theoretical_loss": 3.4430920416648396, | |
| "tokens_seen": 1913651200 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.00015936495176848876, | |
| "loss": 2.6118, | |
| "theoretical_loss": 3.4425967366476957, | |
| "tokens_seen": 1916928000 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 0.00015877387932665028, | |
| "loss": 2.5774, | |
| "theoretical_loss": 3.442102514190956, | |
| "tokens_seen": 1920204800 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 0.00015818280688481183, | |
| "loss": 2.5586, | |
| "theoretical_loss": 3.4416093700872032, | |
| "tokens_seen": 1923481600 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 0.00015759173444297335, | |
| "loss": 2.5404, | |
| "theoretical_loss": 3.441117300152505, | |
| "tokens_seen": 1926758400 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 0.00015700066200113487, | |
| "loss": 2.5354, | |
| "theoretical_loss": 3.440626300226242, | |
| "tokens_seen": 1930035200 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 0.0001564095895592964, | |
| "loss": 2.5394, | |
| "theoretical_loss": 3.4401363661709397, | |
| "tokens_seen": 1933312000 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 0.00015581851711745794, | |
| "loss": 2.5257, | |
| "theoretical_loss": 3.4396474938720996, | |
| "tokens_seen": 1936588800 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 0.00015522744467561946, | |
| "loss": 2.5411, | |
| "theoretical_loss": 3.439159679238035, | |
| "tokens_seen": 1939865600 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 0.00015463637223378098, | |
| "loss": 2.5559, | |
| "theoretical_loss": 3.4386729181997024, | |
| "tokens_seen": 1943142400 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.0001540452997919425, | |
| "loss": 2.5632, | |
| "theoretical_loss": 3.438187206710543, | |
| "tokens_seen": 1946419200 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.00015345422735010402, | |
| "loss": 2.5628, | |
| "theoretical_loss": 3.437702540746318, | |
| "tokens_seen": 1949696000 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.00015286315490826554, | |
| "loss": 2.5258, | |
| "theoretical_loss": 3.437218916304949, | |
| "tokens_seen": 1952972800 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.0001522720824664271, | |
| "loss": 2.5424, | |
| "theoretical_loss": 3.4367363294063575, | |
| "tokens_seen": 1956249600 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.0001516810100245886, | |
| "loss": 2.5398, | |
| "theoretical_loss": 3.4362547760923103, | |
| "tokens_seen": 1959526400 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.00015108993758275013, | |
| "loss": 2.5281, | |
| "theoretical_loss": 3.4357742524262616, | |
| "tokens_seen": 1962803200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.5869520331718451, | |
| "debugging/distinct-1-grams": 0.8018116994946082, | |
| "debugging/distinct-2-grams": 0.9741634621809702, | |
| "debugging/entropy-1-grams": 6.326946638586684, | |
| "debugging/entropy-2-grams": 7.252766261215823, | |
| "debugging/length": 549.25, | |
| "debugging/num_segments": 20, | |
| "epoch": 0.7, | |
| "objective/train/docs_used": 1082436, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.5232810974121094, | |
| "objective/train/theoretical_loss": 3.4352947544931967, | |
| "objective/train/tokens_used": 1986540000, | |
| "theoretical_loss": 3.4352947544931967, | |
| "tokens_seen": 1966080000 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.00015051068658974846, | |
| "loss": 2.4925, | |
| "theoretical_loss": 3.4352947544931967, | |
| "tokens_seen": 1966080000 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.00014991961414790998, | |
| "loss": 2.5493, | |
| "theoretical_loss": 3.4348162783994827, | |
| "tokens_seen": 1969356800 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.0001493285417060715, | |
| "loss": 2.5623, | |
| "theoretical_loss": 3.4343388202727114, | |
| "tokens_seen": 1972633600 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "learning_rate": 0.00014873746926423304, | |
| "loss": 2.6066, | |
| "theoretical_loss": 3.4338623762615526, | |
| "tokens_seen": 1975910400 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "learning_rate": 0.00014814639682239457, | |
| "loss": 2.6001, | |
| "theoretical_loss": 3.4333869425356025, | |
| "tokens_seen": 1979187200 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "learning_rate": 0.00014755532438055609, | |
| "loss": 2.6219, | |
| "theoretical_loss": 3.4329125152852367, | |
| "tokens_seen": 1982464000 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "learning_rate": 0.00014696425193871763, | |
| "loss": 2.6312, | |
| "theoretical_loss": 3.4324390907214624, | |
| "tokens_seen": 1985740800 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "learning_rate": 0.00014637317949687915, | |
| "loss": 2.5997, | |
| "theoretical_loss": 3.431966665075773, | |
| "tokens_seen": 1989017600 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "learning_rate": 0.00014578210705504068, | |
| "loss": 2.5963, | |
| "theoretical_loss": 3.4314952346000047, | |
| "tokens_seen": 1992294400 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "learning_rate": 0.00014519103461320222, | |
| "loss": 2.5781, | |
| "theoretical_loss": 3.4310247955661923, | |
| "tokens_seen": 1995571200 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "learning_rate": 0.00014459996217136372, | |
| "loss": 2.594, | |
| "theoretical_loss": 3.430555344266428, | |
| "tokens_seen": 1998848000 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.00014400888972952524, | |
| "loss": 2.5635, | |
| "theoretical_loss": 3.4300868770127213, | |
| "tokens_seen": 2002124800 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.00014341781728768676, | |
| "loss": 2.5736, | |
| "theoretical_loss": 3.4296193901368577, | |
| "tokens_seen": 2005401600 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.0001428267448458483, | |
| "loss": 2.6014, | |
| "theoretical_loss": 3.429152879990263, | |
| "tokens_seen": 2008678400 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.00014223567240400983, | |
| "loss": 2.5633, | |
| "theoretical_loss": 3.428687342943866, | |
| "tokens_seen": 2011955200 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.00014164459996217135, | |
| "loss": 2.5766, | |
| "theoretical_loss": 3.428222775387959, | |
| "tokens_seen": 2015232000 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.0001410535275203329, | |
| "loss": 2.5692, | |
| "theoretical_loss": 3.4277591737320696, | |
| "tokens_seen": 2018508800 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.00014046245507849442, | |
| "loss": 2.6077, | |
| "theoretical_loss": 3.4272965344048214, | |
| "tokens_seen": 2021785600 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.00013987138263665594, | |
| "loss": 2.5601, | |
| "theoretical_loss": 3.426834853853804, | |
| "tokens_seen": 2025062400 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.00013928031019481749, | |
| "loss": 2.5495, | |
| "theoretical_loss": 3.4263741285454428, | |
| "tokens_seen": 2028339200 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.000138689237752979, | |
| "loss": 2.4975, | |
| "theoretical_loss": 3.4259143549648674, | |
| "tokens_seen": 2031616000 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.00013809816531114053, | |
| "loss": 2.5257, | |
| "theoretical_loss": 3.4254555296157827, | |
| "tokens_seen": 2034892800 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.00013750709286930208, | |
| "loss": 2.5409, | |
| "theoretical_loss": 3.424997649020341, | |
| "tokens_seen": 2038169600 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.0001369160204274636, | |
| "loss": 2.5798, | |
| "theoretical_loss": 3.4245407097190173, | |
| "tokens_seen": 2041446400 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.00013632494798562512, | |
| "loss": 2.584, | |
| "theoretical_loss": 3.4240847082704793, | |
| "tokens_seen": 2044723200 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.00013573387554378666, | |
| "loss": 2.6136, | |
| "theoretical_loss": 3.423629641251467, | |
| "tokens_seen": 2048000000 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.00013514280310194819, | |
| "loss": 2.6027, | |
| "theoretical_loss": 3.4231755052566673, | |
| "tokens_seen": 2051276800 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.0001345517306601097, | |
| "loss": 2.6104, | |
| "theoretical_loss": 3.4227222968985904, | |
| "tokens_seen": 2054553600 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.00013396065821827125, | |
| "loss": 2.5917, | |
| "theoretical_loss": 3.4222700128074495, | |
| "tokens_seen": 2057830400 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 0.00013336958577643277, | |
| "loss": 2.6002, | |
| "theoretical_loss": 3.4218186496310414, | |
| "tokens_seen": 2061107200 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 0.0001327785133345943, | |
| "loss": 2.593, | |
| "theoretical_loss": 3.421368204034624, | |
| "tokens_seen": 2064384000 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 0.00013218744089275582, | |
| "loss": 2.5826, | |
| "theoretical_loss": 3.4209186727008, | |
| "tokens_seen": 2067660800 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 0.00013159636845091736, | |
| "loss": 2.5802, | |
| "theoretical_loss": 3.4204700523293985, | |
| "tokens_seen": 2070937600 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 0.00013100529600907889, | |
| "loss": 2.6029, | |
| "theoretical_loss": 3.42002233963736, | |
| "tokens_seen": 2074214400 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 0.00013041422356724038, | |
| "loss": 2.5511, | |
| "theoretical_loss": 3.4195755313586176, | |
| "tokens_seen": 2077491200 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 0.00012982315112540193, | |
| "loss": 2.5326, | |
| "theoretical_loss": 3.419129624243986, | |
| "tokens_seen": 2080768000 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 0.00012923207868356345, | |
| "loss": 2.5378, | |
| "theoretical_loss": 3.4186846150610464, | |
| "tokens_seen": 2084044800 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.00012864100624172497, | |
| "loss": 2.5343, | |
| "theoretical_loss": 3.418240500594031, | |
| "tokens_seen": 2087321600 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.00012804993379988652, | |
| "loss": 2.4991, | |
| "theoretical_loss": 3.417797277643717, | |
| "tokens_seen": 2090598400 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.00012745886135804804, | |
| "loss": 2.5032, | |
| "theoretical_loss": 3.417354943027311, | |
| "tokens_seen": 2093875200 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.00012686778891620956, | |
| "loss": 2.5254, | |
| "theoretical_loss": 3.41691349357834, | |
| "tokens_seen": 2097152000 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.0001262767164743711, | |
| "loss": 2.5469, | |
| "theoretical_loss": 3.416472926146545, | |
| "tokens_seen": 2100428800 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.00012568564403253263, | |
| "loss": 2.5205, | |
| "theoretical_loss": 3.4160332375977687, | |
| "tokens_seen": 2103705600 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.00012509457159069415, | |
| "loss": 2.5215, | |
| "theoretical_loss": 3.4155944248138512, | |
| "tokens_seen": 2106982400 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.0001245034991488557, | |
| "loss": 2.5038, | |
| "theoretical_loss": 3.415156484692524, | |
| "tokens_seen": 2110259200 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.00012391242670701722, | |
| "loss": 2.5025, | |
| "theoretical_loss": 3.4147194141473003, | |
| "tokens_seen": 2113536000 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "learning_rate": 0.00012332135426517874, | |
| "loss": 2.5036, | |
| "theoretical_loss": 3.414283210107377, | |
| "tokens_seen": 2116812800 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "learning_rate": 0.00012273028182334026, | |
| "loss": 2.5318, | |
| "theoretical_loss": 3.4138478695175243, | |
| "tokens_seen": 2120089600 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "learning_rate": 0.0001221392093815018, | |
| "loss": 2.5324, | |
| "theoretical_loss": 3.413413389337988, | |
| "tokens_seen": 2123366400 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "learning_rate": 0.00012154813693966333, | |
| "loss": 2.543, | |
| "theoretical_loss": 3.4129797665443826, | |
| "tokens_seen": 2126643200 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "objective/train/docs_used": 1175222, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.667830467224121, | |
| "objective/train/theoretical_loss": 3.4125469981275955, | |
| "objective/train/tokens_used": 2150380000, | |
| "theoretical_loss": 3.4125469981275955, | |
| "tokens_seen": 2129920000 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "learning_rate": 0.00012095706449782486, | |
| "loss": 2.5555, | |
| "theoretical_loss": 3.4125469981275955, | |
| "tokens_seen": 2129920000 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "learning_rate": 0.0001203659920559864, | |
| "loss": 2.5535, | |
| "theoretical_loss": 3.4121150810936816, | |
| "tokens_seen": 2133196800 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "learning_rate": 0.00011977491961414792, | |
| "loss": 2.572, | |
| "theoretical_loss": 3.411684012463766, | |
| "tokens_seen": 2136473600 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "learning_rate": 0.00011918384717230944, | |
| "loss": 2.541, | |
| "theoretical_loss": 3.411253789273946, | |
| "tokens_seen": 2139750400 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.00011859277473047097, | |
| "loss": 2.5419, | |
| "theoretical_loss": 3.4108244085751904, | |
| "tokens_seen": 2143027200 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.00011800170228863249, | |
| "loss": 2.5691, | |
| "theoretical_loss": 3.410395867433245, | |
| "tokens_seen": 2146304000 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.00011741062984679403, | |
| "loss": 2.5597, | |
| "theoretical_loss": 3.409968162928534, | |
| "tokens_seen": 2149580800 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.00011681955740495555, | |
| "loss": 2.5733, | |
| "theoretical_loss": 3.4095412921560655, | |
| "tokens_seen": 2152857600 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.00011622848496311708, | |
| "loss": 2.5886, | |
| "theoretical_loss": 3.4091152522253374, | |
| "tokens_seen": 2156134400 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.00011563741252127862, | |
| "loss": 2.5424, | |
| "theoretical_loss": 3.40869004026024, | |
| "tokens_seen": 2159411200 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.00011504634007944014, | |
| "loss": 2.5687, | |
| "theoretical_loss": 3.408265653398967, | |
| "tokens_seen": 2162688000 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.00011445526763760167, | |
| "loss": 2.582, | |
| "theoretical_loss": 3.40784208879392, | |
| "tokens_seen": 2165964800 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.0001138641951957632, | |
| "loss": 2.5622, | |
| "theoretical_loss": 3.407419343611616, | |
| "tokens_seen": 2169241600 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "learning_rate": 0.00011327312275392473, | |
| "loss": 2.5888, | |
| "theoretical_loss": 3.406997415032599, | |
| "tokens_seen": 2172518400 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "learning_rate": 0.00011268205031208626, | |
| "loss": 2.5804, | |
| "theoretical_loss": 3.406576300251348, | |
| "tokens_seen": 2175795200 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "learning_rate": 0.00011209097787024777, | |
| "loss": 2.5642, | |
| "theoretical_loss": 3.4061559964761856, | |
| "tokens_seen": 2179072000 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "learning_rate": 0.0001114999054284093, | |
| "loss": 2.538, | |
| "theoretical_loss": 3.4057365009291924, | |
| "tokens_seen": 2182348800 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "learning_rate": 0.00011090883298657084, | |
| "loss": 2.4998, | |
| "theoretical_loss": 3.405317810846116, | |
| "tokens_seen": 2185625600 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "learning_rate": 0.00011031776054473236, | |
| "loss": 2.5399, | |
| "theoretical_loss": 3.404899923476285, | |
| "tokens_seen": 2188902400 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "learning_rate": 0.00010972668810289389, | |
| "loss": 2.5541, | |
| "theoretical_loss": 3.404482836082521, | |
| "tokens_seen": 2192179200 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "learning_rate": 0.00010913561566105543, | |
| "loss": 2.5603, | |
| "theoretical_loss": 3.4040665459410517, | |
| "tokens_seen": 2195456000 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.00010855636466805371, | |
| "loss": 2.5236, | |
| "theoretical_loss": 3.4036510503414283, | |
| "tokens_seen": 2198732800 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.00010796529222621525, | |
| "loss": 2.5294, | |
| "theoretical_loss": 3.4032363465864375, | |
| "tokens_seen": 2202009600 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.00010737421978437678, | |
| "loss": 2.5373, | |
| "theoretical_loss": 3.402822431992018, | |
| "tokens_seen": 2205286400 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.0001067831473425383, | |
| "loss": 2.5897, | |
| "theoretical_loss": 3.402409303887179, | |
| "tokens_seen": 2208563200 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.00010619207490069983, | |
| "loss": 2.5264, | |
| "theoretical_loss": 3.401996959613914, | |
| "tokens_seen": 2211840000 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.00010560100245886137, | |
| "loss": 2.5302, | |
| "theoretical_loss": 3.4015853965271217, | |
| "tokens_seen": 2215116800 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.00010500993001702289, | |
| "loss": 2.5341, | |
| "theoretical_loss": 3.4011746119945214, | |
| "tokens_seen": 2218393600 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.00010441885757518442, | |
| "loss": 2.532, | |
| "theoretical_loss": 3.4007646033965746, | |
| "tokens_seen": 2221670400 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.00010382778513334595, | |
| "loss": 2.5769, | |
| "theoretical_loss": 3.4003553681264025, | |
| "tokens_seen": 2224947200 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 0.00010323671269150748, | |
| "loss": 2.5865, | |
| "theoretical_loss": 3.399946903589709, | |
| "tokens_seen": 2228224000 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 0.000102645640249669, | |
| "loss": 2.5682, | |
| "theoretical_loss": 3.3995392072046986, | |
| "tokens_seen": 2231500800 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 0.00010205456780783052, | |
| "loss": 2.5744, | |
| "theoretical_loss": 3.3991322764019993, | |
| "tokens_seen": 2234777600 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 0.00010146349536599206, | |
| "loss": 2.5492, | |
| "theoretical_loss": 3.398726108624585, | |
| "tokens_seen": 2238054400 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 0.00010087242292415359, | |
| "loss": 2.546, | |
| "theoretical_loss": 3.3983207013276977, | |
| "tokens_seen": 2241331200 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 0.00010028135048231511, | |
| "loss": 2.5121, | |
| "theoretical_loss": 3.3979160519787706, | |
| "tokens_seen": 2244608000 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 9.969027804047664e-05, | |
| "loss": 2.5364, | |
| "theoretical_loss": 3.397512158057353, | |
| "tokens_seen": 2247884800 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 9.909920559863817e-05, | |
| "loss": 2.5646, | |
| "theoretical_loss": 3.3971090170550333, | |
| "tokens_seen": 2251161600 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 9.85081331567997e-05, | |
| "loss": 2.5616, | |
| "theoretical_loss": 3.396706626475366, | |
| "tokens_seen": 2254438400 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 9.791706071496123e-05, | |
| "loss": 2.5831, | |
| "theoretical_loss": 3.396304983833794, | |
| "tokens_seen": 2257715200 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 9.732598827312276e-05, | |
| "loss": 2.5404, | |
| "theoretical_loss": 3.3959040866575805, | |
| "tokens_seen": 2260992000 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 9.673491583128429e-05, | |
| "loss": 2.6035, | |
| "theoretical_loss": 3.3955039324857292, | |
| "tokens_seen": 2264268800 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 9.614384338944582e-05, | |
| "loss": 2.766, | |
| "theoretical_loss": 3.395104518868916, | |
| "tokens_seen": 2267545600 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 9.555277094760733e-05, | |
| "loss": 2.7605, | |
| "theoretical_loss": 3.3947058433694157, | |
| "tokens_seen": 2270822400 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 9.496169850576887e-05, | |
| "loss": 2.7888, | |
| "theoretical_loss": 3.39430790356103, | |
| "tokens_seen": 2274099200 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 9.43706260639304e-05, | |
| "loss": 2.7413, | |
| "theoretical_loss": 3.3939106970290163, | |
| "tokens_seen": 2277376000 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 9.377955362209192e-05, | |
| "loss": 2.7489, | |
| "theoretical_loss": 3.3935142213700185, | |
| "tokens_seen": 2280652800 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 9.318848118025345e-05, | |
| "loss": 2.7259, | |
| "theoretical_loss": 3.393118474191996, | |
| "tokens_seen": 2283929600 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 9.259740873841498e-05, | |
| "loss": 2.6681, | |
| "theoretical_loss": 3.392723453114155, | |
| "tokens_seen": 2287206400 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 9.200633629657651e-05, | |
| "loss": 2.6338, | |
| "theoretical_loss": 3.3923291557668778, | |
| "tokens_seen": 2290483200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.48046349515430614, | |
| "debugging/distinct-1-grams": 0.8042247020937568, | |
| "debugging/distinct-2-grams": 0.9580200876097184, | |
| "debugging/entropy-1-grams": 5.882043341134262, | |
| "debugging/entropy-2-grams": 6.669433189402229, | |
| "debugging/length": 540.6666666666666, | |
| "debugging/num_segments": 12, | |
| "epoch": 0.82, | |
| "objective/train/docs_used": 1259215, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.660310745239258, | |
| "objective/train/theoretical_loss": 3.391935579791658, | |
| "objective/train/tokens_used": 2314220000, | |
| "theoretical_loss": 3.391935579791658, | |
| "tokens_seen": 2293760000 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 9.141526385473804e-05, | |
| "loss": 2.6198, | |
| "theoretical_loss": 3.391935579791658, | |
| "tokens_seen": 2293760000 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 9.082419141289957e-05, | |
| "loss": 2.6102, | |
| "theoretical_loss": 3.3915427228410278, | |
| "tokens_seen": 2297036800 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 9.02331189710611e-05, | |
| "loss": 2.5593, | |
| "theoretical_loss": 3.391150582578495, | |
| "tokens_seen": 2300313600 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 8.964204652922263e-05, | |
| "loss": 2.5466, | |
| "theoretical_loss": 3.390759156678474, | |
| "tokens_seen": 2303590400 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 8.905097408738415e-05, | |
| "loss": 2.5571, | |
| "theoretical_loss": 3.3903684428262197, | |
| "tokens_seen": 2306867200 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 8.845990164554568e-05, | |
| "loss": 2.5319, | |
| "theoretical_loss": 3.3899784387177623, | |
| "tokens_seen": 2310144000 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 8.786882920370721e-05, | |
| "loss": 2.5276, | |
| "theoretical_loss": 3.3895891420598403, | |
| "tokens_seen": 2313420800 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 8.727775676186873e-05, | |
| "loss": 2.5257, | |
| "theoretical_loss": 3.3892005505698384, | |
| "tokens_seen": 2316697600 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 8.668668432003026e-05, | |
| "loss": 2.5555, | |
| "theoretical_loss": 3.3888126619757197, | |
| "tokens_seen": 2319974400 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 8.609561187819179e-05, | |
| "loss": 2.5039, | |
| "theoretical_loss": 3.388425474015966, | |
| "tokens_seen": 2323251200 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 8.550453943635332e-05, | |
| "loss": 2.5474, | |
| "theoretical_loss": 3.3880389844395093, | |
| "tokens_seen": 2326528000 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 8.492528844335162e-05, | |
| "loss": 2.5243, | |
| "theoretical_loss": 3.3876531910056733, | |
| "tokens_seen": 2329804800 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 8.433421600151314e-05, | |
| "loss": 2.5288, | |
| "theoretical_loss": 3.387268091484109, | |
| "tokens_seen": 2333081600 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 8.374314355967467e-05, | |
| "loss": 2.5433, | |
| "theoretical_loss": 3.386883683654732, | |
| "tokens_seen": 2336358400 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 8.315207111783621e-05, | |
| "loss": 2.506, | |
| "theoretical_loss": 3.386499965307662, | |
| "tokens_seen": 2339635200 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 8.256099867599773e-05, | |
| "loss": 2.5493, | |
| "theoretical_loss": 3.3861169342431627, | |
| "tokens_seen": 2342912000 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 8.196992623415926e-05, | |
| "loss": 2.5451, | |
| "theoretical_loss": 3.3857345882715784, | |
| "tokens_seen": 2346188800 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 8.13788537923208e-05, | |
| "loss": 2.5648, | |
| "theoretical_loss": 3.3853529252132755, | |
| "tokens_seen": 2349465600 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 8.078778135048232e-05, | |
| "loss": 2.5358, | |
| "theoretical_loss": 3.384971942898583, | |
| "tokens_seen": 2352742400 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 8.019670890864385e-05, | |
| "loss": 2.5684, | |
| "theoretical_loss": 3.384591639167732, | |
| "tokens_seen": 2356019200 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 7.960563646680537e-05, | |
| "loss": 2.5936, | |
| "theoretical_loss": 3.3842120118707983, | |
| "tokens_seen": 2359296000 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 7.90145640249669e-05, | |
| "loss": 2.684, | |
| "theoretical_loss": 3.383833058867643, | |
| "tokens_seen": 2362572800 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 7.842349158312843e-05, | |
| "loss": 2.7256, | |
| "theoretical_loss": 3.3834547780278537, | |
| "tokens_seen": 2365849600 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "learning_rate": 7.783241914128995e-05, | |
| "loss": 2.6697, | |
| "theoretical_loss": 3.3830771672306885, | |
| "tokens_seen": 2369126400 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "learning_rate": 7.724134669945148e-05, | |
| "loss": 2.6742, | |
| "theoretical_loss": 3.382700224365018, | |
| "tokens_seen": 2372403200 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "learning_rate": 7.665027425761302e-05, | |
| "loss": 2.6314, | |
| "theoretical_loss": 3.382323947329268, | |
| "tokens_seen": 2375680000 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "learning_rate": 7.605920181577454e-05, | |
| "loss": 2.6131, | |
| "theoretical_loss": 3.3819483340313647, | |
| "tokens_seen": 2378956800 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "learning_rate": 7.546812937393607e-05, | |
| "loss": 2.5896, | |
| "theoretical_loss": 3.3815733823886776, | |
| "tokens_seen": 2382233600 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "learning_rate": 7.48770569320976e-05, | |
| "loss": 2.585, | |
| "theoretical_loss": 3.381199090327963, | |
| "tokens_seen": 2385510400 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "learning_rate": 7.428598449025913e-05, | |
| "loss": 2.5568, | |
| "theoretical_loss": 3.3808254557853115, | |
| "tokens_seen": 2388787200 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "learning_rate": 7.369491204842066e-05, | |
| "loss": 2.5472, | |
| "theoretical_loss": 3.3804524767060906, | |
| "tokens_seen": 2392064000 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 7.310383960658218e-05, | |
| "loss": 2.5715, | |
| "theoretical_loss": 3.3800801510448917, | |
| "tokens_seen": 2395340800 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 7.251276716474372e-05, | |
| "loss": 2.5446, | |
| "theoretical_loss": 3.379708476765477, | |
| "tokens_seen": 2398617600 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 7.192169472290524e-05, | |
| "loss": 2.5644, | |
| "theoretical_loss": 3.3793374518407235, | |
| "tokens_seen": 2401894400 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 7.133062228106676e-05, | |
| "loss": 2.5413, | |
| "theoretical_loss": 3.3789670742525715, | |
| "tokens_seen": 2405171200 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 7.07395498392283e-05, | |
| "loss": 2.5785, | |
| "theoretical_loss": 3.3785973419919717, | |
| "tokens_seen": 2408448000 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 7.014847739738983e-05, | |
| "loss": 2.5597, | |
| "theoretical_loss": 3.3782282530588335, | |
| "tokens_seen": 2411724800 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 6.955740495555135e-05, | |
| "loss": 2.5616, | |
| "theoretical_loss": 3.3778598054619695, | |
| "tokens_seen": 2415001600 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 6.896633251371288e-05, | |
| "loss": 2.5725, | |
| "theoretical_loss": 3.377491997219049, | |
| "tokens_seen": 2418278400 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 6.83752600718744e-05, | |
| "loss": 2.6175, | |
| "theoretical_loss": 3.3771248263565425, | |
| "tokens_seen": 2421555200 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "learning_rate": 6.778418763003594e-05, | |
| "loss": 2.6412, | |
| "theoretical_loss": 3.376758290909673, | |
| "tokens_seen": 2424832000 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "learning_rate": 6.719311518819747e-05, | |
| "loss": 2.5729, | |
| "theoretical_loss": 3.3763923889223646, | |
| "tokens_seen": 2428108800 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "learning_rate": 6.6602042746359e-05, | |
| "loss": 2.5669, | |
| "theoretical_loss": 3.3760271184471917, | |
| "tokens_seen": 2431385600 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "learning_rate": 6.601097030452053e-05, | |
| "loss": 2.5985, | |
| "theoretical_loss": 3.3756624775453306, | |
| "tokens_seen": 2434662400 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "learning_rate": 6.541989786268206e-05, | |
| "loss": 2.5538, | |
| "theoretical_loss": 3.3752984642865087, | |
| "tokens_seen": 2437939200 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "learning_rate": 6.482882542084357e-05, | |
| "loss": 2.5882, | |
| "theoretical_loss": 3.374935076748957, | |
| "tokens_seen": 2441216000 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "learning_rate": 6.42377529790051e-05, | |
| "loss": 2.5917, | |
| "theoretical_loss": 3.374572313019359, | |
| "tokens_seen": 2444492800 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "learning_rate": 6.364668053716664e-05, | |
| "loss": 2.5866, | |
| "theoretical_loss": 3.3742101711928036, | |
| "tokens_seen": 2447769600 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 6.305560809532816e-05, | |
| "loss": 2.5881, | |
| "theoretical_loss": 3.3738486493727375, | |
| "tokens_seen": 2451046400 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 6.24645356534897e-05, | |
| "loss": 2.5814, | |
| "theoretical_loss": 3.3734877456709165, | |
| "tokens_seen": 2454323200 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "objective/train/docs_used": 1345711, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.9188690185546875, | |
| "objective/train/theoretical_loss": 3.373127458207357, | |
| "objective/train/tokens_used": 2478060000, | |
| "theoretical_loss": 3.373127458207357, | |
| "tokens_seen": 2457600000 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 6.187346321165123e-05, | |
| "loss": 2.6105, | |
| "theoretical_loss": 3.373127458207357, | |
| "tokens_seen": 2457600000 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 6.128239076981275e-05, | |
| "loss": 2.5592, | |
| "theoretical_loss": 3.372767785110293, | |
| "tokens_seen": 2460876800 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 6.0691318327974276e-05, | |
| "loss": 2.5637, | |
| "theoretical_loss": 3.3724087245161227, | |
| "tokens_seen": 2464153600 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 6.010024588613581e-05, | |
| "loss": 2.578, | |
| "theoretical_loss": 3.3720502745693697, | |
| "tokens_seen": 2467430400 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 5.950917344429733e-05, | |
| "loss": 2.5871, | |
| "theoretical_loss": 3.37169243342263, | |
| "tokens_seen": 2470707200 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 5.891810100245886e-05, | |
| "loss": 2.5113, | |
| "theoretical_loss": 3.3713351992365315, | |
| "tokens_seen": 2473984000 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 5.8327028560620387e-05, | |
| "loss": 2.4866, | |
| "theoretical_loss": 3.3709785701796844, | |
| "tokens_seen": 2477260800 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "learning_rate": 5.773595611878192e-05, | |
| "loss": 2.5251, | |
| "theoretical_loss": 3.3706225444286395, | |
| "tokens_seen": 2480537600 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "learning_rate": 5.714488367694345e-05, | |
| "loss": 2.5443, | |
| "theoretical_loss": 3.3702671201678407, | |
| "tokens_seen": 2483814400 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "learning_rate": 5.6553811235104976e-05, | |
| "loss": 2.5642, | |
| "theoretical_loss": 3.3699122955895815, | |
| "tokens_seen": 2487091200 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "learning_rate": 5.596273879326651e-05, | |
| "loss": 2.555, | |
| "theoretical_loss": 3.3695580688939617, | |
| "tokens_seen": 2490368000 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "learning_rate": 5.537166635142803e-05, | |
| "loss": 2.5536, | |
| "theoretical_loss": 3.3692044382888415, | |
| "tokens_seen": 2493644800 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "learning_rate": 5.478059390958956e-05, | |
| "loss": 2.5017, | |
| "theoretical_loss": 3.368851401989799, | |
| "tokens_seen": 2496921600 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "learning_rate": 5.4189521467751086e-05, | |
| "loss": 2.506, | |
| "theoretical_loss": 3.3684989582200884, | |
| "tokens_seen": 2500198400 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "learning_rate": 5.359844902591262e-05, | |
| "loss": 2.5191, | |
| "theoretical_loss": 3.368147105210593, | |
| "tokens_seen": 2503475200 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 5.300737658407415e-05, | |
| "loss": 2.5528, | |
| "theoretical_loss": 3.3677958411997873, | |
| "tokens_seen": 2506752000 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 5.242812559107244e-05, | |
| "loss": 2.5008, | |
| "theoretical_loss": 3.36744516443369, | |
| "tokens_seen": 2510028800 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 5.1837053149233975e-05, | |
| "loss": 2.5105, | |
| "theoretical_loss": 3.3670950731658253, | |
| "tokens_seen": 2513305600 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 5.12459807073955e-05, | |
| "loss": 2.5355, | |
| "theoretical_loss": 3.36674556565718, | |
| "tokens_seen": 2516582400 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 5.065490826555703e-05, | |
| "loss": 2.4907, | |
| "theoretical_loss": 3.3663966401761614, | |
| "tokens_seen": 2519859200 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 5.006383582371855e-05, | |
| "loss": 2.5101, | |
| "theoretical_loss": 3.3660482949985555, | |
| "tokens_seen": 2523136000 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 4.9472763381880085e-05, | |
| "loss": 2.5194, | |
| "theoretical_loss": 3.3657005284074883, | |
| "tokens_seen": 2526412800 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 4.888169094004161e-05, | |
| "loss": 2.5581, | |
| "theoretical_loss": 3.3653533386933834, | |
| "tokens_seen": 2529689600 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 4.829061849820314e-05, | |
| "loss": 2.5165, | |
| "theoretical_loss": 3.3650067241539205, | |
| "tokens_seen": 2532966400 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "learning_rate": 4.769954605636467e-05, | |
| "loss": 2.5328, | |
| "theoretical_loss": 3.364660683093999, | |
| "tokens_seen": 2536243200 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "learning_rate": 4.71084736145262e-05, | |
| "loss": 2.5059, | |
| "theoretical_loss": 3.364315213825694, | |
| "tokens_seen": 2539520000 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "learning_rate": 4.651740117268772e-05, | |
| "loss": 2.5284, | |
| "theoretical_loss": 3.3639703146682187, | |
| "tokens_seen": 2542796800 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "learning_rate": 4.592632873084925e-05, | |
| "loss": 2.5169, | |
| "theoretical_loss": 3.3636259839478853, | |
| "tokens_seen": 2546073600 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "learning_rate": 4.533525628901078e-05, | |
| "loss": 2.5112, | |
| "theoretical_loss": 3.363282219998065, | |
| "tokens_seen": 2549350400 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "learning_rate": 4.474418384717231e-05, | |
| "loss": 2.5054, | |
| "theoretical_loss": 3.3629390211591494, | |
| "tokens_seen": 2552627200 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "learning_rate": 4.415311140533384e-05, | |
| "loss": 2.4644, | |
| "theoretical_loss": 3.3625963857785135, | |
| "tokens_seen": 2555904000 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "learning_rate": 4.356203896349537e-05, | |
| "loss": 2.4737, | |
| "theoretical_loss": 3.3622543122104744, | |
| "tokens_seen": 2559180800 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 4.2970966521656895e-05, | |
| "loss": 2.4844, | |
| "theoretical_loss": 3.361912798816256, | |
| "tokens_seen": 2562457600 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 4.237989407981842e-05, | |
| "loss": 2.4945, | |
| "theoretical_loss": 3.36157184396395, | |
| "tokens_seen": 2565734400 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 4.178882163797995e-05, | |
| "loss": 2.4899, | |
| "theoretical_loss": 3.3612314460284782, | |
| "tokens_seen": 2569011200 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 4.119774919614148e-05, | |
| "loss": 2.4836, | |
| "theoretical_loss": 3.360891603391557, | |
| "tokens_seen": 2572288000 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 4.060667675430301e-05, | |
| "loss": 2.4996, | |
| "theoretical_loss": 3.3605523144416574, | |
| "tokens_seen": 2575564800 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 4.001560431246454e-05, | |
| "loss": 2.5084, | |
| "theoretical_loss": 3.3602135775739708, | |
| "tokens_seen": 2578841600 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 3.942453187062606e-05, | |
| "loss": 2.537, | |
| "theoretical_loss": 3.3598753911903705, | |
| "tokens_seen": 2582118400 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 3.883345942878759e-05, | |
| "loss": 2.5565, | |
| "theoretical_loss": 3.359537753699379, | |
| "tokens_seen": 2585395200 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 3.824238698694912e-05, | |
| "loss": 2.4955, | |
| "theoretical_loss": 3.3592006635161265, | |
| "tokens_seen": 2588672000 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 3.765131454511065e-05, | |
| "loss": 2.3944, | |
| "theoretical_loss": 3.35886411906232, | |
| "tokens_seen": 2591948800 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 3.706024210327218e-05, | |
| "loss": 2.424, | |
| "theoretical_loss": 3.358528118766205, | |
| "tokens_seen": 2595225600 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 3.6469169661433705e-05, | |
| "loss": 2.4393, | |
| "theoretical_loss": 3.3581926610625317, | |
| "tokens_seen": 2598502400 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 3.587809721959524e-05, | |
| "loss": 2.4611, | |
| "theoretical_loss": 3.357857744392518, | |
| "tokens_seen": 2601779200 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 3.528702477775676e-05, | |
| "loss": 2.4475, | |
| "theoretical_loss": 3.3575233672038176, | |
| "tokens_seen": 2605056000 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 3.469595233591829e-05, | |
| "loss": 2.4459, | |
| "theoretical_loss": 3.3571895279504824, | |
| "tokens_seen": 2608332800 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 3.410487989407982e-05, | |
| "loss": 2.4551, | |
| "theoretical_loss": 3.3568562250929306, | |
| "tokens_seen": 2611609600 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 3.351380745224135e-05, | |
| "loss": 2.4631, | |
| "theoretical_loss": 3.3565234570979094, | |
| "tokens_seen": 2614886400 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 3.292273501040288e-05, | |
| "loss": 2.4765, | |
| "theoretical_loss": 3.356191222438466, | |
| "tokens_seen": 2618163200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.5350316281348282, | |
| "debugging/distinct-1-grams": 0.7499811548409807, | |
| "debugging/distinct-2-grams": 0.9503268132289873, | |
| "debugging/entropy-1-grams": 6.017543497187338, | |
| "debugging/entropy-2-grams": 6.914532683826491, | |
| "debugging/length": 535.3333333333334, | |
| "debugging/num_segments": 15, | |
| "epoch": 0.94, | |
| "objective/train/docs_used": 1436793, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.264730453491211, | |
| "objective/train/theoretical_loss": 3.355859519593907, | |
| "objective/train/tokens_used": 2641900000, | |
| "theoretical_loss": 3.355859519593907, | |
| "tokens_seen": 2621440000 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 3.2331662568564405e-05, | |
| "loss": 2.4713, | |
| "theoretical_loss": 3.355859519593907, | |
| "tokens_seen": 2621440000 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 3.174059012672593e-05, | |
| "loss": 2.4439, | |
| "theoretical_loss": 3.355528347049773, | |
| "tokens_seen": 2624716800 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 3.114951768488746e-05, | |
| "loss": 2.4839, | |
| "theoretical_loss": 3.355197703297798, | |
| "tokens_seen": 2627993600 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 3.055844524304899e-05, | |
| "loss": 2.4635, | |
| "theoretical_loss": 3.354867586835882, | |
| "tokens_seen": 2631270400 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 2.996737280121052e-05, | |
| "loss": 2.4607, | |
| "theoretical_loss": 3.354537996168054, | |
| "tokens_seen": 2634547200 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 2.9376300359372043e-05, | |
| "loss": 2.4894, | |
| "theoretical_loss": 3.3542089298044413, | |
| "tokens_seen": 2637824000 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 2.8785227917533574e-05, | |
| "loss": 2.4857, | |
| "theoretical_loss": 3.3538803862612374, | |
| "tokens_seen": 2641100800 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 2.81941554756951e-05, | |
| "loss": 2.4814, | |
| "theoretical_loss": 3.3535523640606697, | |
| "tokens_seen": 2644377600 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 2.760308303385663e-05, | |
| "loss": 2.5029, | |
| "theoretical_loss": 3.353224861730966, | |
| "tokens_seen": 2647654400 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 2.7012010592018156e-05, | |
| "loss": 2.4801, | |
| "theoretical_loss": 3.3528978778063254, | |
| "tokens_seen": 2650931200 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 2.6420938150179687e-05, | |
| "loss": 2.4773, | |
| "theoretical_loss": 3.3525714108268834, | |
| "tokens_seen": 2654208000 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 2.5829865708341215e-05, | |
| "loss": 2.4719, | |
| "theoretical_loss": 3.352245459338684, | |
| "tokens_seen": 2657484800 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 2.5238793266502742e-05, | |
| "loss": 2.4565, | |
| "theoretical_loss": 3.351920021893646, | |
| "tokens_seen": 2660761600 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 2.4647720824664273e-05, | |
| "loss": 2.4615, | |
| "theoretical_loss": 3.3515950970495334, | |
| "tokens_seen": 2664038400 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 2.4056648382825798e-05, | |
| "loss": 2.4938, | |
| "theoretical_loss": 3.3512706833699237, | |
| "tokens_seen": 2667315200 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 2.346557594098733e-05, | |
| "loss": 2.5427, | |
| "theoretical_loss": 3.3509467794241794, | |
| "tokens_seen": 2670592000 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 2.2874503499148856e-05, | |
| "loss": 2.5622, | |
| "theoretical_loss": 3.350623383787414, | |
| "tokens_seen": 2673868800 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 2.2283431057310387e-05, | |
| "loss": 2.5586, | |
| "theoretical_loss": 3.350300495040466, | |
| "tokens_seen": 2677145600 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 2.169235861547191e-05, | |
| "loss": 2.5254, | |
| "theoretical_loss": 3.3499781117698646, | |
| "tokens_seen": 2680422400 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 2.1101286173633442e-05, | |
| "loss": 2.5645, | |
| "theoretical_loss": 3.349656232567805, | |
| "tokens_seen": 2683699200 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 2.051021373179497e-05, | |
| "loss": 2.565, | |
| "theoretical_loss": 3.3493348560321143, | |
| "tokens_seen": 2686976000 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 1.9919141289956497e-05, | |
| "loss": 2.5324, | |
| "theoretical_loss": 3.349013980766224, | |
| "tokens_seen": 2690252800 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 1.9328068848118025e-05, | |
| "loss": 2.5272, | |
| "theoretical_loss": 3.3486936053791423, | |
| "tokens_seen": 2693529600 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 1.8736996406279556e-05, | |
| "loss": 2.5369, | |
| "theoretical_loss": 3.348373728485421, | |
| "tokens_seen": 2696806400 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 1.814592396444108e-05, | |
| "loss": 2.5098, | |
| "theoretical_loss": 3.348054348705131, | |
| "tokens_seen": 2700083200 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 1.755485152260261e-05, | |
| "loss": 2.5098, | |
| "theoretical_loss": 3.3477354646638315, | |
| "tokens_seen": 2703360000 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 1.6963779080764142e-05, | |
| "loss": 2.521, | |
| "theoretical_loss": 3.3474170749925416, | |
| "tokens_seen": 2706636800 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 1.6372706638925666e-05, | |
| "loss": 2.5341, | |
| "theoretical_loss": 3.347099178327713, | |
| "tokens_seen": 2709913600 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 1.5781634197087197e-05, | |
| "loss": 2.5157, | |
| "theoretical_loss": 3.3467817733112, | |
| "tokens_seen": 2713190400 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 1.5190561755248723e-05, | |
| "loss": 2.5076, | |
| "theoretical_loss": 3.3464648585902346, | |
| "tokens_seen": 2716467200 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 1.4599489313410252e-05, | |
| "loss": 2.5107, | |
| "theoretical_loss": 3.3461484328173956, | |
| "tokens_seen": 2719744000 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 1.400841687157178e-05, | |
| "loss": 2.4993, | |
| "theoretical_loss": 3.3458324946505824, | |
| "tokens_seen": 2723020800 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 1.3417344429733307e-05, | |
| "loss": 2.5064, | |
| "theoretical_loss": 3.3455170427529883, | |
| "tokens_seen": 2726297600 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 1.2826271987894837e-05, | |
| "loss": 2.5183, | |
| "theoretical_loss": 3.345202075793073, | |
| "tokens_seen": 2729574400 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 1.2235199546056366e-05, | |
| "loss": 2.5015, | |
| "theoretical_loss": 3.344887592444535, | |
| "tokens_seen": 2732851200 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 1.1644127104217893e-05, | |
| "loss": 2.5255, | |
| "theoretical_loss": 3.3445735913862835, | |
| "tokens_seen": 2736128000 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 1.1053054662379423e-05, | |
| "loss": 2.5233, | |
| "theoretical_loss": 3.344260071302415, | |
| "tokens_seen": 2739404800 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 1.046198222054095e-05, | |
| "loss": 2.5229, | |
| "theoretical_loss": 3.3439470308821844, | |
| "tokens_seen": 2742681600 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 9.870909778702478e-06, | |
| "loss": 2.5185, | |
| "theoretical_loss": 3.3436344688199786, | |
| "tokens_seen": 2745958400 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 9.279837336864007e-06, | |
| "loss": 2.5139, | |
| "theoretical_loss": 3.3433223838152917, | |
| "tokens_seen": 2749235200 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 8.688764895025535e-06, | |
| "loss": 2.5262, | |
| "theoretical_loss": 3.3430107745726976, | |
| "tokens_seen": 2752512000 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 8.097692453187062e-06, | |
| "loss": 2.5189, | |
| "theoretical_loss": 3.3426996398018245, | |
| "tokens_seen": 2755788800 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 7.506620011348591e-06, | |
| "loss": 2.494, | |
| "theoretical_loss": 3.3423889782173304, | |
| "tokens_seen": 2759065600 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 6.915547569510119e-06, | |
| "loss": 2.4834, | |
| "theoretical_loss": 3.342078788538875, | |
| "tokens_seen": 2762342400 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 6.324475127671647e-06, | |
| "loss": 2.4697, | |
| "theoretical_loss": 3.3417690694910975, | |
| "tokens_seen": 2765619200 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 5.733402685833176e-06, | |
| "loss": 2.4931, | |
| "theoretical_loss": 3.3414598198035885, | |
| "tokens_seen": 2768896000 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 5.142330243994704e-06, | |
| "loss": 2.482, | |
| "theoretical_loss": 3.3411510382108665, | |
| "tokens_seen": 2772172800 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 4.563079250993002e-06, | |
| "loss": 2.4989, | |
| "theoretical_loss": 3.340842723452353, | |
| "tokens_seen": 2775449600 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 3.97200680915453e-06, | |
| "loss": 2.5136, | |
| "theoretical_loss": 3.340534874272347, | |
| "tokens_seen": 2778726400 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 3.3809343673160584e-06, | |
| "loss": 2.4836, | |
| "theoretical_loss": 3.340227489420001, | |
| "tokens_seen": 2782003200 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "objective/train/docs_used": 1527878, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.157845973968506, | |
| "objective/train/theoretical_loss": 3.339920567649296, | |
| "objective/train/tokens_used": 2805740000, | |
| "theoretical_loss": 3.339920567649296, | |
| "tokens_seen": 2785280000 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 2.789861925477587e-06, | |
| "loss": 2.4693, | |
| "theoretical_loss": 3.339920567649296, | |
| "tokens_seen": 2785280000 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "learning_rate": 2.198789483639115e-06, | |
| "loss": 2.5378, | |
| "theoretical_loss": 3.339614107719018, | |
| "tokens_seen": 2788556800 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "learning_rate": 1.607717041800643e-06, | |
| "loss": 2.5596, | |
| "theoretical_loss": 3.339308108392733, | |
| "tokens_seen": 2791833600 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "learning_rate": 1.0166445999621714e-06, | |
| "loss": 2.6115, | |
| "theoretical_loss": 3.3390025684387643, | |
| "tokens_seen": 2795110400 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "learning_rate": 4.255721581236997e-07, | |
| "loss": 2.5856, | |
| "theoretical_loss": 3.3386974866301666, | |
| "tokens_seen": 2798387200 | |
| } | |
| ], | |
| "max_steps": 42724, | |
| "num_train_epochs": 9223372036854775807, | |
| "total_flos": 1.428920419221504e+18, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |