|
{ |
|
"best_metric": null, |
|
"best_model_checkpoint": null, |
|
"epoch": 1.0, |
|
"global_step": 42724, |
|
"is_hyper_param_search": false, |
|
"is_local_process_zero": true, |
|
"is_world_process_zero": true, |
|
"log_history": [ |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 1.1682242990654204e-06, |
|
"loss": 10.8686, |
|
"theoretical_loss": 20.81281176760504, |
|
"tokens_seen": 65536 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 5.841121495327103e-05, |
|
"loss": 9.0294, |
|
"theoretical_loss": 8.563476630668958, |
|
"tokens_seen": 3276800 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 0.00011682242990654206, |
|
"loss": 6.976, |
|
"theoretical_loss": 7.477752684105921, |
|
"tokens_seen": 6553600 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 0.00017523364485981307, |
|
"loss": 6.1307, |
|
"theoretical_loss": 6.9337484549527915, |
|
"tokens_seen": 9830400 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 0.00023364485981308412, |
|
"loss": 5.7622, |
|
"theoretical_loss": 6.5835601944843045, |
|
"tokens_seen": 13107200 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.00029205607476635517, |
|
"loss": 5.4993, |
|
"theoretical_loss": 6.3307075311739744, |
|
"tokens_seen": 16384000 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.00035046728971962614, |
|
"loss": 5.2788, |
|
"theoretical_loss": 6.135523197998216, |
|
"tokens_seen": 19660800 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0004088785046728972, |
|
"loss": 5.1139, |
|
"theoretical_loss": 5.978095549927499, |
|
"tokens_seen": 22937600 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.00046728971962616824, |
|
"loss": 5.0316, |
|
"theoretical_loss": 5.847111292323815, |
|
"tokens_seen": 26214400 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.000499739928125591, |
|
"loss": 4.8652, |
|
"theoretical_loss": 5.735570781940016, |
|
"tokens_seen": 29491200 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0004991488556837526, |
|
"loss": 4.7488, |
|
"theoretical_loss": 5.638864110129244, |
|
"tokens_seen": 32768000 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0004985577832419141, |
|
"loss": 4.6745, |
|
"theoretical_loss": 5.553806347902798, |
|
"tokens_seen": 36044800 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0004979667108000757, |
|
"loss": 4.5438, |
|
"theoretical_loss": 5.478112046614329, |
|
"tokens_seen": 39321600 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004973756383582371, |
|
"loss": 4.4609, |
|
"theoretical_loss": 5.410089925637252, |
|
"tokens_seen": 42598400 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004967845659163987, |
|
"loss": 4.4206, |
|
"theoretical_loss": 5.348456049793725, |
|
"tokens_seen": 45875200 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004961934934745603, |
|
"loss": 4.3209, |
|
"theoretical_loss": 5.292214532995457, |
|
"tokens_seen": 49152000 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004956024210327218, |
|
"loss": 4.2638, |
|
"theoretical_loss": 5.240578591827869, |
|
"tokens_seen": 52428800 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004950113485908833, |
|
"loss": 4.1812, |
|
"theoretical_loss": 5.192916690583679, |
|
"tokens_seen": 55705600 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004944202761490448, |
|
"loss": 4.1175, |
|
"theoretical_loss": 5.148714829414331, |
|
"tokens_seen": 58982400 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004938292037072064, |
|
"loss": 3.9738, |
|
"theoretical_loss": 5.107549528462992, |
|
"tokens_seen": 62259200 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004932381312653678, |
|
"loss": 3.8582, |
|
"theoretical_loss": 5.069068083201136, |
|
"tokens_seen": 65536000 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004926470588235294, |
|
"loss": 3.7998, |
|
"theoretical_loss": 5.032973875895897, |
|
"tokens_seen": 68812800 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.000492055986381691, |
|
"loss": 3.7438, |
|
"theoretical_loss": 4.999015274282555, |
|
"tokens_seen": 72089600 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004914649139398525, |
|
"loss": 3.7365, |
|
"theoretical_loss": 4.966977121409853, |
|
"tokens_seen": 75366400 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.000490873841498014, |
|
"loss": 3.6564, |
|
"theoretical_loss": 4.936674127683027, |
|
"tokens_seen": 78643200 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004902827690561755, |
|
"loss": 3.5971, |
|
"theoretical_loss": 4.907945679887972, |
|
"tokens_seen": 81920000 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004896916966143371, |
|
"loss": 3.6012, |
|
"theoretical_loss": 4.8806517198708175, |
|
"tokens_seen": 85196800 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004891006241724985, |
|
"loss": 3.5647, |
|
"theoretical_loss": 4.85466944053967, |
|
"tokens_seen": 88473600 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004885095517306601, |
|
"loss": 3.5473, |
|
"theoretical_loss": 4.829890613366704, |
|
"tokens_seen": 91750400 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004879184792888217, |
|
"loss": 3.4657, |
|
"theoretical_loss": 4.806219408835812, |
|
"tokens_seen": 95027200 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0004873274068469832, |
|
"loss": 3.4688, |
|
"theoretical_loss": 4.783570605334148, |
|
"tokens_seen": 98304000 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.00048673633440514467, |
|
"loss": 3.4763, |
|
"theoretical_loss": 4.761868106830299, |
|
"tokens_seen": 101580800 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0004861452619633062, |
|
"loss": 3.4809, |
|
"theoretical_loss": 4.741043708020364, |
|
"tokens_seen": 104857600 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0004855541895214677, |
|
"loss": 3.4084, |
|
"theoretical_loss": 4.721036059306941, |
|
"tokens_seen": 108134400 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0004849631170796293, |
|
"loss": 3.4574, |
|
"theoretical_loss": 4.701789794289756, |
|
"tokens_seen": 111411200 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0004843720446377908, |
|
"loss": 3.4441, |
|
"theoretical_loss": 4.68325479029382, |
|
"tokens_seen": 114688000 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.00048378097219595233, |
|
"loss": 3.4289, |
|
"theoretical_loss": 4.6653855384841725, |
|
"tokens_seen": 117964800 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.00048318989975411385, |
|
"loss": 3.3685, |
|
"theoretical_loss": 4.6481406047776295, |
|
"tokens_seen": 121241600 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.00048259882731227537, |
|
"loss": 3.3637, |
|
"theoretical_loss": 4.631482166397534, |
|
"tokens_seen": 124518400 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0004820077548704369, |
|
"loss": 3.3737, |
|
"theoretical_loss": 4.615375611773608, |
|
"tokens_seen": 127795200 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.00048141668242859847, |
|
"loss": 3.3587, |
|
"theoretical_loss": 4.5997891937483955, |
|
"tokens_seen": 131072000 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.00048082560998676, |
|
"loss": 3.3354, |
|
"theoretical_loss": 4.584693727850565, |
|
"tokens_seen": 134348800 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0004802345375449215, |
|
"loss": 3.3025, |
|
"theoretical_loss": 4.570062328836407, |
|
"tokens_seen": 137625600 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.00047964346510308303, |
|
"loss": 3.283, |
|
"theoretical_loss": 4.5558701798619285, |
|
"tokens_seen": 140902400 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.00047905239266124455, |
|
"loss": 3.3052, |
|
"theoretical_loss": 4.542094329588689, |
|
"tokens_seen": 144179200 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.00047846132021940607, |
|
"loss": 3.2876, |
|
"theoretical_loss": 4.528713513292708, |
|
"tokens_seen": 147456000 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0004778702477775676, |
|
"loss": 3.2959, |
|
"theoretical_loss": 4.515707994672887, |
|
"tokens_seen": 150732800 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.00047727917533572917, |
|
"loss": 3.2606, |
|
"theoretical_loss": 4.503059425571229, |
|
"tokens_seen": 154009600 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0004766881028938907, |
|
"loss": 3.2412, |
|
"theoretical_loss": 4.490750721243157, |
|
"tokens_seen": 157286400 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0004760970304520522, |
|
"loss": 3.3205, |
|
"theoretical_loss": 4.478765949169858, |
|
"tokens_seen": 160563200 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"objective/train/docs_used": 101104, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 3.560319423675537, |
|
"objective/train/theoretical_loss": 4.46709022969911, |
|
"objective/train/tokens_used": 184300000, |
|
"theoretical_loss": 4.46709022969911, |
|
"tokens_seen": 163840000 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.00047550595801021373, |
|
"loss": 3.2093, |
|
"theoretical_loss": 4.46709022969911, |
|
"tokens_seen": 163840000 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.00047491488556837525, |
|
"loss": 3.2395, |
|
"theoretical_loss": 4.455709647047437, |
|
"tokens_seen": 167116800 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.00047432381312653677, |
|
"loss": 3.1921, |
|
"theoretical_loss": 4.444611169403321, |
|
"tokens_seen": 170393600 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.00047373274068469835, |
|
"loss": 3.1717, |
|
"theoretical_loss": 4.4337825770455375, |
|
"tokens_seen": 173670400 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.00047314166824285987, |
|
"loss": 3.169, |
|
"theoretical_loss": 4.423212397538051, |
|
"tokens_seen": 176947200 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0004725505958010214, |
|
"loss": 3.1623, |
|
"theoretical_loss": 4.4128898471880325, |
|
"tokens_seen": 180224000 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0004719595233591829, |
|
"loss": 3.1722, |
|
"theoretical_loss": 4.40280477805997, |
|
"tokens_seen": 183500800 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.00047136845091734443, |
|
"loss": 3.1483, |
|
"theoretical_loss": 4.392947629929752, |
|
"tokens_seen": 186777600 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.00047077737847550595, |
|
"loss": 3.176, |
|
"theoretical_loss": 4.383309386640423, |
|
"tokens_seen": 190054400 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0004701863060336675, |
|
"loss": 3.1829, |
|
"theoretical_loss": 4.373881536388167, |
|
"tokens_seen": 193331200 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.00046959523359182905, |
|
"loss": 3.1694, |
|
"theoretical_loss": 4.364656035524595, |
|
"tokens_seen": 196608000 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.00046900416114999057, |
|
"loss": 3.1938, |
|
"theoretical_loss": 4.355625275511174, |
|
"tokens_seen": 199884800 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0004684130887081521, |
|
"loss": 3.1654, |
|
"theoretical_loss": 4.346782052704563, |
|
"tokens_seen": 203161600 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0004678220162663136, |
|
"loss": 3.1277, |
|
"theoretical_loss": 4.338119540689052, |
|
"tokens_seen": 206438400 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.00046723094382447513, |
|
"loss": 3.1532, |
|
"theoretical_loss": 4.329631264904703, |
|
"tokens_seen": 209715200 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.00046663987138263665, |
|
"loss": 3.1625, |
|
"theoretical_loss": 4.321311079348144, |
|
"tokens_seen": 212992000 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0004660487989407982, |
|
"loss": 3.0982, |
|
"theoretical_loss": 4.313153145147683, |
|
"tokens_seen": 216268800 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.00046545772649895975, |
|
"loss": 3.1152, |
|
"theoretical_loss": 4.305151910836119, |
|
"tokens_seen": 219545600 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.00046486665405712127, |
|
"loss": 3.073, |
|
"theoretical_loss": 4.2973020941635784, |
|
"tokens_seen": 222822400 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0004642755816152828, |
|
"loss": 3.1169, |
|
"theoretical_loss": 4.28959866530949, |
|
"tokens_seen": 226099200 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0004636845091734443, |
|
"loss": 3.129, |
|
"theoretical_loss": 4.282036831367506, |
|
"tokens_seen": 229376000 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.00046309343673160583, |
|
"loss": 3.1552, |
|
"theoretical_loss": 4.274612021990189, |
|
"tokens_seen": 232652800 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0004625023642897674, |
|
"loss": 3.1511, |
|
"theoretical_loss": 4.267319876091788, |
|
"tokens_seen": 235929600 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004619112918479289, |
|
"loss": 3.1489, |
|
"theoretical_loss": 4.260156229517635, |
|
"tokens_seen": 239206400 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.00046132021940609044, |
|
"loss": 3.1118, |
|
"theoretical_loss": 4.253117103597704, |
|
"tokens_seen": 242483200 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.00046072914696425197, |
|
"loss": 3.0504, |
|
"theoretical_loss": 4.246198694509945, |
|
"tokens_seen": 245760000 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004601380745224135, |
|
"loss": 3.0691, |
|
"theoretical_loss": 4.239397363386152, |
|
"tokens_seen": 249036800 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.000459547002080575, |
|
"loss": 3.0577, |
|
"theoretical_loss": 4.232709627099522, |
|
"tokens_seen": 252313600 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004589559296387366, |
|
"loss": 3.0658, |
|
"theoretical_loss": 4.226132149678757, |
|
"tokens_seen": 255590400 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004583648571968981, |
|
"loss": 3.1149, |
|
"theoretical_loss": 4.219661734298666, |
|
"tokens_seen": 258867200 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004577737847550596, |
|
"loss": 3.0236, |
|
"theoretical_loss": 4.213295315801815, |
|
"tokens_seen": 262144000 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004571827123132211, |
|
"loss": 3.0275, |
|
"theoretical_loss": 4.207029953709861, |
|
"tokens_seen": 265420800 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0004565916398713826, |
|
"loss": 2.9819, |
|
"theoretical_loss": 4.200862825686893, |
|
"tokens_seen": 268697600 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.00045600056742954413, |
|
"loss": 3.0371, |
|
"theoretical_loss": 4.19479122142044, |
|
"tokens_seen": 271974400 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0004554094949877057, |
|
"loss": 2.9829, |
|
"theoretical_loss": 4.188812536888775, |
|
"tokens_seen": 275251200 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.00045481842254586723, |
|
"loss": 2.9501, |
|
"theoretical_loss": 4.182924268985855, |
|
"tokens_seen": 278528000 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.00045422735010402875, |
|
"loss": 2.9744, |
|
"theoretical_loss": 4.177124010477671, |
|
"tokens_seen": 281804800 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.00045363627766219027, |
|
"loss": 3.0125, |
|
"theoretical_loss": 4.171409445265983, |
|
"tokens_seen": 285081600 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0004530452052203518, |
|
"loss": 2.9955, |
|
"theoretical_loss": 4.165778343937409, |
|
"tokens_seen": 288358400 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0004524541327785133, |
|
"loss": 2.9929, |
|
"theoretical_loss": 4.160228559577659, |
|
"tokens_seen": 291635200 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.00045186306033667483, |
|
"loss": 3.0244, |
|
"theoretical_loss": 4.15475802383233, |
|
"tokens_seen": 294912000 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0004512719878948364, |
|
"loss": 3.0282, |
|
"theoretical_loss": 4.149364743197177, |
|
"tokens_seen": 298188800 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.00045068091545299793, |
|
"loss": 2.9998, |
|
"theoretical_loss": 4.14404679552214, |
|
"tokens_seen": 301465600 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.00045008984301115945, |
|
"loss": 3.0129, |
|
"theoretical_loss": 4.138802326714632, |
|
"tokens_seen": 304742400 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.00044949877056932097, |
|
"loss": 2.9824, |
|
"theoretical_loss": 4.133629547628726, |
|
"tokens_seen": 308019200 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0004489076981274825, |
|
"loss": 2.9203, |
|
"theoretical_loss": 4.128526731127894, |
|
"tokens_seen": 311296000 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.000448316625685644, |
|
"loss": 2.9514, |
|
"theoretical_loss": 4.123492209309923, |
|
"tokens_seen": 314572800 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0004477255532438056, |
|
"loss": 2.953, |
|
"theoretical_loss": 4.118524370883447, |
|
"tokens_seen": 317849600 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0004471344808019671, |
|
"loss": 2.9618, |
|
"theoretical_loss": 4.113621658686355, |
|
"tokens_seen": 321126400 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.00044654340836012863, |
|
"loss": 2.9246, |
|
"theoretical_loss": 4.108782567337039, |
|
"tokens_seen": 324403200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.5689206637784514, |
|
"debugging/distinct-1-grams": 0.7604200619411795, |
|
"debugging/distinct-2-grams": 0.9515819938155399, |
|
"debugging/entropy-1-grams": 6.176763024314691, |
|
"debugging/entropy-2-grams": 7.1817278155572515, |
|
"debugging/length": 550.2105263157895, |
|
"debugging/num_segments": 19, |
|
"epoch": 0.12, |
|
"objective/train/docs_used": 191655, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 3.2289257049560547, |
|
"objective/train/theoretical_loss": 4.104005641010112, |
|
"objective/train/tokens_used": 348140000, |
|
"theoretical_loss": 4.104005641010112, |
|
"tokens_seen": 327680000 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.00044595233591829015, |
|
"loss": 2.9268, |
|
"theoretical_loss": 4.104005641010112, |
|
"tokens_seen": 327680000 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.00044536126347645167, |
|
"loss": 2.9683, |
|
"theoretical_loss": 4.099289471328812, |
|
"tokens_seen": 330956800 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0004447701910346132, |
|
"loss": 3.0014, |
|
"theoretical_loss": 4.094632695366921, |
|
"tokens_seen": 334233600 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.00044417911859277476, |
|
"loss": 2.96, |
|
"theoretical_loss": 4.090033993753448, |
|
"tokens_seen": 337510400 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0004435880461509363, |
|
"loss": 2.9557, |
|
"theoretical_loss": 4.085492088873883, |
|
"tokens_seen": 340787200 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0004429969737090978, |
|
"loss": 2.9289, |
|
"theoretical_loss": 4.081005743162224, |
|
"tokens_seen": 344064000 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0004424059012672593, |
|
"loss": 2.9346, |
|
"theoretical_loss": 4.076573757478361, |
|
"tokens_seen": 347340800 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.00044181482882542085, |
|
"loss": 2.9196, |
|
"theoretical_loss": 4.072194969565807, |
|
"tokens_seen": 350617600 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.00044122375638358237, |
|
"loss": 2.9241, |
|
"theoretical_loss": 4.067868252585089, |
|
"tokens_seen": 353894400 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.0004406326839417439, |
|
"loss": 2.9505, |
|
"theoretical_loss": 4.063592513718411, |
|
"tokens_seen": 357171200 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.00044004161149990546, |
|
"loss": 2.9314, |
|
"theoretical_loss": 4.059366692841521, |
|
"tokens_seen": 360448000 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.000439450539058067, |
|
"loss": 2.9642, |
|
"theoretical_loss": 4.055189761258959, |
|
"tokens_seen": 363724800 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.0004388594666162285, |
|
"loss": 2.9659, |
|
"theoretical_loss": 4.051060720499127, |
|
"tokens_seen": 367001600 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.00043826839417439, |
|
"loss": 2.9061, |
|
"theoretical_loss": 4.046978601165831, |
|
"tokens_seen": 370278400 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.00043767732173255155, |
|
"loss": 2.9487, |
|
"theoretical_loss": 4.042942461843204, |
|
"tokens_seen": 373555200 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.00043708624929071307, |
|
"loss": 2.9171, |
|
"theoretical_loss": 4.038951388051044, |
|
"tokens_seen": 376832000 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.00043649517684887464, |
|
"loss": 2.892, |
|
"theoretical_loss": 4.035004491247873, |
|
"tokens_seen": 380108800 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.00043590410440703616, |
|
"loss": 2.9348, |
|
"theoretical_loss": 4.031100907879109, |
|
"tokens_seen": 383385600 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0004353130319651977, |
|
"loss": 2.8516, |
|
"theoretical_loss": 4.02723979846797, |
|
"tokens_seen": 386662400 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0004347219595233592, |
|
"loss": 2.8972, |
|
"theoretical_loss": 4.023420346746835, |
|
"tokens_seen": 389939200 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0004341308870815207, |
|
"loss": 2.9333, |
|
"theoretical_loss": 4.019641758826938, |
|
"tokens_seen": 393216000 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.00043353981463968225, |
|
"loss": 2.9127, |
|
"theoretical_loss": 4.015903262404413, |
|
"tokens_seen": 396492800 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0004329487421978438, |
|
"loss": 2.8845, |
|
"theoretical_loss": 4.012204106000786, |
|
"tokens_seen": 399769600 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.00043235766975600534, |
|
"loss": 2.9178, |
|
"theoretical_loss": 4.008543558236181, |
|
"tokens_seen": 403046400 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.00043176659731416686, |
|
"loss": 2.9254, |
|
"theoretical_loss": 4.004920907133565, |
|
"tokens_seen": 406323200 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0004311755248723284, |
|
"loss": 2.9181, |
|
"theoretical_loss": 4.001335459452449, |
|
"tokens_seen": 409600000 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0004305844524304899, |
|
"loss": 2.906, |
|
"theoretical_loss": 3.997786540050617, |
|
"tokens_seen": 412876800 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0004299933799886514, |
|
"loss": 2.9371, |
|
"theoretical_loss": 3.9942734912724456, |
|
"tokens_seen": 416153600 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.00042940230754681295, |
|
"loss": 2.9134, |
|
"theoretical_loss": 3.9907956723625375, |
|
"tokens_seen": 419430400 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0004288112351049745, |
|
"loss": 2.9714, |
|
"theoretical_loss": 3.9873524589034224, |
|
"tokens_seen": 422707200 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.000428220162663136, |
|
"loss": 2.9432, |
|
"theoretical_loss": 3.9839432422761556, |
|
"tokens_seen": 425984000 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0004276290902212975, |
|
"loss": 2.9521, |
|
"theoretical_loss": 3.980567429142721, |
|
"tokens_seen": 429260800 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.00042703801777945903, |
|
"loss": 2.9304, |
|
"theoretical_loss": 3.977224440949197, |
|
"tokens_seen": 432537600 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.00042644694533762055, |
|
"loss": 2.9453, |
|
"theoretical_loss": 3.9739137134486917, |
|
"tokens_seen": 435814400 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.00042585587289578207, |
|
"loss": 2.9322, |
|
"theoretical_loss": 3.9706346962431396, |
|
"tokens_seen": 439091200 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.00042526480045394365, |
|
"loss": 2.9113, |
|
"theoretical_loss": 3.9673868523430564, |
|
"tokens_seen": 442368000 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.00042467372801210517, |
|
"loss": 2.8985, |
|
"theoretical_loss": 3.9641696577444376, |
|
"tokens_seen": 445644800 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.0004240826555702667, |
|
"loss": 2.9016, |
|
"theoretical_loss": 3.9609826010220033, |
|
"tokens_seen": 448921600 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.0004234915831284282, |
|
"loss": 2.9171, |
|
"theoretical_loss": 3.9578251829380506, |
|
"tokens_seen": 452198400 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.00042290051068658973, |
|
"loss": 2.8842, |
|
"theoretical_loss": 3.954696916066199, |
|
"tokens_seen": 455475200 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.00042230943824475125, |
|
"loss": 2.8876, |
|
"theoretical_loss": 3.9515973244293643, |
|
"tokens_seen": 458752000 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0004217183658029128, |
|
"loss": 2.8417, |
|
"theoretical_loss": 3.948525943151326, |
|
"tokens_seen": 462028800 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.00042112729336107435, |
|
"loss": 2.8284, |
|
"theoretical_loss": 3.9454823181212815, |
|
"tokens_seen": 465305600 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.00042053622091923587, |
|
"loss": 2.8623, |
|
"theoretical_loss": 3.9424660056708167, |
|
"tokens_seen": 468582400 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0004199451484773974, |
|
"loss": 2.8993, |
|
"theoretical_loss": 3.939476572262754, |
|
"tokens_seen": 471859200 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0004193540760355589, |
|
"loss": 2.8604, |
|
"theoretical_loss": 3.9365135941913563, |
|
"tokens_seen": 475136000 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.00041876300359372043, |
|
"loss": 2.7975, |
|
"theoretical_loss": 3.9335766572934023, |
|
"tokens_seen": 478412800 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.00041817193115188195, |
|
"loss": 2.8516, |
|
"theoretical_loss": 3.9306653566696603, |
|
"tokens_seen": 481689600 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0004175808587100435, |
|
"loss": 2.8076, |
|
"theoretical_loss": 3.927779296416332, |
|
"tokens_seen": 484966400 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.00041698978626820505, |
|
"loss": 2.8758, |
|
"theoretical_loss": 3.924918089366024, |
|
"tokens_seen": 488243200 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"objective/train/docs_used": 280313, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.973356246948242, |
|
"objective/train/theoretical_loss": 3.9220813568378707, |
|
"objective/train/tokens_used": 511980000, |
|
"theoretical_loss": 3.9220813568378707, |
|
"tokens_seen": 491520000 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.00041639871382636657, |
|
"loss": 2.8567, |
|
"theoretical_loss": 3.9220813568378707, |
|
"tokens_seen": 491520000 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0004158076413845281, |
|
"loss": 2.9071, |
|
"theoretical_loss": 3.9192687283964096, |
|
"tokens_seen": 494796800 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0004152165689426896, |
|
"loss": 2.878, |
|
"theoretical_loss": 3.9164798416188527, |
|
"tokens_seen": 498073600 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.00041462549650085113, |
|
"loss": 2.8168, |
|
"theoretical_loss": 3.913714341870409, |
|
"tokens_seen": 501350400 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0004140344240590127, |
|
"loss": 2.8544, |
|
"theoretical_loss": 3.9109718820873303, |
|
"tokens_seen": 504627200 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0004134433516171742, |
|
"loss": 2.8518, |
|
"theoretical_loss": 3.9082521225673625, |
|
"tokens_seen": 507904000 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.00041285227917533575, |
|
"loss": 2.8566, |
|
"theoretical_loss": 3.9055547307673075, |
|
"tokens_seen": 511180800 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.00041226120673349727, |
|
"loss": 2.8176, |
|
"theoretical_loss": 3.9028793811074056, |
|
"tokens_seen": 514457600 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0004116701342916588, |
|
"loss": 2.7896, |
|
"theoretical_loss": 3.900225754782274, |
|
"tokens_seen": 517734400 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0004110790618498203, |
|
"loss": 2.8166, |
|
"theoretical_loss": 3.897593539578138, |
|
"tokens_seen": 521011200 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0004104879894079819, |
|
"loss": 2.7744, |
|
"theoretical_loss": 3.8949824296961015, |
|
"tokens_seen": 524288000 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0004098969169661434, |
|
"loss": 2.8054, |
|
"theoretical_loss": 3.8923921255812353, |
|
"tokens_seen": 527564800 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0004093058445243049, |
|
"loss": 2.795, |
|
"theoretical_loss": 3.8898223337572393, |
|
"tokens_seen": 530841600 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.00040871477208246645, |
|
"loss": 2.8216, |
|
"theoretical_loss": 3.88727276666648, |
|
"tokens_seen": 534118400 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.00040812369964062797, |
|
"loss": 2.8089, |
|
"theoretical_loss": 3.884743142515184, |
|
"tokens_seen": 537395200 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0004075326271987895, |
|
"loss": 2.8525, |
|
"theoretical_loss": 3.8822331851235985, |
|
"tokens_seen": 540672000 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.000406941554756951, |
|
"loss": 2.8511, |
|
"theoretical_loss": 3.87974262378093, |
|
"tokens_seen": 543948800 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0004063504823151126, |
|
"loss": 2.8256, |
|
"theoretical_loss": 3.877271193104873, |
|
"tokens_seen": 547225600 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0004057594098732741, |
|
"loss": 2.8307, |
|
"theoretical_loss": 3.8748186329055736, |
|
"tokens_seen": 550502400 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0004051683374314356, |
|
"loss": 2.8738, |
|
"theoretical_loss": 3.87238468805384, |
|
"tokens_seen": 553779200 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.00040457726498959715, |
|
"loss": 2.8065, |
|
"theoretical_loss": 3.8699691083534633, |
|
"tokens_seen": 557056000 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.00040398619254775867, |
|
"loss": 2.8596, |
|
"theoretical_loss": 3.8675716484174907, |
|
"tokens_seen": 560332800 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0004033951201059202, |
|
"loss": 2.8416, |
|
"theoretical_loss": 3.8651920675482936, |
|
"tokens_seen": 563609600 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.00040280404766408176, |
|
"loss": 2.8393, |
|
"theoretical_loss": 3.862830129621318, |
|
"tokens_seen": 566886400 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0004022129752222433, |
|
"loss": 2.8942, |
|
"theoretical_loss": 3.8604856029723575, |
|
"tokens_seen": 570163200 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0004016219027804048, |
|
"loss": 2.8987, |
|
"theoretical_loss": 3.8581582602882447, |
|
"tokens_seen": 573440000 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0004010308303385663, |
|
"loss": 2.8711, |
|
"theoretical_loss": 3.8558478785008203, |
|
"tokens_seen": 576716800 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.00040043975789672785, |
|
"loss": 2.869, |
|
"theoretical_loss": 3.8535542386840778, |
|
"tokens_seen": 579993600 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0003998486854548893, |
|
"loss": 2.8397, |
|
"theoretical_loss": 3.8512771259543586, |
|
"tokens_seen": 583270400 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0003992694344618877, |
|
"loss": 2.8679, |
|
"theoretical_loss": 3.8490163293735082, |
|
"tokens_seen": 586547200 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0003986783620200492, |
|
"loss": 2.8821, |
|
"theoretical_loss": 3.8467716418548648, |
|
"tokens_seen": 589824000 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.00039808728957821073, |
|
"loss": 2.8444, |
|
"theoretical_loss": 3.844542860072007, |
|
"tokens_seen": 593100800 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.00039749621713637225, |
|
"loss": 2.8058, |
|
"theoretical_loss": 3.8423297843701496, |
|
"tokens_seen": 596377600 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0003969051446945338, |
|
"loss": 2.82, |
|
"theoretical_loss": 3.8401322186800995, |
|
"tokens_seen": 599654400 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0003963140722526953, |
|
"loss": 2.8123, |
|
"theoretical_loss": 3.83794997043469, |
|
"tokens_seen": 602931200 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.00039572299981085687, |
|
"loss": 2.8265, |
|
"theoretical_loss": 3.8357828504876004, |
|
"tokens_seen": 606208000 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0003951319273690184, |
|
"loss": 2.7866, |
|
"theoretical_loss": 3.833630673034487, |
|
"tokens_seen": 609484800 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0003945408549271799, |
|
"loss": 2.7965, |
|
"theoretical_loss": 3.831493255536345, |
|
"tokens_seen": 612761600 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.00039394978248534143, |
|
"loss": 2.8207, |
|
"theoretical_loss": 3.8293704186450253, |
|
"tokens_seen": 616038400 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.00039335871004350295, |
|
"loss": 2.8041, |
|
"theoretical_loss": 3.827261986130839, |
|
"tokens_seen": 619315200 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0003927676376016645, |
|
"loss": 2.81, |
|
"theoretical_loss": 3.825167784812175, |
|
"tokens_seen": 622592000 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.000392176565159826, |
|
"loss": 2.7531, |
|
"theoretical_loss": 3.823087644487069, |
|
"tokens_seen": 625868800 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.00039158549271798757, |
|
"loss": 2.781, |
|
"theoretical_loss": 3.8210213978666565, |
|
"tokens_seen": 629145600 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0003909944202761491, |
|
"loss": 2.8013, |
|
"theoretical_loss": 3.8189688805104476, |
|
"tokens_seen": 632422400 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.00039040334783431056, |
|
"loss": 2.8092, |
|
"theoretical_loss": 3.816929930763374, |
|
"tokens_seen": 635699200 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0003898122753924721, |
|
"loss": 2.8463, |
|
"theoretical_loss": 3.8149043896945347, |
|
"tokens_seen": 638976000 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0003892212029506336, |
|
"loss": 2.8477, |
|
"theoretical_loss": 3.812892101037601, |
|
"tokens_seen": 642252800 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0003886301305087951, |
|
"loss": 2.8368, |
|
"theoretical_loss": 3.81089291113282, |
|
"tokens_seen": 645529600 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0003880390580669567, |
|
"loss": 2.8435, |
|
"theoretical_loss": 3.8089066688705673, |
|
"tokens_seen": 648806400 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0003874479856251182, |
|
"loss": 2.8113, |
|
"theoretical_loss": 3.8069332256363992, |
|
"tokens_seen": 652083200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.5486494935750937, |
|
"debugging/distinct-1-grams": 0.8031186825041262, |
|
"debugging/distinct-2-grams": 0.959647362572758, |
|
"debugging/entropy-1-grams": 6.201095616386137, |
|
"debugging/entropy-2-grams": 7.080013788468194, |
|
"debugging/length": 487.95, |
|
"debugging/num_segments": 20, |
|
"epoch": 0.23, |
|
"objective/train/docs_used": 369191, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.8638339042663574, |
|
"objective/train/theoretical_loss": 3.80497243525756, |
|
"objective/train/tokens_used": 675820000, |
|
"theoretical_loss": 3.80497243525756, |
|
"tokens_seen": 655360000 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.00038685691318327974, |
|
"loss": 2.7935, |
|
"theoretical_loss": 3.80497243525756, |
|
"tokens_seen": 655360000 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.00038626584074144126, |
|
"loss": 2.7789, |
|
"theoretical_loss": 3.8030241539508958, |
|
"tokens_seen": 658636800 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0003856747682996028, |
|
"loss": 2.8171, |
|
"theoretical_loss": 3.8010882402721324, |
|
"tokens_seen": 661913600 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0003850836958577643, |
|
"loss": 2.8064, |
|
"theoretical_loss": 3.7991645550664757, |
|
"tokens_seen": 665190400 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0003844926234159259, |
|
"loss": 2.8106, |
|
"theoretical_loss": 3.797252961420492, |
|
"tokens_seen": 668467200 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0003839015509740874, |
|
"loss": 2.8205, |
|
"theoretical_loss": 3.795353324615228, |
|
"tokens_seen": 671744000 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0003833104785322489, |
|
"loss": 2.833, |
|
"theoretical_loss": 3.793465512080541, |
|
"tokens_seen": 675020800 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.00038271940609041044, |
|
"loss": 2.836, |
|
"theoretical_loss": 3.791589393350587, |
|
"tokens_seen": 678297600 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.00038212833364857196, |
|
"loss": 2.8456, |
|
"theoretical_loss": 3.7897248400204475, |
|
"tokens_seen": 681574400 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0003815372612067335, |
|
"loss": 2.8388, |
|
"theoretical_loss": 3.7878717257038534, |
|
"tokens_seen": 684851200 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.000380946188764895, |
|
"loss": 2.8519, |
|
"theoretical_loss": 3.7860299259919685, |
|
"tokens_seen": 688128000 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0003803551163230566, |
|
"loss": 2.8263, |
|
"theoretical_loss": 3.7841993184132114, |
|
"tokens_seen": 691404800 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0003797640438812181, |
|
"loss": 2.8164, |
|
"theoretical_loss": 3.78237978239408, |
|
"tokens_seen": 694681600 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0003791729714393796, |
|
"loss": 2.8002, |
|
"theoretical_loss": 3.780571199220942, |
|
"tokens_seen": 697958400 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.00037858189899754114, |
|
"loss": 2.7888, |
|
"theoretical_loss": 3.7787734520027803, |
|
"tokens_seen": 701235200 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.00037799082655570266, |
|
"loss": 2.776, |
|
"theoretical_loss": 3.7769864256348455, |
|
"tokens_seen": 704512000 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0003773997541138642, |
|
"loss": 2.7873, |
|
"theoretical_loss": 3.775210006763202, |
|
"tokens_seen": 707788800 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.00037680868167202575, |
|
"loss": 2.7815, |
|
"theoretical_loss": 3.7734440837501406, |
|
"tokens_seen": 711065600 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0003762176092301873, |
|
"loss": 2.7815, |
|
"theoretical_loss": 3.7716885466404246, |
|
"tokens_seen": 714342400 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0003756265367883488, |
|
"loss": 2.8186, |
|
"theoretical_loss": 3.769943287128357, |
|
"tokens_seen": 717619200 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0003750354643465103, |
|
"loss": 2.7491, |
|
"theoretical_loss": 3.7682081985256364, |
|
"tokens_seen": 720896000 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.00037444439190467184, |
|
"loss": 2.7574, |
|
"theoretical_loss": 3.7664831757299795, |
|
"tokens_seen": 724172800 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.00037385331946283336, |
|
"loss": 2.7375, |
|
"theoretical_loss": 3.7647681151944976, |
|
"tokens_seen": 727449600 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.00037326224702099493, |
|
"loss": 2.7197, |
|
"theoretical_loss": 3.7630629148977937, |
|
"tokens_seen": 730726400 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.00037267117457915645, |
|
"loss": 2.7261, |
|
"theoretical_loss": 3.761367474314768, |
|
"tokens_seen": 734003200 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.000372080102137318, |
|
"loss": 2.7753, |
|
"theoretical_loss": 3.7596816943881084, |
|
"tokens_seen": 737280000 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0003714890296954795, |
|
"loss": 2.7859, |
|
"theoretical_loss": 3.758005477500451, |
|
"tokens_seen": 740556800 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.000370897957253641, |
|
"loss": 2.8045, |
|
"theoretical_loss": 3.756338727447186, |
|
"tokens_seen": 743833600 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.00037030688481180254, |
|
"loss": 2.8171, |
|
"theoretical_loss": 3.7546813494098945, |
|
"tokens_seen": 747110400 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.00036971581236996406, |
|
"loss": 2.7877, |
|
"theoretical_loss": 3.7530332499304007, |
|
"tokens_seen": 750387200 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.00036912473992812563, |
|
"loss": 2.7664, |
|
"theoretical_loss": 3.7513943368854195, |
|
"tokens_seen": 753664000 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.00036853366748628715, |
|
"loss": 2.7609, |
|
"theoretical_loss": 3.7497645194617863, |
|
"tokens_seen": 756940800 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0003679425950444487, |
|
"loss": 2.7662, |
|
"theoretical_loss": 3.748143708132246, |
|
"tokens_seen": 760217600 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0003673515226026102, |
|
"loss": 2.7662, |
|
"theoretical_loss": 3.7465318146317994, |
|
"tokens_seen": 763494400 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0003667604501607717, |
|
"loss": 2.6799, |
|
"theoretical_loss": 3.7449287519345766, |
|
"tokens_seen": 766771200 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.00036616937771893324, |
|
"loss": 2.7247, |
|
"theoretical_loss": 3.7433344342312385, |
|
"tokens_seen": 770048000 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0003655783052770948, |
|
"loss": 2.7386, |
|
"theoretical_loss": 3.7417487769068756, |
|
"tokens_seen": 773324800 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.00036498723283525633, |
|
"loss": 2.7687, |
|
"theoretical_loss": 3.7401716965194076, |
|
"tokens_seen": 776601600 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.00036439616039341785, |
|
"loss": 2.7443, |
|
"theoretical_loss": 3.738603110778461, |
|
"tokens_seen": 779878400 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0003638050879515794, |
|
"loss": 2.8035, |
|
"theoretical_loss": 3.73704293852471, |
|
"tokens_seen": 783155200 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0003632140155097409, |
|
"loss": 2.7902, |
|
"theoretical_loss": 3.7354910997096793, |
|
"tokens_seen": 786432000 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0003626229430679024, |
|
"loss": 2.79, |
|
"theoretical_loss": 3.7339475153759825, |
|
"tokens_seen": 789708800 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.000362031870626064, |
|
"loss": 2.792, |
|
"theoretical_loss": 3.732412107638, |
|
"tokens_seen": 792985600 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.00036144079818422546, |
|
"loss": 2.8192, |
|
"theoretical_loss": 3.7308847996629724, |
|
"tokens_seen": 796262400 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.000360849725742387, |
|
"loss": 2.7965, |
|
"theoretical_loss": 3.7293655156525043, |
|
"tokens_seen": 799539200 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0003602586533005485, |
|
"loss": 2.8091, |
|
"theoretical_loss": 3.727854180824469, |
|
"tokens_seen": 802816000 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.00035966758085871, |
|
"loss": 2.7943, |
|
"theoretical_loss": 3.7263507213952978, |
|
"tokens_seen": 806092800 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.00035907650841687154, |
|
"loss": 2.7427, |
|
"theoretical_loss": 3.724855064562658, |
|
"tokens_seen": 809369600 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.00035848543597503306, |
|
"loss": 2.807, |
|
"theoretical_loss": 3.723367138488488, |
|
"tokens_seen": 812646400 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.00035790618498203144, |
|
"loss": 2.7934, |
|
"theoretical_loss": 3.7218868722824014, |
|
"tokens_seen": 815923200 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"objective/train/docs_used": 458194, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.9236245155334473, |
|
"objective/train/theoretical_loss": 3.7204141959854384, |
|
"objective/train/tokens_used": 839660000, |
|
"theoretical_loss": 3.7204141959854384, |
|
"tokens_seen": 819200000 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.00035731511254019296, |
|
"loss": 2.8012, |
|
"theoretical_loss": 3.7204141959854384, |
|
"tokens_seen": 819200000 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0003567240400983545, |
|
"loss": 2.7962, |
|
"theoretical_loss": 3.718949040554162, |
|
"tokens_seen": 822476800 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.000356132967656516, |
|
"loss": 2.7889, |
|
"theoretical_loss": 3.7174913378450833, |
|
"tokens_seen": 825753600 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0003555418952146775, |
|
"loss": 2.7913, |
|
"theoretical_loss": 3.7160410205994183, |
|
"tokens_seen": 829030400 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.00035495082277283904, |
|
"loss": 2.7615, |
|
"theoretical_loss": 3.7145980224281585, |
|
"tokens_seen": 832307200 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0003543597503310006, |
|
"loss": 2.7366, |
|
"theoretical_loss": 3.713162277797449, |
|
"tokens_seen": 835584000 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.00035376867788916214, |
|
"loss": 2.7191, |
|
"theoretical_loss": 3.7117337220142748, |
|
"tokens_seen": 838860800 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.00035317760544732366, |
|
"loss": 2.7099, |
|
"theoretical_loss": 3.7103122912124364, |
|
"tokens_seen": 842137600 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0003525865330054852, |
|
"loss": 2.7524, |
|
"theoretical_loss": 3.7088979223388128, |
|
"tokens_seen": 845414400 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.00035199546056364665, |
|
"loss": 2.7272, |
|
"theoretical_loss": 3.70749055313991, |
|
"tokens_seen": 848691200 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.00035140438812180817, |
|
"loss": 2.7231, |
|
"theoretical_loss": 3.7060901221486766, |
|
"tokens_seen": 851968000 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00035081331567996974, |
|
"loss": 2.7398, |
|
"theoretical_loss": 3.704696568671591, |
|
"tokens_seen": 855244800 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00035022224323813127, |
|
"loss": 2.7242, |
|
"theoretical_loss": 3.7033098327760063, |
|
"tokens_seen": 858521600 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0003496311707962928, |
|
"loss": 2.712, |
|
"theoretical_loss": 3.7019298552777533, |
|
"tokens_seen": 861798400 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0003490400983544543, |
|
"loss": 2.6854, |
|
"theoretical_loss": 3.700556577728988, |
|
"tokens_seen": 865075200 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00034844902591261583, |
|
"loss": 2.7172, |
|
"theoretical_loss": 3.6991899424062815, |
|
"tokens_seen": 868352000 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00034785795347077735, |
|
"loss": 2.7286, |
|
"theoretical_loss": 3.697829892298951, |
|
"tokens_seen": 871628800 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0003472668810289389, |
|
"loss": 2.7241, |
|
"theoretical_loss": 3.696476371097618, |
|
"tokens_seen": 874905600 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00034667580858710044, |
|
"loss": 2.721, |
|
"theoretical_loss": 3.695129323182993, |
|
"tokens_seen": 878182400 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00034608473614526196, |
|
"loss": 2.7104, |
|
"theoretical_loss": 3.693788693614879, |
|
"tokens_seen": 881459200 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0003454936637034235, |
|
"loss": 2.701, |
|
"theoretical_loss": 3.6924544281213967, |
|
"tokens_seen": 884736000 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.000344902591261585, |
|
"loss": 2.7402, |
|
"theoretical_loss": 3.691126473088412, |
|
"tokens_seen": 888012800 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.00034431151881974653, |
|
"loss": 2.7404, |
|
"theoretical_loss": 3.689804775549173, |
|
"tokens_seen": 891289600 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.00034372044637790805, |
|
"loss": 2.712, |
|
"theoretical_loss": 3.688489283174146, |
|
"tokens_seen": 894566400 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0003431293739360696, |
|
"loss": 2.7283, |
|
"theoretical_loss": 3.6871799442610538, |
|
"tokens_seen": 897843200 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.00034253830149423114, |
|
"loss": 2.7265, |
|
"theoretical_loss": 3.685876707725093, |
|
"tokens_seen": 901120000 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.00034194722905239266, |
|
"loss": 2.727, |
|
"theoretical_loss": 3.6845795230893517, |
|
"tokens_seen": 904396800 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0003413561566105542, |
|
"loss": 2.6982, |
|
"theoretical_loss": 3.6832883404754035, |
|
"tokens_seen": 907673600 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.0003407650841687157, |
|
"loss": 2.7179, |
|
"theoretical_loss": 3.6820031105940796, |
|
"tokens_seen": 910950400 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.0003401740117268772, |
|
"loss": 2.7446, |
|
"theoretical_loss": 3.6807237847364176, |
|
"tokens_seen": 914227200 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.0003395829392850388, |
|
"loss": 2.7403, |
|
"theoretical_loss": 3.6794503147647846, |
|
"tokens_seen": 917504000 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.0003389918668432003, |
|
"loss": 2.7113, |
|
"theoretical_loss": 3.67818265310416, |
|
"tokens_seen": 920780800 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00033840079440136184, |
|
"loss": 2.7173, |
|
"theoretical_loss": 3.6769207527335888, |
|
"tokens_seen": 924057600 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00033780972195952336, |
|
"loss": 2.7436, |
|
"theoretical_loss": 3.675664567177787, |
|
"tokens_seen": 927334400 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.0003372186495176849, |
|
"loss": 2.7563, |
|
"theoretical_loss": 3.674414050498913, |
|
"tokens_seen": 930611200 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.0003366275770758464, |
|
"loss": 2.7517, |
|
"theoretical_loss": 3.6731691572884824, |
|
"tokens_seen": 933888000 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.000336036504634008, |
|
"loss": 2.729, |
|
"theoretical_loss": 3.671929842659438, |
|
"tokens_seen": 937164800 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0003354454321921695, |
|
"loss": 2.7153, |
|
"theoretical_loss": 3.6706960622383624, |
|
"tokens_seen": 940441600 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.000334854359750331, |
|
"loss": 2.7734, |
|
"theoretical_loss": 3.6694677721578377, |
|
"tokens_seen": 943718400 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.00033426328730849254, |
|
"loss": 2.7185, |
|
"theoretical_loss": 3.66824492904894, |
|
"tokens_seen": 946995200 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.00033367221486665406, |
|
"loss": 2.716, |
|
"theoretical_loss": 3.667027490033874, |
|
"tokens_seen": 950272000 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0003330811424248156, |
|
"loss": 2.6816, |
|
"theoretical_loss": 3.6658154127187412, |
|
"tokens_seen": 953548800 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.00033250189143181386, |
|
"loss": 2.6909, |
|
"theoretical_loss": 3.664608655186437, |
|
"tokens_seen": 956825600 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.00033191081898997543, |
|
"loss": 2.6883, |
|
"theoretical_loss": 3.663407175989679, |
|
"tokens_seen": 960102400 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.00033131974654813695, |
|
"loss": 2.6588, |
|
"theoretical_loss": 3.662210934144158, |
|
"tokens_seen": 963379200 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0003307286741062985, |
|
"loss": 2.6515, |
|
"theoretical_loss": 3.661019889121812, |
|
"tokens_seen": 966656000 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00033013760166446, |
|
"loss": 2.667, |
|
"theoretical_loss": 3.6598340008442234, |
|
"tokens_seen": 969932800 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0003295465292226215, |
|
"loss": 2.7195, |
|
"theoretical_loss": 3.6586532296761285, |
|
"tokens_seen": 973209600 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00032895545678078304, |
|
"loss": 2.6838, |
|
"theoretical_loss": 3.657477536419047, |
|
"tokens_seen": 976486400 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0003283643843389446, |
|
"loss": 2.644, |
|
"theoretical_loss": 3.656306882305022, |
|
"tokens_seen": 979763200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.36836749701066873, |
|
"debugging/distinct-1-grams": 0.8145084309773442, |
|
"debugging/distinct-2-grams": 0.9777845741796146, |
|
"debugging/entropy-1-grams": 5.565961728673207, |
|
"debugging/entropy-2-grams": 6.204744460068355, |
|
"debugging/length": 558.2857142857143, |
|
"debugging/num_segments": 7, |
|
"epoch": 0.35, |
|
"objective/train/docs_used": 548756, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.5849809646606445, |
|
"objective/train/theoretical_loss": 3.6551412289904697, |
|
"objective/train/tokens_used": 1003500000, |
|
"theoretical_loss": 3.6551412289904697, |
|
"tokens_seen": 983040000 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00032777331189710613, |
|
"loss": 2.6889, |
|
"theoretical_loss": 3.6551412289904697, |
|
"tokens_seen": 983040000 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00032718223945526765, |
|
"loss": 2.6755, |
|
"theoretical_loss": 3.6539805385501376, |
|
"tokens_seen": 986316800 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0003265911670134292, |
|
"loss": 2.6964, |
|
"theoretical_loss": 3.652824773471171, |
|
"tokens_seen": 989593600 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0003260000945715907, |
|
"loss": 2.7238, |
|
"theoretical_loss": 3.651673896647277, |
|
"tokens_seen": 992870400 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0003254090221297522, |
|
"loss": 2.7876, |
|
"theoretical_loss": 3.6505278713729985, |
|
"tokens_seen": 996147200 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0003248179496879138, |
|
"loss": 2.8354, |
|
"theoretical_loss": 3.6493866613380774, |
|
"tokens_seen": 999424000 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0003242268772460753, |
|
"loss": 2.8495, |
|
"theoretical_loss": 3.648250230621924, |
|
"tokens_seen": 1002700800 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.00032363580480423683, |
|
"loss": 2.7892, |
|
"theoretical_loss": 3.647118543688179, |
|
"tokens_seen": 1005977600 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.00032304473236239835, |
|
"loss": 2.8044, |
|
"theoretical_loss": 3.6459915653793633, |
|
"tokens_seen": 1009254400 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.00032245365992055987, |
|
"loss": 2.7869, |
|
"theoretical_loss": 3.644869260911628, |
|
"tokens_seen": 1012531200 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0003218625874787214, |
|
"loss": 2.8025, |
|
"theoretical_loss": 3.64375159586959, |
|
"tokens_seen": 1015808000 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0003212715150368829, |
|
"loss": 2.7588, |
|
"theoretical_loss": 3.642638536201252, |
|
"tokens_seen": 1019084800 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0003206804425950445, |
|
"loss": 2.761, |
|
"theoretical_loss": 3.6415300482130135, |
|
"tokens_seen": 1022361600 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.000320089370153206, |
|
"loss": 2.7571, |
|
"theoretical_loss": 3.6404260985647667, |
|
"tokens_seen": 1025638400 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00031949829771136753, |
|
"loss": 2.7637, |
|
"theoretical_loss": 3.6393266542650684, |
|
"tokens_seen": 1028915200 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00031890722526952905, |
|
"loss": 2.7481, |
|
"theoretical_loss": 3.638231682666401, |
|
"tokens_seen": 1032192000 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00031831615282769057, |
|
"loss": 2.7808, |
|
"theoretical_loss": 3.637141151460505, |
|
"tokens_seen": 1035468800 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0003177250803858521, |
|
"loss": 2.7641, |
|
"theoretical_loss": 3.636055028673799, |
|
"tokens_seen": 1038745600 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00031713400794401367, |
|
"loss": 2.7917, |
|
"theoretical_loss": 3.634973282662864, |
|
"tokens_seen": 1042022400 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0003165429355021752, |
|
"loss": 2.7501, |
|
"theoretical_loss": 3.6338958821100107, |
|
"tokens_seen": 1045299200 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0003159518630603367, |
|
"loss": 2.7534, |
|
"theoretical_loss": 3.63282279601892, |
|
"tokens_seen": 1048576000 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.00031536079061849823, |
|
"loss": 2.7475, |
|
"theoretical_loss": 3.631753993710352, |
|
"tokens_seen": 1051852800 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.00031476971817665975, |
|
"loss": 2.7884, |
|
"theoretical_loss": 3.630689444817925, |
|
"tokens_seen": 1055129600 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0003141786457348212, |
|
"loss": 2.7667, |
|
"theoretical_loss": 3.629629119283967, |
|
"tokens_seen": 1058406400 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0003135875732929828, |
|
"loss": 2.7601, |
|
"theoretical_loss": 3.628572987355434, |
|
"tokens_seen": 1061683200 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0003129965008511443, |
|
"loss": 2.7147, |
|
"theoretical_loss": 3.6275210195798913, |
|
"tokens_seen": 1064960000 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.00031240542840930583, |
|
"loss": 2.7162, |
|
"theoretical_loss": 3.626473186801564, |
|
"tokens_seen": 1068236800 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.00031181435596746736, |
|
"loss": 2.6936, |
|
"theoretical_loss": 3.6254294601574495, |
|
"tokens_seen": 1071513600 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0003112232835256289, |
|
"loss": 2.6913, |
|
"theoretical_loss": 3.624389811073493, |
|
"tokens_seen": 1074790400 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0003106322110837904, |
|
"loss": 2.6906, |
|
"theoretical_loss": 3.6233542112608257, |
|
"tokens_seen": 1078067200 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0003100411386419519, |
|
"loss": 2.7232, |
|
"theoretical_loss": 3.6223226327120592, |
|
"tokens_seen": 1081344000 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0003094500662001135, |
|
"loss": 2.6992, |
|
"theoretical_loss": 3.621295047697644, |
|
"tokens_seen": 1084620800 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.000308858993758275, |
|
"loss": 2.6927, |
|
"theoretical_loss": 3.6202714287622833, |
|
"tokens_seen": 1087897600 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.00030826792131643653, |
|
"loss": 2.7535, |
|
"theoretical_loss": 3.6192517487214038, |
|
"tokens_seen": 1091174400 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.00030767684887459806, |
|
"loss": 2.7177, |
|
"theoretical_loss": 3.6182359806576834, |
|
"tokens_seen": 1094451200 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0003070857764327596, |
|
"loss": 2.7207, |
|
"theoretical_loss": 3.6172240979176333, |
|
"tokens_seen": 1097728000 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0003064947039909211, |
|
"loss": 2.701, |
|
"theoretical_loss": 3.616216074108232, |
|
"tokens_seen": 1101004800 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.00030590363154908267, |
|
"loss": 2.6874, |
|
"theoretical_loss": 3.6152118830936164, |
|
"tokens_seen": 1104281600 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0003053125591072442, |
|
"loss": 2.7029, |
|
"theoretical_loss": 3.6142114989918195, |
|
"tokens_seen": 1107558400 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0003047214866654057, |
|
"loss": 2.6927, |
|
"theoretical_loss": 3.6132148961715624, |
|
"tokens_seen": 1110835200 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00030413041422356723, |
|
"loss": 2.7024, |
|
"theoretical_loss": 3.6122220492490964, |
|
"tokens_seen": 1114112000 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00030353934178172875, |
|
"loss": 2.7201, |
|
"theoretical_loss": 3.6112329330850894, |
|
"tokens_seen": 1117388800 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0003029482693398903, |
|
"loss": 2.6931, |
|
"theoretical_loss": 3.61024752278157, |
|
"tokens_seen": 1120665600 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00030235719689805185, |
|
"loss": 2.6943, |
|
"theoretical_loss": 3.6092657936789054, |
|
"tokens_seen": 1123942400 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00030176612445621337, |
|
"loss": 2.667, |
|
"theoretical_loss": 3.6082877213528377, |
|
"tokens_seen": 1127219200 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0003011750520143749, |
|
"loss": 2.6584, |
|
"theoretical_loss": 3.60731328161156, |
|
"tokens_seen": 1130496000 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0003005839795725364, |
|
"loss": 2.6984, |
|
"theoretical_loss": 3.6063424504928365, |
|
"tokens_seen": 1133772800 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00029999290713069793, |
|
"loss": 2.6699, |
|
"theoretical_loss": 3.60537520426117, |
|
"tokens_seen": 1137049600 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00029940183468885945, |
|
"loss": 2.6932, |
|
"theoretical_loss": 3.6044115194050086, |
|
"tokens_seen": 1140326400 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00029881076224702103, |
|
"loss": 2.685, |
|
"theoretical_loss": 3.603451372633997, |
|
"tokens_seen": 1143603200 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"objective/train/docs_used": 634175, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.4585118293762207, |
|
"objective/train/theoretical_loss": 3.6024947408762698, |
|
"objective/train/tokens_used": 1167340000, |
|
"theoretical_loss": 3.6024947408762698, |
|
"tokens_seen": 1146880000 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00029821968980518255, |
|
"loss": 2.6539, |
|
"theoretical_loss": 3.6024947408762698, |
|
"tokens_seen": 1146880000 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00029762861736334407, |
|
"loss": 2.756, |
|
"theoretical_loss": 3.601541601275783, |
|
"tokens_seen": 1150156800 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.0002970375449215056, |
|
"loss": 2.7502, |
|
"theoretical_loss": 3.6005919311896886, |
|
"tokens_seen": 1153433600 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.0002964464724796671, |
|
"loss": 2.7149, |
|
"theoretical_loss": 3.5996457081857454, |
|
"tokens_seen": 1156710400 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00029585540003782863, |
|
"loss": 2.7312, |
|
"theoretical_loss": 3.598702910039772, |
|
"tokens_seen": 1159987200 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00029526432759599015, |
|
"loss": 2.7239, |
|
"theoretical_loss": 3.597763514733133, |
|
"tokens_seen": 1163264000 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00029467325515415173, |
|
"loss": 2.7269, |
|
"theoretical_loss": 3.59682750045027, |
|
"tokens_seen": 1166540800 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00029408218271231325, |
|
"loss": 2.685, |
|
"theoretical_loss": 3.5958948455762583, |
|
"tokens_seen": 1169817600 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00029349111027047477, |
|
"loss": 2.699, |
|
"theoretical_loss": 3.594965528694412, |
|
"tokens_seen": 1173094400 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0002929000378286363, |
|
"loss": 2.6904, |
|
"theoretical_loss": 3.594039528583913, |
|
"tokens_seen": 1176371200 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0002923089653867978, |
|
"loss": 2.7125, |
|
"theoretical_loss": 3.5931168242174847, |
|
"tokens_seen": 1179648000 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00029171789294495933, |
|
"loss": 2.6923, |
|
"theoretical_loss": 3.59219739475909, |
|
"tokens_seen": 1182924800 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0002911268205031209, |
|
"loss": 2.711, |
|
"theoretical_loss": 3.5912812195616732, |
|
"tokens_seen": 1186201600 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00029053574806128243, |
|
"loss": 2.7153, |
|
"theoretical_loss": 3.590368278164926, |
|
"tokens_seen": 1189478400 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.00028994467561944395, |
|
"loss": 2.6884, |
|
"theoretical_loss": 3.5894585502930902, |
|
"tokens_seen": 1192755200 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.00028935360317760547, |
|
"loss": 2.7023, |
|
"theoretical_loss": 3.588552015852793, |
|
"tokens_seen": 1196032000 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.000288762530735767, |
|
"loss": 2.6901, |
|
"theoretical_loss": 3.5876486549309097, |
|
"tokens_seen": 1199308800 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.0002881714582939285, |
|
"loss": 2.7197, |
|
"theoretical_loss": 3.586748447792462, |
|
"tokens_seen": 1202585600 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.0002875803858520901, |
|
"loss": 2.6774, |
|
"theoretical_loss": 3.5858513748785423, |
|
"tokens_seen": 1205862400 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.0002869893134102516, |
|
"loss": 2.6985, |
|
"theoretical_loss": 3.5849574168042704, |
|
"tokens_seen": 1209139200 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.00028639824096841313, |
|
"loss": 2.7111, |
|
"theoretical_loss": 3.5840665543567782, |
|
"tokens_seen": 1212416000 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.00028580716852657465, |
|
"loss": 2.7243, |
|
"theoretical_loss": 3.583178768493222, |
|
"tokens_seen": 1215692800 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0002852160960847361, |
|
"loss": 2.7112, |
|
"theoretical_loss": 3.5822940403388284, |
|
"tokens_seen": 1218969600 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00028462502364289764, |
|
"loss": 2.7068, |
|
"theoretical_loss": 3.581412351184958, |
|
"tokens_seen": 1222246400 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00028403395120105916, |
|
"loss": 2.7265, |
|
"theoretical_loss": 3.580533682487208, |
|
"tokens_seen": 1225523200 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00028344287875922073, |
|
"loss": 2.7063, |
|
"theoretical_loss": 3.579658015863532, |
|
"tokens_seen": 1228800000 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00028285180631738225, |
|
"loss": 2.6991, |
|
"theoretical_loss": 3.5787853330923927, |
|
"tokens_seen": 1232076800 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0002822607338755438, |
|
"loss": 2.7184, |
|
"theoretical_loss": 3.577915616110936, |
|
"tokens_seen": 1235353600 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0002816696614337053, |
|
"loss": 2.6728, |
|
"theoretical_loss": 3.577048847013194, |
|
"tokens_seen": 1238630400 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0002810785889918668, |
|
"loss": 2.6903, |
|
"theoretical_loss": 3.57618500804831, |
|
"tokens_seen": 1241907200 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00028048751655002834, |
|
"loss": 2.764, |
|
"theoretical_loss": 3.575324081618793, |
|
"tokens_seen": 1245184000 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.0002798964441081899, |
|
"loss": 2.7676, |
|
"theoretical_loss": 3.5744660502787875, |
|
"tokens_seen": 1248460800 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00027930537166635143, |
|
"loss": 2.7343, |
|
"theoretical_loss": 3.5736108967323794, |
|
"tokens_seen": 1251737600 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00027871429922451295, |
|
"loss": 2.6927, |
|
"theoretical_loss": 3.5727586038319155, |
|
"tokens_seen": 1255014400 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.0002781232267826745, |
|
"loss": 2.7064, |
|
"theoretical_loss": 3.571909154576348, |
|
"tokens_seen": 1258291200 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.000277532154340836, |
|
"loss": 2.6997, |
|
"theoretical_loss": 3.5710625321096074, |
|
"tokens_seen": 1261568000 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.0002769410818989975, |
|
"loss": 2.71, |
|
"theoretical_loss": 3.570218719718989, |
|
"tokens_seen": 1264844800 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.0002763500094571591, |
|
"loss": 2.6884, |
|
"theoretical_loss": 3.569377700833569, |
|
"tokens_seen": 1268121600 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.0002757589370153206, |
|
"loss": 2.6788, |
|
"theoretical_loss": 3.568539459022639, |
|
"tokens_seen": 1271398400 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00027516786457348213, |
|
"loss": 2.659, |
|
"theoretical_loss": 3.5677039779941584, |
|
"tokens_seen": 1274675200 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00027457679213164365, |
|
"loss": 2.6688, |
|
"theoretical_loss": 3.566871241593236, |
|
"tokens_seen": 1277952000 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0002739857196898052, |
|
"loss": 2.6803, |
|
"theoretical_loss": 3.5660412338006235, |
|
"tokens_seen": 1281228800 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0002733946472479667, |
|
"loss": 2.6511, |
|
"theoretical_loss": 3.565213938731236, |
|
"tokens_seen": 1284505600 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0002728035748061282, |
|
"loss": 2.6606, |
|
"theoretical_loss": 3.5643893406326868, |
|
"tokens_seen": 1287782400 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0002722125023642898, |
|
"loss": 2.6488, |
|
"theoretical_loss": 3.5635674238838466, |
|
"tokens_seen": 1291059200 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0002716214299224513, |
|
"loss": 2.6633, |
|
"theoretical_loss": 3.5627481729934196, |
|
"tokens_seen": 1294336000 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00027103035748061283, |
|
"loss": 2.7017, |
|
"theoretical_loss": 3.561931572598538, |
|
"tokens_seen": 1297612800 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00027043928503877435, |
|
"loss": 2.6935, |
|
"theoretical_loss": 3.5611176074633777, |
|
"tokens_seen": 1300889600 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.0002698482125969359, |
|
"loss": 2.6385, |
|
"theoretical_loss": 3.5603062624777895, |
|
"tokens_seen": 1304166400 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.0002692571401550974, |
|
"loss": 2.6251, |
|
"theoretical_loss": 3.559497522655951, |
|
"tokens_seen": 1307443200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.5495772280672173, |
|
"debugging/distinct-1-grams": 0.7717677537029302, |
|
"debugging/distinct-2-grams": 0.9421298772004165, |
|
"debugging/entropy-1-grams": 6.12011243732356, |
|
"debugging/entropy-2-grams": 7.053666959871009, |
|
"debugging/length": 531.0, |
|
"debugging/num_segments": 18, |
|
"epoch": 0.47, |
|
"objective/train/docs_used": 722422, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.7103114128112793, |
|
"objective/train/theoretical_loss": 3.5586913731350327, |
|
"objective/train/tokens_used": 1331180000, |
|
"theoretical_loss": 3.5586913731350327, |
|
"tokens_seen": 1310720000 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00026866606771325897, |
|
"loss": 2.598, |
|
"theoretical_loss": 3.5586913731350327, |
|
"tokens_seen": 1310720000 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.0002680749952714205, |
|
"loss": 2.6352, |
|
"theoretical_loss": 3.557887799173889, |
|
"tokens_seen": 1313996800 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.000267483922829582, |
|
"loss": 2.6513, |
|
"theoretical_loss": 3.557086786151754, |
|
"tokens_seen": 1317273600 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00026689285038774353, |
|
"loss": 2.6301, |
|
"theoretical_loss": 3.5562883195669697, |
|
"tokens_seen": 1320550400 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00026630177794590505, |
|
"loss": 2.6476, |
|
"theoretical_loss": 3.555492385035719, |
|
"tokens_seen": 1323827200 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.0002657107055040666, |
|
"loss": 2.6642, |
|
"theoretical_loss": 3.5546989682907784, |
|
"tokens_seen": 1327104000 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026511963306222815, |
|
"loss": 2.6362, |
|
"theoretical_loss": 3.5539080551802895, |
|
"tokens_seen": 1330380800 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026452856062038967, |
|
"loss": 2.6501, |
|
"theoretical_loss": 3.553119631666546, |
|
"tokens_seen": 1333657600 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.0002639374881785512, |
|
"loss": 2.6677, |
|
"theoretical_loss": 3.5523336838247914, |
|
"tokens_seen": 1336934400 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.0002633464157367127, |
|
"loss": 2.6693, |
|
"theoretical_loss": 3.55155019784204, |
|
"tokens_seen": 1340211200 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026275534329487423, |
|
"loss": 2.6824, |
|
"theoretical_loss": 3.5507691600159053, |
|
"tokens_seen": 1343488000 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026216427085303575, |
|
"loss": 2.6875, |
|
"theoretical_loss": 3.5499905567534515, |
|
"tokens_seen": 1346764800 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.0002615731984111973, |
|
"loss": 2.6603, |
|
"theoretical_loss": 3.549214374570052, |
|
"tokens_seen": 1350041600 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.0002609939474181956, |
|
"loss": 2.6334, |
|
"theoretical_loss": 3.5484406000882665, |
|
"tokens_seen": 1353318400 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.0002604028749763571, |
|
"loss": 2.6658, |
|
"theoretical_loss": 3.5476692200367346, |
|
"tokens_seen": 1356595200 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00025981180253451864, |
|
"loss": 2.6842, |
|
"theoretical_loss": 3.546900221249076, |
|
"tokens_seen": 1359872000 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00025922073009268016, |
|
"loss": 2.6731, |
|
"theoretical_loss": 3.5461335906628157, |
|
"tokens_seen": 1363148800 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.0002586296576508417, |
|
"loss": 2.6841, |
|
"theoretical_loss": 3.54536931531831, |
|
"tokens_seen": 1366425600 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.0002580385852090032, |
|
"loss": 2.6728, |
|
"theoretical_loss": 3.5446073823576985, |
|
"tokens_seen": 1369702400 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.0002574475127671648, |
|
"loss": 2.6405, |
|
"theoretical_loss": 3.543847779023859, |
|
"tokens_seen": 1372979200 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.0002568564403253263, |
|
"loss": 2.6784, |
|
"theoretical_loss": 3.543090492659384, |
|
"tokens_seen": 1376256000 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.0002562653678834878, |
|
"loss": 2.6239, |
|
"theoretical_loss": 3.542335510705562, |
|
"tokens_seen": 1379532800 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00025567429544164934, |
|
"loss": 2.624, |
|
"theoretical_loss": 3.541582820701378, |
|
"tokens_seen": 1382809600 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.00025508322299981086, |
|
"loss": 2.6031, |
|
"theoretical_loss": 3.5408324102825253, |
|
"tokens_seen": 1386086400 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.0002544921505579724, |
|
"loss": 2.6351, |
|
"theoretical_loss": 3.5400842671804265, |
|
"tokens_seen": 1389363200 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.00025390107811613396, |
|
"loss": 2.6747, |
|
"theoretical_loss": 3.5393383792212676, |
|
"tokens_seen": 1392640000 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.0002533100056742955, |
|
"loss": 2.6658, |
|
"theoretical_loss": 3.5385947343250486, |
|
"tokens_seen": 1395916800 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.000252718933232457, |
|
"loss": 2.6749, |
|
"theoretical_loss": 3.53785332050464, |
|
"tokens_seen": 1399193600 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.0002521278607906185, |
|
"loss": 2.6646, |
|
"theoretical_loss": 3.537114125864853, |
|
"tokens_seen": 1402470400 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.0002515486097976168, |
|
"loss": 2.6794, |
|
"theoretical_loss": 3.536377138601523, |
|
"tokens_seen": 1405747200 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.0002509575373557783, |
|
"loss": 2.6845, |
|
"theoretical_loss": 3.5356423470006018, |
|
"tokens_seen": 1409024000 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.0002503664649139399, |
|
"loss": 2.6747, |
|
"theoretical_loss": 3.5349097394372615, |
|
"tokens_seen": 1412300800 |
|
}, |
|
{ |
|
"epoch": 0.51, |
|
"learning_rate": 0.00024977539247210135, |
|
"loss": 2.6765, |
|
"theoretical_loss": 3.5341793043750114, |
|
"tokens_seen": 1415577600 |
|
}, |
|
{ |
|
"epoch": 0.51, |
|
"learning_rate": 0.00024918432003026293, |
|
"loss": 2.6938, |
|
"theoretical_loss": 3.533451030364823, |
|
"tokens_seen": 1418854400 |
|
}, |
|
{ |
|
"epoch": 0.51, |
|
"learning_rate": 0.00024859324758842445, |
|
"loss": 2.6275, |
|
"theoretical_loss": 3.5327249060442676, |
|
"tokens_seen": 1422131200 |
|
}, |
|
{ |
|
"epoch": 0.51, |
|
"learning_rate": 0.00024800217514658597, |
|
"loss": 2.6179, |
|
"theoretical_loss": 3.5320009201366616, |
|
"tokens_seen": 1425408000 |
|
}, |
|
{ |
|
"epoch": 0.51, |
|
"learning_rate": 0.0002474111027047475, |
|
"loss": 2.6515, |
|
"theoretical_loss": 3.531279061450226, |
|
"tokens_seen": 1428684800 |
|
}, |
|
{ |
|
"epoch": 0.51, |
|
"learning_rate": 0.000246820030262909, |
|
"loss": 2.6765, |
|
"theoretical_loss": 3.5305593188772533, |
|
"tokens_seen": 1431961600 |
|
}, |
|
{ |
|
"epoch": 0.51, |
|
"learning_rate": 0.00024622895782107053, |
|
"loss": 2.6617, |
|
"theoretical_loss": 3.5298416813932865, |
|
"tokens_seen": 1435238400 |
|
}, |
|
{ |
|
"epoch": 0.51, |
|
"learning_rate": 0.0002456378853792321, |
|
"loss": 2.658, |
|
"theoretical_loss": 3.529126138056305, |
|
"tokens_seen": 1438515200 |
|
}, |
|
{ |
|
"epoch": 0.51, |
|
"learning_rate": 0.00024504681293739363, |
|
"loss": 2.6307, |
|
"theoretical_loss": 3.528412678005922, |
|
"tokens_seen": 1441792000 |
|
}, |
|
{ |
|
"epoch": 0.52, |
|
"learning_rate": 0.00024445574049555515, |
|
"loss": 2.6269, |
|
"theoretical_loss": 3.527701290462595, |
|
"tokens_seen": 1445068800 |
|
}, |
|
{ |
|
"epoch": 0.52, |
|
"learning_rate": 0.00024386466805371667, |
|
"loss": 2.5892, |
|
"theoretical_loss": 3.5269919647268377, |
|
"tokens_seen": 1448345600 |
|
}, |
|
{ |
|
"epoch": 0.52, |
|
"learning_rate": 0.0002432735956118782, |
|
"loss": 2.6232, |
|
"theoretical_loss": 3.526284690178448, |
|
"tokens_seen": 1451622400 |
|
}, |
|
{ |
|
"epoch": 0.52, |
|
"learning_rate": 0.0002426825231700397, |
|
"loss": 2.6585, |
|
"theoretical_loss": 3.5255794562757443, |
|
"tokens_seen": 1454899200 |
|
}, |
|
{ |
|
"epoch": 0.52, |
|
"learning_rate": 0.00024209145072820123, |
|
"loss": 2.625, |
|
"theoretical_loss": 3.524876252554808, |
|
"tokens_seen": 1458176000 |
|
}, |
|
{ |
|
"epoch": 0.52, |
|
"learning_rate": 0.00024150037828636278, |
|
"loss": 2.6278, |
|
"theoretical_loss": 3.524175068628737, |
|
"tokens_seen": 1461452800 |
|
}, |
|
{ |
|
"epoch": 0.52, |
|
"learning_rate": 0.0002409093058445243, |
|
"loss": 2.6124, |
|
"theoretical_loss": 3.5234758941869075, |
|
"tokens_seen": 1464729600 |
|
}, |
|
{ |
|
"epoch": 0.52, |
|
"learning_rate": 0.00024031823340268582, |
|
"loss": 2.6236, |
|
"theoretical_loss": 3.522778718994246, |
|
"tokens_seen": 1468006400 |
|
}, |
|
{ |
|
"epoch": 0.53, |
|
"learning_rate": 0.00023972716096084737, |
|
"loss": 2.5968, |
|
"theoretical_loss": 3.5220835328905062, |
|
"tokens_seen": 1471283200 |
|
}, |
|
{ |
|
"epoch": 0.53, |
|
"objective/train/docs_used": 814205, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 1.9243988990783691, |
|
"objective/train/theoretical_loss": 3.521390325789559, |
|
"objective/train/tokens_used": 1495020000, |
|
"theoretical_loss": 3.521390325789559, |
|
"tokens_seen": 1474560000 |
|
}, |
|
{ |
|
"epoch": 0.53, |
|
"learning_rate": 0.0002391360885190089, |
|
"loss": 2.6074, |
|
"theoretical_loss": 3.521390325789559, |
|
"tokens_seen": 1474560000 |
|
}, |
|
{ |
|
"epoch": 0.53, |
|
"learning_rate": 0.0002385450160771704, |
|
"loss": 2.5987, |
|
"theoretical_loss": 3.520699087678688, |
|
"tokens_seen": 1477836800 |
|
}, |
|
{ |
|
"epoch": 0.53, |
|
"learning_rate": 0.00023795394363533196, |
|
"loss": 2.5673, |
|
"theoretical_loss": 3.5200098086178926, |
|
"tokens_seen": 1481113600 |
|
}, |
|
{ |
|
"epoch": 0.53, |
|
"learning_rate": 0.00023736287119349348, |
|
"loss": 2.5868, |
|
"theoretical_loss": 3.5193224787392015, |
|
"tokens_seen": 1484390400 |
|
}, |
|
{ |
|
"epoch": 0.53, |
|
"learning_rate": 0.000236771798751655, |
|
"loss": 2.6053, |
|
"theoretical_loss": 3.518637088245993, |
|
"tokens_seen": 1487667200 |
|
}, |
|
{ |
|
"epoch": 0.53, |
|
"learning_rate": 0.00023618072630981655, |
|
"loss": 2.5797, |
|
"theoretical_loss": 3.5179536274123233, |
|
"tokens_seen": 1490944000 |
|
}, |
|
{ |
|
"epoch": 0.53, |
|
"learning_rate": 0.00023558965386797807, |
|
"loss": 2.5724, |
|
"theoretical_loss": 3.5172720865822633, |
|
"tokens_seen": 1494220800 |
|
}, |
|
{ |
|
"epoch": 0.53, |
|
"learning_rate": 0.0002349985814261396, |
|
"loss": 2.5803, |
|
"theoretical_loss": 3.5165924561692394, |
|
"tokens_seen": 1497497600 |
|
}, |
|
{ |
|
"epoch": 0.54, |
|
"learning_rate": 0.0002344193304331379, |
|
"loss": 2.6168, |
|
"theoretical_loss": 3.5159147266553887, |
|
"tokens_seen": 1500774400 |
|
}, |
|
{ |
|
"epoch": 0.54, |
|
"learning_rate": 0.0002338282579912994, |
|
"loss": 2.5949, |
|
"theoretical_loss": 3.515238888590916, |
|
"tokens_seen": 1504051200 |
|
}, |
|
{ |
|
"epoch": 0.54, |
|
"learning_rate": 0.00023323718554946093, |
|
"loss": 2.6213, |
|
"theoretical_loss": 3.51456493259346, |
|
"tokens_seen": 1507328000 |
|
}, |
|
{ |
|
"epoch": 0.54, |
|
"learning_rate": 0.00023264611310762248, |
|
"loss": 2.6106, |
|
"theoretical_loss": 3.5138928493474664, |
|
"tokens_seen": 1510604800 |
|
}, |
|
{ |
|
"epoch": 0.54, |
|
"learning_rate": 0.000232055040665784, |
|
"loss": 2.62, |
|
"theoretical_loss": 3.513222629603569, |
|
"tokens_seen": 1513881600 |
|
}, |
|
{ |
|
"epoch": 0.54, |
|
"learning_rate": 0.00023146396822394552, |
|
"loss": 2.6522, |
|
"theoretical_loss": 3.512554264177979, |
|
"tokens_seen": 1517158400 |
|
}, |
|
{ |
|
"epoch": 0.54, |
|
"learning_rate": 0.00023087289578210707, |
|
"loss": 2.647, |
|
"theoretical_loss": 3.511887743951874, |
|
"tokens_seen": 1520435200 |
|
}, |
|
{ |
|
"epoch": 0.54, |
|
"learning_rate": 0.0002302818233402686, |
|
"loss": 2.6618, |
|
"theoretical_loss": 3.511223059870809, |
|
"tokens_seen": 1523712000 |
|
}, |
|
{ |
|
"epoch": 0.55, |
|
"learning_rate": 0.0002296907508984301, |
|
"loss": 2.6569, |
|
"theoretical_loss": 3.5105602029441143, |
|
"tokens_seen": 1526988800 |
|
}, |
|
{ |
|
"epoch": 0.55, |
|
"learning_rate": 0.00022909967845659163, |
|
"loss": 2.6362, |
|
"theoretical_loss": 3.5098991642443185, |
|
"tokens_seen": 1530265600 |
|
}, |
|
{ |
|
"epoch": 0.55, |
|
"learning_rate": 0.00022850860601475318, |
|
"loss": 2.6692, |
|
"theoretical_loss": 3.5092399349065655, |
|
"tokens_seen": 1533542400 |
|
}, |
|
{ |
|
"epoch": 0.55, |
|
"learning_rate": 0.0002279175335729147, |
|
"loss": 2.6821, |
|
"theoretical_loss": 3.5085825061280445, |
|
"tokens_seen": 1536819200 |
|
}, |
|
{ |
|
"epoch": 0.55, |
|
"learning_rate": 0.00022732646113107622, |
|
"loss": 2.6715, |
|
"theoretical_loss": 3.507926869167425, |
|
"tokens_seen": 1540096000 |
|
}, |
|
{ |
|
"epoch": 0.55, |
|
"learning_rate": 0.00022673538868923777, |
|
"loss": 2.6172, |
|
"theoretical_loss": 3.507273015344296, |
|
"tokens_seen": 1543372800 |
|
}, |
|
{ |
|
"epoch": 0.55, |
|
"learning_rate": 0.0002261443162473993, |
|
"loss": 2.6211, |
|
"theoretical_loss": 3.5066209360386154, |
|
"tokens_seen": 1546649600 |
|
}, |
|
{ |
|
"epoch": 0.55, |
|
"learning_rate": 0.0002255532438055608, |
|
"loss": 2.6592, |
|
"theoretical_loss": 3.5059706226901595, |
|
"tokens_seen": 1549926400 |
|
}, |
|
{ |
|
"epoch": 0.55, |
|
"learning_rate": 0.00022496217136372236, |
|
"loss": 2.6639, |
|
"theoretical_loss": 3.505322066797988, |
|
"tokens_seen": 1553203200 |
|
}, |
|
{ |
|
"epoch": 0.56, |
|
"learning_rate": 0.00022437109892188388, |
|
"loss": 2.6234, |
|
"theoretical_loss": 3.504675259919904, |
|
"tokens_seen": 1556480000 |
|
}, |
|
{ |
|
"epoch": 0.56, |
|
"learning_rate": 0.0002237800264800454, |
|
"loss": 2.6441, |
|
"theoretical_loss": 3.5040301936719302, |
|
"tokens_seen": 1559756800 |
|
}, |
|
{ |
|
"epoch": 0.56, |
|
"learning_rate": 0.00022318895403820695, |
|
"loss": 2.6508, |
|
"theoretical_loss": 3.5033868597277813, |
|
"tokens_seen": 1563033600 |
|
}, |
|
{ |
|
"epoch": 0.56, |
|
"learning_rate": 0.00022259788159636847, |
|
"loss": 2.6046, |
|
"theoretical_loss": 3.502745249818351, |
|
"tokens_seen": 1566310400 |
|
}, |
|
{ |
|
"epoch": 0.56, |
|
"learning_rate": 0.00022200680915453, |
|
"loss": 2.6174, |
|
"theoretical_loss": 3.5021053557311985, |
|
"tokens_seen": 1569587200 |
|
}, |
|
{ |
|
"epoch": 0.56, |
|
"learning_rate": 0.00022141573671269154, |
|
"loss": 2.5895, |
|
"theoretical_loss": 3.501467169310044, |
|
"tokens_seen": 1572864000 |
|
}, |
|
{ |
|
"epoch": 0.56, |
|
"learning_rate": 0.00022082466427085303, |
|
"loss": 2.5947, |
|
"theoretical_loss": 3.500830682454268, |
|
"tokens_seen": 1576140800 |
|
}, |
|
{ |
|
"epoch": 0.56, |
|
"learning_rate": 0.00022023359182901455, |
|
"loss": 2.6105, |
|
"theoretical_loss": 3.5001958871184167, |
|
"tokens_seen": 1579417600 |
|
}, |
|
{ |
|
"epoch": 0.57, |
|
"learning_rate": 0.0002196425193871761, |
|
"loss": 2.6151, |
|
"theoretical_loss": 3.4995627753117127, |
|
"tokens_seen": 1582694400 |
|
}, |
|
{ |
|
"epoch": 0.57, |
|
"learning_rate": 0.00021905144694533762, |
|
"loss": 2.6236, |
|
"theoretical_loss": 3.498931339097571, |
|
"tokens_seen": 1585971200 |
|
}, |
|
{ |
|
"epoch": 0.57, |
|
"learning_rate": 0.00021846037450349914, |
|
"loss": 2.6656, |
|
"theoretical_loss": 3.4983015705931226, |
|
"tokens_seen": 1589248000 |
|
}, |
|
{ |
|
"epoch": 0.57, |
|
"learning_rate": 0.00021786930206166066, |
|
"loss": 2.6424, |
|
"theoretical_loss": 3.4976734619687377, |
|
"tokens_seen": 1592524800 |
|
}, |
|
{ |
|
"epoch": 0.57, |
|
"learning_rate": 0.0002172782296198222, |
|
"loss": 2.6222, |
|
"theoretical_loss": 3.497047005447558, |
|
"tokens_seen": 1595801600 |
|
}, |
|
{ |
|
"epoch": 0.57, |
|
"learning_rate": 0.00021668715717798373, |
|
"loss": 2.6319, |
|
"theoretical_loss": 3.4964221933050377, |
|
"tokens_seen": 1599078400 |
|
}, |
|
{ |
|
"epoch": 0.57, |
|
"learning_rate": 0.00021609608473614525, |
|
"loss": 2.5903, |
|
"theoretical_loss": 3.495799017868479, |
|
"tokens_seen": 1602355200 |
|
}, |
|
{ |
|
"epoch": 0.57, |
|
"learning_rate": 0.0002155050122943068, |
|
"loss": 2.6379, |
|
"theoretical_loss": 3.495177471516584, |
|
"tokens_seen": 1605632000 |
|
}, |
|
{ |
|
"epoch": 0.57, |
|
"learning_rate": 0.00021491393985246832, |
|
"loss": 2.5803, |
|
"theoretical_loss": 3.4945575466790024, |
|
"tokens_seen": 1608908800 |
|
}, |
|
{ |
|
"epoch": 0.58, |
|
"learning_rate": 0.00021432286741062984, |
|
"loss": 2.6055, |
|
"theoretical_loss": 3.4939392358358923, |
|
"tokens_seen": 1612185600 |
|
}, |
|
{ |
|
"epoch": 0.58, |
|
"learning_rate": 0.0002137317949687914, |
|
"loss": 2.6657, |
|
"theoretical_loss": 3.493322531517478, |
|
"tokens_seen": 1615462400 |
|
}, |
|
{ |
|
"epoch": 0.58, |
|
"learning_rate": 0.0002131407225269529, |
|
"loss": 2.6619, |
|
"theoretical_loss": 3.4927074263036157, |
|
"tokens_seen": 1618739200 |
|
}, |
|
{ |
|
"epoch": 0.58, |
|
"learning_rate": 0.00021254965008511443, |
|
"loss": 2.6281, |
|
"theoretical_loss": 3.492093912823367, |
|
"tokens_seen": 1622016000 |
|
}, |
|
{ |
|
"epoch": 0.58, |
|
"learning_rate": 0.00021195857764327598, |
|
"loss": 2.6244, |
|
"theoretical_loss": 3.4914819837545705, |
|
"tokens_seen": 1625292800 |
|
}, |
|
{ |
|
"epoch": 0.58, |
|
"learning_rate": 0.0002113675052014375, |
|
"loss": 2.6007, |
|
"theoretical_loss": 3.490871631823426, |
|
"tokens_seen": 1628569600 |
|
}, |
|
{ |
|
"epoch": 0.58, |
|
"learning_rate": 0.00021077643275959902, |
|
"loss": 2.5869, |
|
"theoretical_loss": 3.490262849804073, |
|
"tokens_seen": 1631846400 |
|
}, |
|
{ |
|
"epoch": 0.58, |
|
"learning_rate": 0.00021018536031776057, |
|
"loss": 2.5682, |
|
"theoretical_loss": 3.4896556305181834, |
|
"tokens_seen": 1635123200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.3460857686736783, |
|
"debugging/distinct-1-grams": 0.7664962955849348, |
|
"debugging/distinct-2-grams": 0.9583333333333334, |
|
"debugging/entropy-1-grams": 4.7939259983502716, |
|
"debugging/entropy-2-grams": 5.366801704259744, |
|
"debugging/length": 505.0, |
|
"debugging/num_segments": 3, |
|
"epoch": 0.59, |
|
"objective/train/docs_used": 904189, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.677978277206421, |
|
"objective/train/theoretical_loss": 3.4890499668345516, |
|
"objective/train/tokens_used": 1658860000, |
|
"theoretical_loss": 3.4890499668345516, |
|
"tokens_seen": 1638400000 |
|
}, |
|
{ |
|
"epoch": 0.59, |
|
"learning_rate": 0.0002095942878759221, |
|
"loss": 2.6157, |
|
"theoretical_loss": 3.4890499668345516, |
|
"tokens_seen": 1638400000 |
|
}, |
|
{ |
|
"epoch": 0.59, |
|
"learning_rate": 0.0002090032154340836, |
|
"loss": 2.6266, |
|
"theoretical_loss": 3.4884458516686947, |
|
"tokens_seen": 1641676800 |
|
}, |
|
{ |
|
"epoch": 0.59, |
|
"learning_rate": 0.00020841214299224516, |
|
"loss": 2.585, |
|
"theoretical_loss": 3.487843277982451, |
|
"tokens_seen": 1644953600 |
|
}, |
|
{ |
|
"epoch": 0.59, |
|
"learning_rate": 0.00020782107055040668, |
|
"loss": 2.6062, |
|
"theoretical_loss": 3.4872422387835877, |
|
"tokens_seen": 1648230400 |
|
}, |
|
{ |
|
"epoch": 0.59, |
|
"learning_rate": 0.0002072299981085682, |
|
"loss": 2.6191, |
|
"theoretical_loss": 3.4866427271254077, |
|
"tokens_seen": 1651507200 |
|
}, |
|
{ |
|
"epoch": 0.59, |
|
"learning_rate": 0.00020663892566672972, |
|
"loss": 2.5949, |
|
"theoretical_loss": 3.4860447361063667, |
|
"tokens_seen": 1654784000 |
|
}, |
|
{ |
|
"epoch": 0.59, |
|
"learning_rate": 0.00020604785322489124, |
|
"loss": 2.6252, |
|
"theoretical_loss": 3.48544825886969, |
|
"tokens_seen": 1658060800 |
|
}, |
|
{ |
|
"epoch": 0.59, |
|
"learning_rate": 0.00020545678078305276, |
|
"loss": 2.6031, |
|
"theoretical_loss": 3.4848532886029924, |
|
"tokens_seen": 1661337600 |
|
}, |
|
{ |
|
"epoch": 0.59, |
|
"learning_rate": 0.00020486570834121428, |
|
"loss": 2.625, |
|
"theoretical_loss": 3.4842598185379066, |
|
"tokens_seen": 1664614400 |
|
}, |
|
{ |
|
"epoch": 0.6, |
|
"learning_rate": 0.00020427463589937583, |
|
"loss": 2.6257, |
|
"theoretical_loss": 3.4836678419497105, |
|
"tokens_seen": 1667891200 |
|
}, |
|
{ |
|
"epoch": 0.6, |
|
"learning_rate": 0.00020368356345753735, |
|
"loss": 2.5742, |
|
"theoretical_loss": 3.483077352156964, |
|
"tokens_seen": 1671168000 |
|
}, |
|
{ |
|
"epoch": 0.6, |
|
"learning_rate": 0.00020309249101569887, |
|
"loss": 2.5602, |
|
"theoretical_loss": 3.4824883425211413, |
|
"tokens_seen": 1674444800 |
|
}, |
|
{ |
|
"epoch": 0.6, |
|
"learning_rate": 0.00020250141857386042, |
|
"loss": 2.577, |
|
"theoretical_loss": 3.4819008064462764, |
|
"tokens_seen": 1677721600 |
|
}, |
|
{ |
|
"epoch": 0.6, |
|
"learning_rate": 0.00020191034613202194, |
|
"loss": 2.585, |
|
"theoretical_loss": 3.481314737378605, |
|
"tokens_seen": 1680998400 |
|
}, |
|
{ |
|
"epoch": 0.6, |
|
"learning_rate": 0.00020133109513902024, |
|
"loss": 2.5956, |
|
"theoretical_loss": 3.4807301288062145, |
|
"tokens_seen": 1684275200 |
|
}, |
|
{ |
|
"epoch": 0.6, |
|
"learning_rate": 0.00020074002269718179, |
|
"loss": 2.5643, |
|
"theoretical_loss": 3.4801469742586955, |
|
"tokens_seen": 1687552000 |
|
}, |
|
{ |
|
"epoch": 0.6, |
|
"learning_rate": 0.0002001489502553433, |
|
"loss": 2.6222, |
|
"theoretical_loss": 3.4795652673067954, |
|
"tokens_seen": 1690828800 |
|
}, |
|
{ |
|
"epoch": 0.61, |
|
"learning_rate": 0.00019955787781350483, |
|
"loss": 2.6145, |
|
"theoretical_loss": 3.478985001562082, |
|
"tokens_seen": 1694105600 |
|
}, |
|
{ |
|
"epoch": 0.61, |
|
"learning_rate": 0.00019896680537166638, |
|
"loss": 2.5763, |
|
"theoretical_loss": 3.4784061706766, |
|
"tokens_seen": 1697382400 |
|
}, |
|
{ |
|
"epoch": 0.61, |
|
"learning_rate": 0.0001983757329298279, |
|
"loss": 2.5693, |
|
"theoretical_loss": 3.477828768342543, |
|
"tokens_seen": 1700659200 |
|
}, |
|
{ |
|
"epoch": 0.61, |
|
"learning_rate": 0.00019778466048798942, |
|
"loss": 2.5406, |
|
"theoretical_loss": 3.477252788291917, |
|
"tokens_seen": 1703936000 |
|
}, |
|
{ |
|
"epoch": 0.61, |
|
"learning_rate": 0.00019719358804615094, |
|
"loss": 2.5817, |
|
"theoretical_loss": 3.4766782242962186, |
|
"tokens_seen": 1707212800 |
|
}, |
|
{ |
|
"epoch": 0.61, |
|
"learning_rate": 0.00019660251560431246, |
|
"loss": 2.546, |
|
"theoretical_loss": 3.476105070166107, |
|
"tokens_seen": 1710489600 |
|
}, |
|
{ |
|
"epoch": 0.61, |
|
"learning_rate": 0.00019601144316247398, |
|
"loss": 2.5882, |
|
"theoretical_loss": 3.475533319751085, |
|
"tokens_seen": 1713766400 |
|
}, |
|
{ |
|
"epoch": 0.61, |
|
"learning_rate": 0.00019542037072063553, |
|
"loss": 2.6089, |
|
"theoretical_loss": 3.4749629669391813, |
|
"tokens_seen": 1717043200 |
|
}, |
|
{ |
|
"epoch": 0.61, |
|
"learning_rate": 0.00019482929827879705, |
|
"loss": 2.6046, |
|
"theoretical_loss": 3.474394005656636, |
|
"tokens_seen": 1720320000 |
|
}, |
|
{ |
|
"epoch": 0.62, |
|
"learning_rate": 0.00019423822583695857, |
|
"loss": 2.608, |
|
"theoretical_loss": 3.473826429867589, |
|
"tokens_seen": 1723596800 |
|
}, |
|
{ |
|
"epoch": 0.62, |
|
"learning_rate": 0.00019364715339512012, |
|
"loss": 2.6189, |
|
"theoretical_loss": 3.4732602335737734, |
|
"tokens_seen": 1726873600 |
|
}, |
|
{ |
|
"epoch": 0.62, |
|
"learning_rate": 0.00019305608095328164, |
|
"loss": 2.6216, |
|
"theoretical_loss": 3.47269541081421, |
|
"tokens_seen": 1730150400 |
|
}, |
|
{ |
|
"epoch": 0.62, |
|
"learning_rate": 0.00019246500851144316, |
|
"loss": 2.6166, |
|
"theoretical_loss": 3.4721319556649046, |
|
"tokens_seen": 1733427200 |
|
}, |
|
{ |
|
"epoch": 0.62, |
|
"learning_rate": 0.00019187393606960468, |
|
"loss": 2.6073, |
|
"theoretical_loss": 3.4715698622385487, |
|
"tokens_seen": 1736704000 |
|
}, |
|
{ |
|
"epoch": 0.62, |
|
"learning_rate": 0.00019128286362776623, |
|
"loss": 2.6233, |
|
"theoretical_loss": 3.471009124684226, |
|
"tokens_seen": 1739980800 |
|
}, |
|
{ |
|
"epoch": 0.62, |
|
"learning_rate": 0.00019069179118592775, |
|
"loss": 2.5942, |
|
"theoretical_loss": 3.470449737187118, |
|
"tokens_seen": 1743257600 |
|
}, |
|
{ |
|
"epoch": 0.62, |
|
"learning_rate": 0.00019010071874408927, |
|
"loss": 2.6462, |
|
"theoretical_loss": 3.4698916939682123, |
|
"tokens_seen": 1746534400 |
|
}, |
|
{ |
|
"epoch": 0.62, |
|
"learning_rate": 0.00018950964630225082, |
|
"loss": 2.617, |
|
"theoretical_loss": 3.4693349892840186, |
|
"tokens_seen": 1749811200 |
|
}, |
|
{ |
|
"epoch": 0.63, |
|
"learning_rate": 0.00018891857386041234, |
|
"loss": 2.5931, |
|
"theoretical_loss": 3.4687796174262826, |
|
"tokens_seen": 1753088000 |
|
}, |
|
{ |
|
"epoch": 0.63, |
|
"learning_rate": 0.00018832750141857386, |
|
"loss": 2.5993, |
|
"theoretical_loss": 3.468225572721704, |
|
"tokens_seen": 1756364800 |
|
}, |
|
{ |
|
"epoch": 0.63, |
|
"learning_rate": 0.0001877364289767354, |
|
"loss": 2.6055, |
|
"theoretical_loss": 3.4676728495316587, |
|
"tokens_seen": 1759641600 |
|
}, |
|
{ |
|
"epoch": 0.63, |
|
"learning_rate": 0.00018714535653489693, |
|
"loss": 2.6027, |
|
"theoretical_loss": 3.467121442251922, |
|
"tokens_seen": 1762918400 |
|
}, |
|
{ |
|
"epoch": 0.63, |
|
"learning_rate": 0.00018655428409305845, |
|
"loss": 2.6185, |
|
"theoretical_loss": 3.4665713453123965, |
|
"tokens_seen": 1766195200 |
|
}, |
|
{ |
|
"epoch": 0.63, |
|
"learning_rate": 0.00018596321165122, |
|
"loss": 2.6066, |
|
"theoretical_loss": 3.46602255317684, |
|
"tokens_seen": 1769472000 |
|
}, |
|
{ |
|
"epoch": 0.63, |
|
"learning_rate": 0.00018537213920938152, |
|
"loss": 2.5993, |
|
"theoretical_loss": 3.465475060342598, |
|
"tokens_seen": 1772748800 |
|
}, |
|
{ |
|
"epoch": 0.63, |
|
"learning_rate": 0.00018478106676754304, |
|
"loss": 2.6411, |
|
"theoretical_loss": 3.4649288613403386, |
|
"tokens_seen": 1776025600 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"learning_rate": 0.00018418999432570459, |
|
"loss": 2.6456, |
|
"theoretical_loss": 3.4643839507337892, |
|
"tokens_seen": 1779302400 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"learning_rate": 0.0001835989218838661, |
|
"loss": 2.6619, |
|
"theoretical_loss": 3.4638403231194763, |
|
"tokens_seen": 1782579200 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"learning_rate": 0.0001830078494420276, |
|
"loss": 2.5926, |
|
"theoretical_loss": 3.463297973126469, |
|
"tokens_seen": 1785856000 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"learning_rate": 0.00018241677700018915, |
|
"loss": 2.6025, |
|
"theoretical_loss": 3.462756895416121, |
|
"tokens_seen": 1789132800 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"learning_rate": 0.00018182570455835067, |
|
"loss": 2.6366, |
|
"theoretical_loss": 3.4622170846818205, |
|
"tokens_seen": 1792409600 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"learning_rate": 0.0001812346321165122, |
|
"loss": 2.6881, |
|
"theoretical_loss": 3.4616785356487387, |
|
"tokens_seen": 1795686400 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"learning_rate": 0.0001806435596746737, |
|
"loss": 2.6392, |
|
"theoretical_loss": 3.461141243073582, |
|
"tokens_seen": 1798963200 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"objective/train/docs_used": 992677, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.878973960876465, |
|
"objective/train/theoretical_loss": 3.4606052017443467, |
|
"objective/train/tokens_used": 1822700000, |
|
"theoretical_loss": 3.4606052017443467, |
|
"tokens_seen": 1802240000 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"learning_rate": 0.00018005248723283526, |
|
"loss": 2.6433, |
|
"theoretical_loss": 3.4606052017443467, |
|
"tokens_seen": 1802240000 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"learning_rate": 0.00017946141479099678, |
|
"loss": 2.6122, |
|
"theoretical_loss": 3.4600704064800745, |
|
"tokens_seen": 1805516800 |
|
}, |
|
{ |
|
"epoch": 0.65, |
|
"learning_rate": 0.0001788703423491583, |
|
"loss": 2.6283, |
|
"theoretical_loss": 3.459536852130613, |
|
"tokens_seen": 1808793600 |
|
}, |
|
{ |
|
"epoch": 0.65, |
|
"learning_rate": 0.00017827926990731985, |
|
"loss": 2.6224, |
|
"theoretical_loss": 3.4590045335763775, |
|
"tokens_seen": 1812070400 |
|
}, |
|
{ |
|
"epoch": 0.65, |
|
"learning_rate": 0.00017768819746548137, |
|
"loss": 2.6079, |
|
"theoretical_loss": 3.4584734457281128, |
|
"tokens_seen": 1815347200 |
|
}, |
|
{ |
|
"epoch": 0.65, |
|
"learning_rate": 0.0001770971250236429, |
|
"loss": 2.6239, |
|
"theoretical_loss": 3.457943583526662, |
|
"tokens_seen": 1818624000 |
|
}, |
|
{ |
|
"epoch": 0.65, |
|
"learning_rate": 0.00017650605258180444, |
|
"loss": 2.6323, |
|
"theoretical_loss": 3.45741494194273, |
|
"tokens_seen": 1821900800 |
|
}, |
|
{ |
|
"epoch": 0.65, |
|
"learning_rate": 0.00017591498013996596, |
|
"loss": 2.6063, |
|
"theoretical_loss": 3.4568875159766606, |
|
"tokens_seen": 1825177600 |
|
}, |
|
{ |
|
"epoch": 0.65, |
|
"learning_rate": 0.00017532390769812748, |
|
"loss": 2.6021, |
|
"theoretical_loss": 3.4563613006582017, |
|
"tokens_seen": 1828454400 |
|
}, |
|
{ |
|
"epoch": 0.65, |
|
"learning_rate": 0.00017473283525628903, |
|
"loss": 2.6058, |
|
"theoretical_loss": 3.4558362910462863, |
|
"tokens_seen": 1831731200 |
|
}, |
|
{ |
|
"epoch": 0.66, |
|
"learning_rate": 0.00017414176281445055, |
|
"loss": 2.572, |
|
"theoretical_loss": 3.4553124822288055, |
|
"tokens_seen": 1835008000 |
|
}, |
|
{ |
|
"epoch": 0.66, |
|
"learning_rate": 0.00017355069037261207, |
|
"loss": 2.6485, |
|
"theoretical_loss": 3.4547898693223873, |
|
"tokens_seen": 1838284800 |
|
}, |
|
{ |
|
"epoch": 0.66, |
|
"learning_rate": 0.00017295961793077362, |
|
"loss": 2.5612, |
|
"theoretical_loss": 3.4542684474721805, |
|
"tokens_seen": 1841561600 |
|
}, |
|
{ |
|
"epoch": 0.66, |
|
"learning_rate": 0.00017236854548893514, |
|
"loss": 2.5701, |
|
"theoretical_loss": 3.453748211851636, |
|
"tokens_seen": 1844838400 |
|
}, |
|
{ |
|
"epoch": 0.66, |
|
"learning_rate": 0.00017177747304709666, |
|
"loss": 2.5829, |
|
"theoretical_loss": 3.453229157662291, |
|
"tokens_seen": 1848115200 |
|
}, |
|
{ |
|
"epoch": 0.66, |
|
"learning_rate": 0.0001711864006052582, |
|
"loss": 2.5624, |
|
"theoretical_loss": 3.452711280133558, |
|
"tokens_seen": 1851392000 |
|
}, |
|
{ |
|
"epoch": 0.66, |
|
"learning_rate": 0.00017059532816341973, |
|
"loss": 2.5653, |
|
"theoretical_loss": 3.4521945745225118, |
|
"tokens_seen": 1854668800 |
|
}, |
|
{ |
|
"epoch": 0.66, |
|
"learning_rate": 0.00017000425572158125, |
|
"loss": 2.5339, |
|
"theoretical_loss": 3.451679036113682, |
|
"tokens_seen": 1857945600 |
|
}, |
|
{ |
|
"epoch": 0.66, |
|
"learning_rate": 0.00016941318327974277, |
|
"loss": 2.575, |
|
"theoretical_loss": 3.451164660218848, |
|
"tokens_seen": 1861222400 |
|
}, |
|
{ |
|
"epoch": 0.67, |
|
"learning_rate": 0.00016883393228674107, |
|
"loss": 2.5483, |
|
"theoretical_loss": 3.4506514421768273, |
|
"tokens_seen": 1864499200 |
|
}, |
|
{ |
|
"epoch": 0.67, |
|
"learning_rate": 0.0001682428598449026, |
|
"loss": 2.5241, |
|
"theoretical_loss": 3.45013937735328, |
|
"tokens_seen": 1867776000 |
|
}, |
|
{ |
|
"epoch": 0.67, |
|
"learning_rate": 0.0001676517874030641, |
|
"loss": 2.5883, |
|
"theoretical_loss": 3.4496284611405024, |
|
"tokens_seen": 1871052800 |
|
}, |
|
{ |
|
"epoch": 0.67, |
|
"learning_rate": 0.00016706071496122566, |
|
"loss": 2.5769, |
|
"theoretical_loss": 3.4491186889572303, |
|
"tokens_seen": 1874329600 |
|
}, |
|
{ |
|
"epoch": 0.67, |
|
"learning_rate": 0.00016646964251938718, |
|
"loss": 2.5537, |
|
"theoretical_loss": 3.4486100562484396, |
|
"tokens_seen": 1877606400 |
|
}, |
|
{ |
|
"epoch": 0.67, |
|
"learning_rate": 0.0001658785700775487, |
|
"loss": 2.5726, |
|
"theoretical_loss": 3.448102558485152, |
|
"tokens_seen": 1880883200 |
|
}, |
|
{ |
|
"epoch": 0.67, |
|
"learning_rate": 0.00016528749763571025, |
|
"loss": 2.6023, |
|
"theoretical_loss": 3.447596191164239, |
|
"tokens_seen": 1884160000 |
|
}, |
|
{ |
|
"epoch": 0.67, |
|
"learning_rate": 0.00016469642519387177, |
|
"loss": 2.5822, |
|
"theoretical_loss": 3.447090949808233, |
|
"tokens_seen": 1887436800 |
|
}, |
|
{ |
|
"epoch": 0.68, |
|
"learning_rate": 0.0001641053527520333, |
|
"loss": 2.5766, |
|
"theoretical_loss": 3.446586829965132, |
|
"tokens_seen": 1890713600 |
|
}, |
|
{ |
|
"epoch": 0.68, |
|
"learning_rate": 0.00016351428031019484, |
|
"loss": 2.6042, |
|
"theoretical_loss": 3.4460838272082173, |
|
"tokens_seen": 1893990400 |
|
}, |
|
{ |
|
"epoch": 0.68, |
|
"learning_rate": 0.00016292320786835636, |
|
"loss": 2.6269, |
|
"theoretical_loss": 3.4455819371358594, |
|
"tokens_seen": 1897267200 |
|
}, |
|
{ |
|
"epoch": 0.68, |
|
"learning_rate": 0.00016233213542651788, |
|
"loss": 2.6439, |
|
"theoretical_loss": 3.445081155371338, |
|
"tokens_seen": 1900544000 |
|
}, |
|
{ |
|
"epoch": 0.68, |
|
"learning_rate": 0.00016174106298467942, |
|
"loss": 2.6668, |
|
"theoretical_loss": 3.4445814775626564, |
|
"tokens_seen": 1903820800 |
|
}, |
|
{ |
|
"epoch": 0.68, |
|
"learning_rate": 0.00016114999054284095, |
|
"loss": 2.6412, |
|
"theoretical_loss": 3.4440828993823605, |
|
"tokens_seen": 1907097600 |
|
}, |
|
{ |
|
"epoch": 0.68, |
|
"learning_rate": 0.00016055891810100247, |
|
"loss": 2.6627, |
|
"theoretical_loss": 3.4435854165273567, |
|
"tokens_seen": 1910374400 |
|
}, |
|
{ |
|
"epoch": 0.68, |
|
"learning_rate": 0.00015996784565916401, |
|
"loss": 2.6179, |
|
"theoretical_loss": 3.443089024718735, |
|
"tokens_seen": 1913651200 |
|
}, |
|
{ |
|
"epoch": 0.68, |
|
"learning_rate": 0.0001593767732173255, |
|
"loss": 2.6268, |
|
"theoretical_loss": 3.4425937197015912, |
|
"tokens_seen": 1916928000 |
|
}, |
|
{ |
|
"epoch": 0.69, |
|
"learning_rate": 0.00015878570077548703, |
|
"loss": 2.6197, |
|
"theoretical_loss": 3.4420994972448513, |
|
"tokens_seen": 1920204800 |
|
}, |
|
{ |
|
"epoch": 0.69, |
|
"learning_rate": 0.00015819462833364858, |
|
"loss": 2.576, |
|
"theoretical_loss": 3.4416063531410988, |
|
"tokens_seen": 1923481600 |
|
}, |
|
{ |
|
"epoch": 0.69, |
|
"learning_rate": 0.0001576035558918101, |
|
"loss": 2.5762, |
|
"theoretical_loss": 3.4411142832064003, |
|
"tokens_seen": 1926758400 |
|
}, |
|
{ |
|
"epoch": 0.69, |
|
"learning_rate": 0.00015701248344997162, |
|
"loss": 2.512, |
|
"theoretical_loss": 3.4406232832801376, |
|
"tokens_seen": 1930035200 |
|
}, |
|
{ |
|
"epoch": 0.69, |
|
"learning_rate": 0.00015642141100813314, |
|
"loss": 2.565, |
|
"theoretical_loss": 3.440133349224835, |
|
"tokens_seen": 1933312000 |
|
}, |
|
{ |
|
"epoch": 0.69, |
|
"learning_rate": 0.00015584216001513147, |
|
"loss": 2.5391, |
|
"theoretical_loss": 3.439644476925995, |
|
"tokens_seen": 1936588800 |
|
}, |
|
{ |
|
"epoch": 0.69, |
|
"learning_rate": 0.00015525108757329299, |
|
"loss": 2.5388, |
|
"theoretical_loss": 3.4391566622919303, |
|
"tokens_seen": 1939865600 |
|
}, |
|
{ |
|
"epoch": 0.69, |
|
"learning_rate": 0.0001546600151314545, |
|
"loss": 2.5713, |
|
"theoretical_loss": 3.438669901253598, |
|
"tokens_seen": 1943142400 |
|
}, |
|
{ |
|
"epoch": 0.7, |
|
"learning_rate": 0.00015406894268961605, |
|
"loss": 2.5858, |
|
"theoretical_loss": 3.4381841897644385, |
|
"tokens_seen": 1946419200 |
|
}, |
|
{ |
|
"epoch": 0.7, |
|
"learning_rate": 0.00015347787024777758, |
|
"loss": 2.5812, |
|
"theoretical_loss": 3.4376995238002137, |
|
"tokens_seen": 1949696000 |
|
}, |
|
{ |
|
"epoch": 0.7, |
|
"learning_rate": 0.0001528867978059391, |
|
"loss": 2.5584, |
|
"theoretical_loss": 3.4372158993588444, |
|
"tokens_seen": 1952972800 |
|
}, |
|
{ |
|
"epoch": 0.7, |
|
"learning_rate": 0.00015229572536410064, |
|
"loss": 2.5319, |
|
"theoretical_loss": 3.436733312460253, |
|
"tokens_seen": 1956249600 |
|
}, |
|
{ |
|
"epoch": 0.7, |
|
"learning_rate": 0.00015170465292226216, |
|
"loss": 2.5283, |
|
"theoretical_loss": 3.436251759146206, |
|
"tokens_seen": 1959526400 |
|
}, |
|
{ |
|
"epoch": 0.7, |
|
"learning_rate": 0.00015111358048042369, |
|
"loss": 2.5268, |
|
"theoretical_loss": 3.435771235480157, |
|
"tokens_seen": 1962803200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.5079118920805717, |
|
"debugging/distinct-1-grams": 0.7766201352507275, |
|
"debugging/distinct-2-grams": 0.9614427024360547, |
|
"debugging/entropy-1-grams": 5.978513760011541, |
|
"debugging/entropy-2-grams": 6.893552160625838, |
|
"debugging/length": 529.4, |
|
"debugging/num_segments": 15, |
|
"epoch": 0.7, |
|
"objective/train/docs_used": 1082414, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.6117191314697266, |
|
"objective/train/theoretical_loss": 3.435291737547092, |
|
"objective/train/tokens_used": 1986540000, |
|
"theoretical_loss": 3.435291737547092, |
|
"tokens_seen": 1966080000 |
|
}, |
|
{ |
|
"epoch": 0.7, |
|
"learning_rate": 0.00015052250803858523, |
|
"loss": 2.5221, |
|
"theoretical_loss": 3.435291737547092, |
|
"tokens_seen": 1966080000 |
|
}, |
|
{ |
|
"epoch": 0.7, |
|
"learning_rate": 0.00014993143559674673, |
|
"loss": 2.5626, |
|
"theoretical_loss": 3.434813261453378, |
|
"tokens_seen": 1969356800 |
|
}, |
|
{ |
|
"epoch": 0.7, |
|
"learning_rate": 0.00014934036315490825, |
|
"loss": 2.597, |
|
"theoretical_loss": 3.434335803326607, |
|
"tokens_seen": 1972633600 |
|
}, |
|
{ |
|
"epoch": 0.71, |
|
"learning_rate": 0.0001487492907130698, |
|
"loss": 2.6264, |
|
"theoretical_loss": 3.433859359315448, |
|
"tokens_seen": 1975910400 |
|
}, |
|
{ |
|
"epoch": 0.71, |
|
"learning_rate": 0.00014815821827123132, |
|
"loss": 2.634, |
|
"theoretical_loss": 3.433383925589498, |
|
"tokens_seen": 1979187200 |
|
}, |
|
{ |
|
"epoch": 0.71, |
|
"learning_rate": 0.00014756714582939284, |
|
"loss": 2.6205, |
|
"theoretical_loss": 3.432909498339132, |
|
"tokens_seen": 1982464000 |
|
}, |
|
{ |
|
"epoch": 0.71, |
|
"learning_rate": 0.00014697607338755439, |
|
"loss": 2.5945, |
|
"theoretical_loss": 3.432436073775358, |
|
"tokens_seen": 1985740800 |
|
}, |
|
{ |
|
"epoch": 0.71, |
|
"learning_rate": 0.0001463850009457159, |
|
"loss": 2.603, |
|
"theoretical_loss": 3.4319636481296687, |
|
"tokens_seen": 1989017600 |
|
}, |
|
{ |
|
"epoch": 0.71, |
|
"learning_rate": 0.00014579392850387743, |
|
"loss": 2.6237, |
|
"theoretical_loss": 3.4314922176539002, |
|
"tokens_seen": 1992294400 |
|
}, |
|
{ |
|
"epoch": 0.71, |
|
"learning_rate": 0.00014520285606203897, |
|
"loss": 2.6082, |
|
"theoretical_loss": 3.431021778620088, |
|
"tokens_seen": 1995571200 |
|
}, |
|
{ |
|
"epoch": 0.71, |
|
"learning_rate": 0.0001446117836202005, |
|
"loss": 2.5486, |
|
"theoretical_loss": 3.4305523273203233, |
|
"tokens_seen": 1998848000 |
|
}, |
|
{ |
|
"epoch": 0.72, |
|
"learning_rate": 0.00014402071117836202, |
|
"loss": 2.5873, |
|
"theoretical_loss": 3.430083860066617, |
|
"tokens_seen": 2002124800 |
|
}, |
|
{ |
|
"epoch": 0.72, |
|
"learning_rate": 0.00014342963873652354, |
|
"loss": 2.5733, |
|
"theoretical_loss": 3.429616373190753, |
|
"tokens_seen": 2005401600 |
|
}, |
|
{ |
|
"epoch": 0.72, |
|
"learning_rate": 0.00014283856629468509, |
|
"loss": 2.5949, |
|
"theoretical_loss": 3.4291498630441586, |
|
"tokens_seen": 2008678400 |
|
}, |
|
{ |
|
"epoch": 0.72, |
|
"learning_rate": 0.0001422474938528466, |
|
"loss": 2.5914, |
|
"theoretical_loss": 3.4286843259977613, |
|
"tokens_seen": 2011955200 |
|
}, |
|
{ |
|
"epoch": 0.72, |
|
"learning_rate": 0.00014165642141100813, |
|
"loss": 2.5759, |
|
"theoretical_loss": 3.4282197584418546, |
|
"tokens_seen": 2015232000 |
|
}, |
|
{ |
|
"epoch": 0.72, |
|
"learning_rate": 0.00014106534896916967, |
|
"loss": 2.5687, |
|
"theoretical_loss": 3.427756156785965, |
|
"tokens_seen": 2018508800 |
|
}, |
|
{ |
|
"epoch": 0.72, |
|
"learning_rate": 0.0001404742765273312, |
|
"loss": 2.5484, |
|
"theoretical_loss": 3.427293517458717, |
|
"tokens_seen": 2021785600 |
|
}, |
|
{ |
|
"epoch": 0.72, |
|
"learning_rate": 0.00013988320408549272, |
|
"loss": 2.5743, |
|
"theoretical_loss": 3.4268318369076995, |
|
"tokens_seen": 2025062400 |
|
}, |
|
{ |
|
"epoch": 0.72, |
|
"learning_rate": 0.00013929213164365426, |
|
"loss": 2.5631, |
|
"theoretical_loss": 3.4263711115993383, |
|
"tokens_seen": 2028339200 |
|
}, |
|
{ |
|
"epoch": 0.73, |
|
"learning_rate": 0.00013870105920181578, |
|
"loss": 2.5336, |
|
"theoretical_loss": 3.425911338018763, |
|
"tokens_seen": 2031616000 |
|
}, |
|
{ |
|
"epoch": 0.73, |
|
"learning_rate": 0.0001381099867599773, |
|
"loss": 2.543, |
|
"theoretical_loss": 3.425452512669678, |
|
"tokens_seen": 2034892800 |
|
}, |
|
{ |
|
"epoch": 0.73, |
|
"learning_rate": 0.00013751891431813885, |
|
"loss": 2.5836, |
|
"theoretical_loss": 3.4249946320742364, |
|
"tokens_seen": 2038169600 |
|
}, |
|
{ |
|
"epoch": 0.73, |
|
"learning_rate": 0.00013692784187630037, |
|
"loss": 2.5969, |
|
"theoretical_loss": 3.4245376927729128, |
|
"tokens_seen": 2041446400 |
|
}, |
|
{ |
|
"epoch": 0.73, |
|
"learning_rate": 0.0001363367694344619, |
|
"loss": 2.6024, |
|
"theoretical_loss": 3.424081691324375, |
|
"tokens_seen": 2044723200 |
|
}, |
|
{ |
|
"epoch": 0.73, |
|
"learning_rate": 0.00013574569699262344, |
|
"loss": 2.5964, |
|
"theoretical_loss": 3.4236266243053626, |
|
"tokens_seen": 2048000000 |
|
}, |
|
{ |
|
"epoch": 0.73, |
|
"learning_rate": 0.00013515462455078494, |
|
"loss": 2.6139, |
|
"theoretical_loss": 3.4231724883105628, |
|
"tokens_seen": 2051276800 |
|
}, |
|
{ |
|
"epoch": 0.73, |
|
"learning_rate": 0.00013456355210894646, |
|
"loss": 2.6353, |
|
"theoretical_loss": 3.422719279952486, |
|
"tokens_seen": 2054553600 |
|
}, |
|
{ |
|
"epoch": 0.73, |
|
"learning_rate": 0.000133972479667108, |
|
"loss": 2.5992, |
|
"theoretical_loss": 3.422266995861345, |
|
"tokens_seen": 2057830400 |
|
}, |
|
{ |
|
"epoch": 0.74, |
|
"learning_rate": 0.00013338140722526953, |
|
"loss": 2.6145, |
|
"theoretical_loss": 3.421815632684937, |
|
"tokens_seen": 2061107200 |
|
}, |
|
{ |
|
"epoch": 0.74, |
|
"learning_rate": 0.00013279033478343105, |
|
"loss": 2.5986, |
|
"theoretical_loss": 3.4213651870885196, |
|
"tokens_seen": 2064384000 |
|
}, |
|
{ |
|
"epoch": 0.74, |
|
"learning_rate": 0.0001321992623415926, |
|
"loss": 2.6047, |
|
"theoretical_loss": 3.4209156557546954, |
|
"tokens_seen": 2067660800 |
|
}, |
|
{ |
|
"epoch": 0.74, |
|
"learning_rate": 0.00013160818989975412, |
|
"loss": 2.5892, |
|
"theoretical_loss": 3.420467035383294, |
|
"tokens_seen": 2070937600 |
|
}, |
|
{ |
|
"epoch": 0.74, |
|
"learning_rate": 0.00013101711745791564, |
|
"loss": 2.5483, |
|
"theoretical_loss": 3.4200193226912554, |
|
"tokens_seen": 2074214400 |
|
}, |
|
{ |
|
"epoch": 0.74, |
|
"learning_rate": 0.00013042604501607716, |
|
"loss": 2.5657, |
|
"theoretical_loss": 3.419572514412513, |
|
"tokens_seen": 2077491200 |
|
}, |
|
{ |
|
"epoch": 0.74, |
|
"learning_rate": 0.0001298349725742387, |
|
"loss": 2.5535, |
|
"theoretical_loss": 3.4191266072978816, |
|
"tokens_seen": 2080768000 |
|
}, |
|
{ |
|
"epoch": 0.74, |
|
"learning_rate": 0.00012924390013240023, |
|
"loss": 2.5329, |
|
"theoretical_loss": 3.418681598114942, |
|
"tokens_seen": 2084044800 |
|
}, |
|
{ |
|
"epoch": 0.75, |
|
"learning_rate": 0.00012865282769056175, |
|
"loss": 2.5723, |
|
"theoretical_loss": 3.4182374836479266, |
|
"tokens_seen": 2087321600 |
|
}, |
|
{ |
|
"epoch": 0.75, |
|
"learning_rate": 0.0001280617552487233, |
|
"loss": 2.5494, |
|
"theoretical_loss": 3.4177942606976126, |
|
"tokens_seen": 2090598400 |
|
}, |
|
{ |
|
"epoch": 0.75, |
|
"learning_rate": 0.00012747068280688482, |
|
"loss": 2.5239, |
|
"theoretical_loss": 3.4173519260812064, |
|
"tokens_seen": 2093875200 |
|
}, |
|
{ |
|
"epoch": 0.75, |
|
"learning_rate": 0.00012687961036504634, |
|
"loss": 2.5423, |
|
"theoretical_loss": 3.4169104766322356, |
|
"tokens_seen": 2097152000 |
|
}, |
|
{ |
|
"epoch": 0.75, |
|
"learning_rate": 0.00012628853792320788, |
|
"loss": 2.5346, |
|
"theoretical_loss": 3.4164699092004405, |
|
"tokens_seen": 2100428800 |
|
}, |
|
{ |
|
"epoch": 0.75, |
|
"learning_rate": 0.0001256974654813694, |
|
"loss": 2.56, |
|
"theoretical_loss": 3.416030220651664, |
|
"tokens_seen": 2103705600 |
|
}, |
|
{ |
|
"epoch": 0.75, |
|
"learning_rate": 0.00012510639303953093, |
|
"loss": 2.5017, |
|
"theoretical_loss": 3.4155914078677467, |
|
"tokens_seen": 2106982400 |
|
}, |
|
{ |
|
"epoch": 0.75, |
|
"learning_rate": 0.00012451532059769245, |
|
"loss": 2.5379, |
|
"theoretical_loss": 3.4151534677464195, |
|
"tokens_seen": 2110259200 |
|
}, |
|
{ |
|
"epoch": 0.75, |
|
"learning_rate": 0.000123924248155854, |
|
"loss": 2.5354, |
|
"theoretical_loss": 3.414716397201196, |
|
"tokens_seen": 2113536000 |
|
}, |
|
{ |
|
"epoch": 0.76, |
|
"learning_rate": 0.00012333317571401552, |
|
"loss": 2.5039, |
|
"theoretical_loss": 3.4142801931612725, |
|
"tokens_seen": 2116812800 |
|
}, |
|
{ |
|
"epoch": 0.76, |
|
"learning_rate": 0.00012274210327217704, |
|
"loss": 2.5356, |
|
"theoretical_loss": 3.41384485257142, |
|
"tokens_seen": 2120089600 |
|
}, |
|
{ |
|
"epoch": 0.76, |
|
"learning_rate": 0.00012215103083033858, |
|
"loss": 2.5503, |
|
"theoretical_loss": 3.4134103723918834, |
|
"tokens_seen": 2123366400 |
|
}, |
|
{ |
|
"epoch": 0.76, |
|
"learning_rate": 0.00012155995838850009, |
|
"loss": 2.526, |
|
"theoretical_loss": 3.412976749598278, |
|
"tokens_seen": 2126643200 |
|
}, |
|
{ |
|
"epoch": 0.76, |
|
"objective/train/docs_used": 1175218, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.4588398933410645, |
|
"objective/train/theoretical_loss": 3.412543981181491, |
|
"objective/train/tokens_used": 2150380000, |
|
"theoretical_loss": 3.412543981181491, |
|
"tokens_seen": 2129920000 |
|
}, |
|
{ |
|
"epoch": 0.76, |
|
"learning_rate": 0.00012096888594666163, |
|
"loss": 2.5746, |
|
"theoretical_loss": 3.412543981181491, |
|
"tokens_seen": 2129920000 |
|
}, |
|
{ |
|
"epoch": 0.76, |
|
"learning_rate": 0.00012037781350482316, |
|
"loss": 2.5522, |
|
"theoretical_loss": 3.412112064147577, |
|
"tokens_seen": 2133196800 |
|
}, |
|
{ |
|
"epoch": 0.76, |
|
"learning_rate": 0.00011978674106298468, |
|
"loss": 2.5646, |
|
"theoretical_loss": 3.4116809955176617, |
|
"tokens_seen": 2136473600 |
|
}, |
|
{ |
|
"epoch": 0.76, |
|
"learning_rate": 0.00011919566862114621, |
|
"loss": 2.5608, |
|
"theoretical_loss": 3.4112507723278416, |
|
"tokens_seen": 2139750400 |
|
}, |
|
{ |
|
"epoch": 0.77, |
|
"learning_rate": 0.00011860459617930775, |
|
"loss": 2.5713, |
|
"theoretical_loss": 3.410821391629086, |
|
"tokens_seen": 2143027200 |
|
}, |
|
{ |
|
"epoch": 0.77, |
|
"learning_rate": 0.00011801352373746927, |
|
"loss": 2.5533, |
|
"theoretical_loss": 3.4103928504871406, |
|
"tokens_seen": 2146304000 |
|
}, |
|
{ |
|
"epoch": 0.77, |
|
"learning_rate": 0.00011742245129563079, |
|
"loss": 2.5653, |
|
"theoretical_loss": 3.4099651459824294, |
|
"tokens_seen": 2149580800 |
|
}, |
|
{ |
|
"epoch": 0.77, |
|
"learning_rate": 0.00011683137885379231, |
|
"loss": 2.5656, |
|
"theoretical_loss": 3.409538275209961, |
|
"tokens_seen": 2152857600 |
|
}, |
|
{ |
|
"epoch": 0.77, |
|
"learning_rate": 0.00011624030641195385, |
|
"loss": 2.6038, |
|
"theoretical_loss": 3.409112235279233, |
|
"tokens_seen": 2156134400 |
|
}, |
|
{ |
|
"epoch": 0.77, |
|
"learning_rate": 0.00011564923397011538, |
|
"loss": 2.5604, |
|
"theoretical_loss": 3.4086870233141355, |
|
"tokens_seen": 2159411200 |
|
}, |
|
{ |
|
"epoch": 0.77, |
|
"learning_rate": 0.0001150581615282769, |
|
"loss": 2.5729, |
|
"theoretical_loss": 3.4082626364528625, |
|
"tokens_seen": 2162688000 |
|
}, |
|
{ |
|
"epoch": 0.77, |
|
"learning_rate": 0.00011446708908643844, |
|
"loss": 2.5849, |
|
"theoretical_loss": 3.4078390718478153, |
|
"tokens_seen": 2165964800 |
|
}, |
|
{ |
|
"epoch": 0.77, |
|
"learning_rate": 0.00011387601664459997, |
|
"loss": 2.6054, |
|
"theoretical_loss": 3.4074163266655115, |
|
"tokens_seen": 2169241600 |
|
}, |
|
{ |
|
"epoch": 0.78, |
|
"learning_rate": 0.00011328494420276149, |
|
"loss": 2.5867, |
|
"theoretical_loss": 3.4069943980864945, |
|
"tokens_seen": 2172518400 |
|
}, |
|
{ |
|
"epoch": 0.78, |
|
"learning_rate": 0.00011269387176092302, |
|
"loss": 2.5776, |
|
"theoretical_loss": 3.4065732833052436, |
|
"tokens_seen": 2175795200 |
|
}, |
|
{ |
|
"epoch": 0.78, |
|
"learning_rate": 0.00011210279931908456, |
|
"loss": 2.5782, |
|
"theoretical_loss": 3.406152979530081, |
|
"tokens_seen": 2179072000 |
|
}, |
|
{ |
|
"epoch": 0.78, |
|
"learning_rate": 0.00011151172687724608, |
|
"loss": 2.5612, |
|
"theoretical_loss": 3.405733483983088, |
|
"tokens_seen": 2182348800 |
|
}, |
|
{ |
|
"epoch": 0.78, |
|
"learning_rate": 0.00011092065443540761, |
|
"loss": 2.5263, |
|
"theoretical_loss": 3.4053147939000117, |
|
"tokens_seen": 2185625600 |
|
}, |
|
{ |
|
"epoch": 0.78, |
|
"learning_rate": 0.00011032958199356912, |
|
"loss": 2.5404, |
|
"theoretical_loss": 3.4048969065301806, |
|
"tokens_seen": 2188902400 |
|
}, |
|
{ |
|
"epoch": 0.78, |
|
"learning_rate": 0.00010973850955173066, |
|
"loss": 2.5435, |
|
"theoretical_loss": 3.4044798191364163, |
|
"tokens_seen": 2192179200 |
|
}, |
|
{ |
|
"epoch": 0.78, |
|
"learning_rate": 0.00010914743710989219, |
|
"loss": 2.5307, |
|
"theoretical_loss": 3.4040635289949472, |
|
"tokens_seen": 2195456000 |
|
}, |
|
{ |
|
"epoch": 0.79, |
|
"learning_rate": 0.00010855636466805371, |
|
"loss": 2.5695, |
|
"theoretical_loss": 3.403648033395324, |
|
"tokens_seen": 2198732800 |
|
}, |
|
{ |
|
"epoch": 0.79, |
|
"learning_rate": 0.00010796529222621525, |
|
"loss": 2.5625, |
|
"theoretical_loss": 3.403233329640333, |
|
"tokens_seen": 2202009600 |
|
}, |
|
{ |
|
"epoch": 0.79, |
|
"learning_rate": 0.00010737421978437678, |
|
"loss": 2.5458, |
|
"theoretical_loss": 3.4028194150459137, |
|
"tokens_seen": 2205286400 |
|
}, |
|
{ |
|
"epoch": 0.79, |
|
"learning_rate": 0.0001067831473425383, |
|
"loss": 2.5627, |
|
"theoretical_loss": 3.4024062869410745, |
|
"tokens_seen": 2208563200 |
|
}, |
|
{ |
|
"epoch": 0.79, |
|
"learning_rate": 0.00010619207490069983, |
|
"loss": 2.5525, |
|
"theoretical_loss": 3.4019939426678096, |
|
"tokens_seen": 2211840000 |
|
}, |
|
{ |
|
"epoch": 0.79, |
|
"learning_rate": 0.00010561282390769813, |
|
"loss": 2.55, |
|
"theoretical_loss": 3.401582379581017, |
|
"tokens_seen": 2215116800 |
|
}, |
|
{ |
|
"epoch": 0.79, |
|
"learning_rate": 0.00010502175146585965, |
|
"loss": 2.544, |
|
"theoretical_loss": 3.401171595048417, |
|
"tokens_seen": 2218393600 |
|
}, |
|
{ |
|
"epoch": 0.79, |
|
"learning_rate": 0.00010444250047285795, |
|
"loss": 2.5597, |
|
"theoretical_loss": 3.40076158645047, |
|
"tokens_seen": 2221670400 |
|
}, |
|
{ |
|
"epoch": 0.79, |
|
"learning_rate": 0.00010385142803101949, |
|
"loss": 2.5823, |
|
"theoretical_loss": 3.400352351180298, |
|
"tokens_seen": 2224947200 |
|
}, |
|
{ |
|
"epoch": 0.8, |
|
"learning_rate": 0.00010326035558918101, |
|
"loss": 2.5944, |
|
"theoretical_loss": 3.3999438866436047, |
|
"tokens_seen": 2228224000 |
|
}, |
|
{ |
|
"epoch": 0.8, |
|
"learning_rate": 0.00010266928314734254, |
|
"loss": 2.5659, |
|
"theoretical_loss": 3.399536190258594, |
|
"tokens_seen": 2231500800 |
|
}, |
|
{ |
|
"epoch": 0.8, |
|
"learning_rate": 0.00010207821070550406, |
|
"loss": 2.5591, |
|
"theoretical_loss": 3.399129259455895, |
|
"tokens_seen": 2234777600 |
|
}, |
|
{ |
|
"epoch": 0.8, |
|
"learning_rate": 0.0001014871382636656, |
|
"loss": 2.5303, |
|
"theoretical_loss": 3.3987230916784803, |
|
"tokens_seen": 2238054400 |
|
}, |
|
{ |
|
"epoch": 0.8, |
|
"learning_rate": 0.00010089606582182713, |
|
"loss": 2.5886, |
|
"theoretical_loss": 3.398317684381593, |
|
"tokens_seen": 2241331200 |
|
}, |
|
{ |
|
"epoch": 0.8, |
|
"learning_rate": 0.00010030499337998865, |
|
"loss": 2.5194, |
|
"theoretical_loss": 3.397913035032666, |
|
"tokens_seen": 2244608000 |
|
}, |
|
{ |
|
"epoch": 0.8, |
|
"learning_rate": 9.971392093815019e-05, |
|
"loss": 2.559, |
|
"theoretical_loss": 3.3975091411112484, |
|
"tokens_seen": 2247884800 |
|
}, |
|
{ |
|
"epoch": 0.8, |
|
"learning_rate": 9.912284849631172e-05, |
|
"loss": 2.5484, |
|
"theoretical_loss": 3.397106000108929, |
|
"tokens_seen": 2251161600 |
|
}, |
|
{ |
|
"epoch": 0.81, |
|
"learning_rate": 9.853177605447323e-05, |
|
"loss": 2.5295, |
|
"theoretical_loss": 3.3967036095292613, |
|
"tokens_seen": 2254438400 |
|
}, |
|
{ |
|
"epoch": 0.81, |
|
"learning_rate": 9.794070361263476e-05, |
|
"loss": 2.5844, |
|
"theoretical_loss": 3.3963019668876897, |
|
"tokens_seen": 2257715200 |
|
}, |
|
{ |
|
"epoch": 0.81, |
|
"learning_rate": 9.73496311707963e-05, |
|
"loss": 2.5484, |
|
"theoretical_loss": 3.395901069711476, |
|
"tokens_seen": 2260992000 |
|
}, |
|
{ |
|
"epoch": 0.81, |
|
"learning_rate": 9.675855872895782e-05, |
|
"loss": 2.6276, |
|
"theoretical_loss": 3.3955009155396247, |
|
"tokens_seen": 2264268800 |
|
}, |
|
{ |
|
"epoch": 0.81, |
|
"learning_rate": 9.616748628711935e-05, |
|
"loss": 2.7715, |
|
"theoretical_loss": 3.3951015019228117, |
|
"tokens_seen": 2267545600 |
|
}, |
|
{ |
|
"epoch": 0.81, |
|
"learning_rate": 9.557641384528087e-05, |
|
"loss": 2.804, |
|
"theoretical_loss": 3.3947028264233112, |
|
"tokens_seen": 2270822400 |
|
}, |
|
{ |
|
"epoch": 0.81, |
|
"learning_rate": 9.498534140344241e-05, |
|
"loss": 2.8085, |
|
"theoretical_loss": 3.3943048866149255, |
|
"tokens_seen": 2274099200 |
|
}, |
|
{ |
|
"epoch": 0.81, |
|
"learning_rate": 9.439426896160394e-05, |
|
"loss": 2.7798, |
|
"theoretical_loss": 3.3939076800829118, |
|
"tokens_seen": 2277376000 |
|
}, |
|
{ |
|
"epoch": 0.81, |
|
"learning_rate": 9.380319651976546e-05, |
|
"loss": 2.7643, |
|
"theoretical_loss": 3.393511204423914, |
|
"tokens_seen": 2280652800 |
|
}, |
|
{ |
|
"epoch": 0.82, |
|
"learning_rate": 9.3212124077927e-05, |
|
"loss": 2.7242, |
|
"theoretical_loss": 3.3931154572458917, |
|
"tokens_seen": 2283929600 |
|
}, |
|
{ |
|
"epoch": 0.82, |
|
"learning_rate": 9.262105163608853e-05, |
|
"loss": 2.6973, |
|
"theoretical_loss": 3.3927204361680503, |
|
"tokens_seen": 2287206400 |
|
}, |
|
{ |
|
"epoch": 0.82, |
|
"learning_rate": 9.202997919425005e-05, |
|
"loss": 2.6357, |
|
"theoretical_loss": 3.3923261388207733, |
|
"tokens_seen": 2290483200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.46869240972374726, |
|
"debugging/distinct-1-grams": 0.8121206810717653, |
|
"debugging/distinct-2-grams": 0.9660047807645101, |
|
"debugging/entropy-1-grams": 5.89482428427087, |
|
"debugging/entropy-2-grams": 6.645682303295068, |
|
"debugging/length": 500.8333333333333, |
|
"debugging/num_segments": 12, |
|
"epoch": 0.82, |
|
"objective/train/docs_used": 1259114, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.5495071411132812, |
|
"objective/train/theoretical_loss": 3.3919325628455534, |
|
"objective/train/tokens_used": 2314220000, |
|
"theoretical_loss": 3.3919325628455534, |
|
"tokens_seen": 2293760000 |
|
}, |
|
{ |
|
"epoch": 0.82, |
|
"learning_rate": 9.143890675241157e-05, |
|
"loss": 2.6191, |
|
"theoretical_loss": 3.3919325628455534, |
|
"tokens_seen": 2293760000 |
|
}, |
|
{ |
|
"epoch": 0.82, |
|
"learning_rate": 9.08478343105731e-05, |
|
"loss": 2.6266, |
|
"theoretical_loss": 3.3915397058949233, |
|
"tokens_seen": 2297036800 |
|
}, |
|
{ |
|
"epoch": 0.82, |
|
"learning_rate": 9.025676186873463e-05, |
|
"loss": 2.575, |
|
"theoretical_loss": 3.3911475656323904, |
|
"tokens_seen": 2300313600 |
|
}, |
|
{ |
|
"epoch": 0.82, |
|
"learning_rate": 8.966568942689616e-05, |
|
"loss": 2.5819, |
|
"theoretical_loss": 3.3907561397323693, |
|
"tokens_seen": 2303590400 |
|
}, |
|
{ |
|
"epoch": 0.82, |
|
"learning_rate": 8.907461698505768e-05, |
|
"loss": 2.5973, |
|
"theoretical_loss": 3.3903654258801152, |
|
"tokens_seen": 2306867200 |
|
}, |
|
{ |
|
"epoch": 0.83, |
|
"learning_rate": 8.848354454321922e-05, |
|
"loss": 2.5693, |
|
"theoretical_loss": 3.389975421771658, |
|
"tokens_seen": 2310144000 |
|
}, |
|
{ |
|
"epoch": 0.83, |
|
"learning_rate": 8.789247210138075e-05, |
|
"loss": 2.5431, |
|
"theoretical_loss": 3.389586125113736, |
|
"tokens_seen": 2313420800 |
|
}, |
|
{ |
|
"epoch": 0.83, |
|
"learning_rate": 8.730139965954227e-05, |
|
"loss": 2.5469, |
|
"theoretical_loss": 3.389197533623734, |
|
"tokens_seen": 2316697600 |
|
}, |
|
{ |
|
"epoch": 0.83, |
|
"learning_rate": 8.671032721770381e-05, |
|
"loss": 2.5209, |
|
"theoretical_loss": 3.3888096450296152, |
|
"tokens_seen": 2319974400 |
|
}, |
|
{ |
|
"epoch": 0.83, |
|
"learning_rate": 8.611925477586534e-05, |
|
"loss": 2.5672, |
|
"theoretical_loss": 3.3884224570698613, |
|
"tokens_seen": 2323251200 |
|
}, |
|
{ |
|
"epoch": 0.83, |
|
"learning_rate": 8.552818233402686e-05, |
|
"loss": 2.5477, |
|
"theoretical_loss": 3.388035967493405, |
|
"tokens_seen": 2326528000 |
|
}, |
|
{ |
|
"epoch": 0.83, |
|
"learning_rate": 8.49371098921884e-05, |
|
"loss": 2.5493, |
|
"theoretical_loss": 3.387650174059569, |
|
"tokens_seen": 2329804800 |
|
}, |
|
{ |
|
"epoch": 0.83, |
|
"learning_rate": 8.43460374503499e-05, |
|
"loss": 2.5458, |
|
"theoretical_loss": 3.3872650745380044, |
|
"tokens_seen": 2333081600 |
|
}, |
|
{ |
|
"epoch": 0.83, |
|
"learning_rate": 8.375496500851144e-05, |
|
"loss": 2.5468, |
|
"theoretical_loss": 3.3868806667086275, |
|
"tokens_seen": 2336358400 |
|
}, |
|
{ |
|
"epoch": 0.84, |
|
"learning_rate": 8.316389256667297e-05, |
|
"loss": 2.523, |
|
"theoretical_loss": 3.3864969483615575, |
|
"tokens_seen": 2339635200 |
|
}, |
|
{ |
|
"epoch": 0.84, |
|
"learning_rate": 8.25728201248345e-05, |
|
"loss": 2.5145, |
|
"theoretical_loss": 3.3861139172970582, |
|
"tokens_seen": 2342912000 |
|
}, |
|
{ |
|
"epoch": 0.84, |
|
"learning_rate": 8.198174768299603e-05, |
|
"loss": 2.5311, |
|
"theoretical_loss": 3.385731571325474, |
|
"tokens_seen": 2346188800 |
|
}, |
|
{ |
|
"epoch": 0.84, |
|
"learning_rate": 8.139067524115756e-05, |
|
"loss": 2.526, |
|
"theoretical_loss": 3.385349908267171, |
|
"tokens_seen": 2349465600 |
|
}, |
|
{ |
|
"epoch": 0.84, |
|
"learning_rate": 8.079960279931908e-05, |
|
"loss": 2.5695, |
|
"theoretical_loss": 3.3849689259524784, |
|
"tokens_seen": 2352742400 |
|
}, |
|
{ |
|
"epoch": 0.84, |
|
"learning_rate": 8.020853035748062e-05, |
|
"loss": 2.591, |
|
"theoretical_loss": 3.3845886222216275, |
|
"tokens_seen": 2356019200 |
|
}, |
|
{ |
|
"epoch": 0.84, |
|
"learning_rate": 7.961745791564215e-05, |
|
"loss": 2.6254, |
|
"theoretical_loss": 3.384208994924694, |
|
"tokens_seen": 2359296000 |
|
}, |
|
{ |
|
"epoch": 0.84, |
|
"learning_rate": 7.902638547380367e-05, |
|
"loss": 2.7161, |
|
"theoretical_loss": 3.3838300419215384, |
|
"tokens_seen": 2362572800 |
|
}, |
|
{ |
|
"epoch": 0.84, |
|
"learning_rate": 7.843531303196521e-05, |
|
"loss": 2.7479, |
|
"theoretical_loss": 3.383451761081749, |
|
"tokens_seen": 2365849600 |
|
}, |
|
{ |
|
"epoch": 0.85, |
|
"learning_rate": 7.784424059012673e-05, |
|
"loss": 2.6841, |
|
"theoretical_loss": 3.383074150284584, |
|
"tokens_seen": 2369126400 |
|
}, |
|
{ |
|
"epoch": 0.85, |
|
"learning_rate": 7.725316814828825e-05, |
|
"loss": 2.6805, |
|
"theoretical_loss": 3.3826972074189134, |
|
"tokens_seen": 2372403200 |
|
}, |
|
{ |
|
"epoch": 0.85, |
|
"learning_rate": 7.666209570644978e-05, |
|
"loss": 2.6507, |
|
"theoretical_loss": 3.3823209303831634, |
|
"tokens_seen": 2375680000 |
|
}, |
|
{ |
|
"epoch": 0.85, |
|
"learning_rate": 7.60710232646113e-05, |
|
"loss": 2.5914, |
|
"theoretical_loss": 3.3819453170852602, |
|
"tokens_seen": 2378956800 |
|
}, |
|
{ |
|
"epoch": 0.85, |
|
"learning_rate": 7.547995082277284e-05, |
|
"loss": 2.5932, |
|
"theoretical_loss": 3.381570365442573, |
|
"tokens_seen": 2382233600 |
|
}, |
|
{ |
|
"epoch": 0.85, |
|
"learning_rate": 7.488887838093437e-05, |
|
"loss": 2.5743, |
|
"theoretical_loss": 3.3811960733818585, |
|
"tokens_seen": 2385510400 |
|
}, |
|
{ |
|
"epoch": 0.85, |
|
"learning_rate": 7.42978059390959e-05, |
|
"loss": 2.587, |
|
"theoretical_loss": 3.380822438839207, |
|
"tokens_seen": 2388787200 |
|
}, |
|
{ |
|
"epoch": 0.85, |
|
"learning_rate": 7.370673349725743e-05, |
|
"loss": 2.5342, |
|
"theoretical_loss": 3.380449459759986, |
|
"tokens_seen": 2392064000 |
|
}, |
|
{ |
|
"epoch": 0.86, |
|
"learning_rate": 7.311566105541895e-05, |
|
"loss": 2.5706, |
|
"theoretical_loss": 3.380077134098787, |
|
"tokens_seen": 2395340800 |
|
}, |
|
{ |
|
"epoch": 0.86, |
|
"learning_rate": 7.252458861358048e-05, |
|
"loss": 2.5504, |
|
"theoretical_loss": 3.3797054598193723, |
|
"tokens_seen": 2398617600 |
|
}, |
|
{ |
|
"epoch": 0.86, |
|
"learning_rate": 7.193351617174202e-05, |
|
"loss": 2.5328, |
|
"theoretical_loss": 3.379334434894619, |
|
"tokens_seen": 2401894400 |
|
}, |
|
{ |
|
"epoch": 0.86, |
|
"learning_rate": 7.134244372990354e-05, |
|
"loss": 2.5776, |
|
"theoretical_loss": 3.378964057306467, |
|
"tokens_seen": 2405171200 |
|
}, |
|
{ |
|
"epoch": 0.86, |
|
"learning_rate": 7.075137128806507e-05, |
|
"loss": 2.5716, |
|
"theoretical_loss": 3.378594325045867, |
|
"tokens_seen": 2408448000 |
|
}, |
|
{ |
|
"epoch": 0.86, |
|
"learning_rate": 7.01602988462266e-05, |
|
"loss": 2.5901, |
|
"theoretical_loss": 3.378225236112729, |
|
"tokens_seen": 2411724800 |
|
}, |
|
{ |
|
"epoch": 0.86, |
|
"learning_rate": 6.956922640438811e-05, |
|
"loss": 2.5617, |
|
"theoretical_loss": 3.377856788515865, |
|
"tokens_seen": 2415001600 |
|
}, |
|
{ |
|
"epoch": 0.86, |
|
"learning_rate": 6.897815396254965e-05, |
|
"loss": 2.568, |
|
"theoretical_loss": 3.3774889802729446, |
|
"tokens_seen": 2418278400 |
|
}, |
|
{ |
|
"epoch": 0.86, |
|
"learning_rate": 6.838708152071118e-05, |
|
"loss": 2.6481, |
|
"theoretical_loss": 3.377121809410438, |
|
"tokens_seen": 2421555200 |
|
}, |
|
{ |
|
"epoch": 0.87, |
|
"learning_rate": 6.77960090788727e-05, |
|
"loss": 2.6591, |
|
"theoretical_loss": 3.3767552739635684, |
|
"tokens_seen": 2424832000 |
|
}, |
|
{ |
|
"epoch": 0.87, |
|
"learning_rate": 6.720493663703424e-05, |
|
"loss": 2.6231, |
|
"theoretical_loss": 3.37638937197626, |
|
"tokens_seen": 2428108800 |
|
}, |
|
{ |
|
"epoch": 0.87, |
|
"learning_rate": 6.661386419519576e-05, |
|
"loss": 2.5929, |
|
"theoretical_loss": 3.3760241015010872, |
|
"tokens_seen": 2431385600 |
|
}, |
|
{ |
|
"epoch": 0.87, |
|
"learning_rate": 6.602279175335729e-05, |
|
"loss": 2.6067, |
|
"theoretical_loss": 3.375659460599226, |
|
"tokens_seen": 2434662400 |
|
}, |
|
{ |
|
"epoch": 0.87, |
|
"learning_rate": 6.543171931151883e-05, |
|
"loss": 2.5783, |
|
"theoretical_loss": 3.375295447340404, |
|
"tokens_seen": 2437939200 |
|
}, |
|
{ |
|
"epoch": 0.87, |
|
"learning_rate": 6.484064686968035e-05, |
|
"loss": 2.6045, |
|
"theoretical_loss": 3.3749320598028527, |
|
"tokens_seen": 2441216000 |
|
}, |
|
{ |
|
"epoch": 0.87, |
|
"learning_rate": 6.424957442784188e-05, |
|
"loss": 2.5871, |
|
"theoretical_loss": 3.3745692960732545, |
|
"tokens_seen": 2444492800 |
|
}, |
|
{ |
|
"epoch": 0.87, |
|
"learning_rate": 6.365850198600342e-05, |
|
"loss": 2.589, |
|
"theoretical_loss": 3.374207154246699, |
|
"tokens_seen": 2447769600 |
|
}, |
|
{ |
|
"epoch": 0.88, |
|
"learning_rate": 6.306742954416494e-05, |
|
"loss": 2.6142, |
|
"theoretical_loss": 3.373845632426633, |
|
"tokens_seen": 2451046400 |
|
}, |
|
{ |
|
"epoch": 0.88, |
|
"learning_rate": 6.247635710232646e-05, |
|
"loss": 2.5846, |
|
"theoretical_loss": 3.373484728724812, |
|
"tokens_seen": 2454323200 |
|
}, |
|
{ |
|
"epoch": 0.88, |
|
"objective/train/docs_used": 1345695, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.3688066005706787, |
|
"objective/train/theoretical_loss": 3.3731244412612527, |
|
"objective/train/tokens_used": 2478060000, |
|
"theoretical_loss": 3.3731244412612527, |
|
"tokens_seen": 2457600000 |
|
}, |
|
{ |
|
"epoch": 0.88, |
|
"learning_rate": 6.188528466048799e-05, |
|
"loss": 2.5939, |
|
"theoretical_loss": 3.3731244412612527, |
|
"tokens_seen": 2457600000 |
|
}, |
|
{ |
|
"epoch": 0.88, |
|
"learning_rate": 6.129421221864951e-05, |
|
"loss": 2.5738, |
|
"theoretical_loss": 3.3727647681641884, |
|
"tokens_seen": 2460876800 |
|
}, |
|
{ |
|
"epoch": 0.88, |
|
"learning_rate": 6.070313977681105e-05, |
|
"loss": 2.5583, |
|
"theoretical_loss": 3.3724057075700182, |
|
"tokens_seen": 2464153600 |
|
}, |
|
{ |
|
"epoch": 0.88, |
|
"learning_rate": 6.0112067334972575e-05, |
|
"loss": 2.6023, |
|
"theoretical_loss": 3.372047257623265, |
|
"tokens_seen": 2467430400 |
|
}, |
|
{ |
|
"epoch": 0.88, |
|
"learning_rate": 5.95209948931341e-05, |
|
"loss": 2.5494, |
|
"theoretical_loss": 3.3716894164765256, |
|
"tokens_seen": 2470707200 |
|
}, |
|
{ |
|
"epoch": 0.88, |
|
"learning_rate": 5.892992245129563e-05, |
|
"loss": 2.5666, |
|
"theoretical_loss": 3.371332182290427, |
|
"tokens_seen": 2473984000 |
|
}, |
|
{ |
|
"epoch": 0.88, |
|
"learning_rate": 5.8338850009457165e-05, |
|
"loss": 2.5024, |
|
"theoretical_loss": 3.37097555323358, |
|
"tokens_seen": 2477260800 |
|
}, |
|
{ |
|
"epoch": 0.89, |
|
"learning_rate": 5.774777756761869e-05, |
|
"loss": 2.5472, |
|
"theoretical_loss": 3.370619527482535, |
|
"tokens_seen": 2480537600 |
|
}, |
|
{ |
|
"epoch": 0.89, |
|
"learning_rate": 5.715670512578021e-05, |
|
"loss": 2.5807, |
|
"theoretical_loss": 3.370264103221736, |
|
"tokens_seen": 2483814400 |
|
}, |
|
{ |
|
"epoch": 0.89, |
|
"learning_rate": 5.656563268394174e-05, |
|
"loss": 2.5391, |
|
"theoretical_loss": 3.369909278643477, |
|
"tokens_seen": 2487091200 |
|
}, |
|
{ |
|
"epoch": 0.89, |
|
"learning_rate": 5.5974560242103275e-05, |
|
"loss": 2.578, |
|
"theoretical_loss": 3.369555051947857, |
|
"tokens_seen": 2490368000 |
|
}, |
|
{ |
|
"epoch": 0.89, |
|
"learning_rate": 5.53834878002648e-05, |
|
"loss": 2.5469, |
|
"theoretical_loss": 3.369201421342737, |
|
"tokens_seen": 2493644800 |
|
}, |
|
{ |
|
"epoch": 0.89, |
|
"learning_rate": 5.479241535842633e-05, |
|
"loss": 2.5474, |
|
"theoretical_loss": 3.3688483850436945, |
|
"tokens_seen": 2496921600 |
|
}, |
|
{ |
|
"epoch": 0.89, |
|
"learning_rate": 5.420134291658786e-05, |
|
"loss": 2.612, |
|
"theoretical_loss": 3.368495941273984, |
|
"tokens_seen": 2500198400 |
|
}, |
|
{ |
|
"epoch": 0.89, |
|
"learning_rate": 5.3610270474749385e-05, |
|
"loss": 2.5129, |
|
"theoretical_loss": 3.3681440882644886, |
|
"tokens_seen": 2503475200 |
|
}, |
|
{ |
|
"epoch": 0.9, |
|
"learning_rate": 5.301919803291091e-05, |
|
"loss": 2.5014, |
|
"theoretical_loss": 3.3677928242536828, |
|
"tokens_seen": 2506752000 |
|
}, |
|
{ |
|
"epoch": 0.9, |
|
"learning_rate": 5.242812559107244e-05, |
|
"loss": 2.5047, |
|
"theoretical_loss": 3.3674421474875853, |
|
"tokens_seen": 2510028800 |
|
}, |
|
{ |
|
"epoch": 0.9, |
|
"learning_rate": 5.1837053149233975e-05, |
|
"loss": 2.5355, |
|
"theoretical_loss": 3.367092056219721, |
|
"tokens_seen": 2513305600 |
|
}, |
|
{ |
|
"epoch": 0.9, |
|
"learning_rate": 5.12459807073955e-05, |
|
"loss": 2.5223, |
|
"theoretical_loss": 3.3667425487110756, |
|
"tokens_seen": 2516582400 |
|
}, |
|
{ |
|
"epoch": 0.9, |
|
"learning_rate": 5.065490826555703e-05, |
|
"loss": 2.5242, |
|
"theoretical_loss": 3.366393623230057, |
|
"tokens_seen": 2519859200 |
|
}, |
|
{ |
|
"epoch": 0.9, |
|
"learning_rate": 5.006383582371855e-05, |
|
"loss": 2.5309, |
|
"theoretical_loss": 3.366045278052451, |
|
"tokens_seen": 2523136000 |
|
}, |
|
{ |
|
"epoch": 0.9, |
|
"learning_rate": 4.9472763381880085e-05, |
|
"loss": 2.5341, |
|
"theoretical_loss": 3.365697511461384, |
|
"tokens_seen": 2526412800 |
|
}, |
|
{ |
|
"epoch": 0.9, |
|
"learning_rate": 4.888169094004161e-05, |
|
"loss": 2.5344, |
|
"theoretical_loss": 3.365350321747279, |
|
"tokens_seen": 2529689600 |
|
}, |
|
{ |
|
"epoch": 0.9, |
|
"learning_rate": 4.829061849820314e-05, |
|
"loss": 2.5221, |
|
"theoretical_loss": 3.365003707207816, |
|
"tokens_seen": 2532966400 |
|
}, |
|
{ |
|
"epoch": 0.91, |
|
"learning_rate": 4.769954605636467e-05, |
|
"loss": 2.5458, |
|
"theoretical_loss": 3.3646576661478944, |
|
"tokens_seen": 2536243200 |
|
}, |
|
{ |
|
"epoch": 0.91, |
|
"learning_rate": 4.71084736145262e-05, |
|
"loss": 2.5181, |
|
"theoretical_loss": 3.3643121968795895, |
|
"tokens_seen": 2539520000 |
|
}, |
|
{ |
|
"epoch": 0.91, |
|
"learning_rate": 4.651740117268772e-05, |
|
"loss": 2.5356, |
|
"theoretical_loss": 3.3639672977221142, |
|
"tokens_seen": 2542796800 |
|
}, |
|
{ |
|
"epoch": 0.91, |
|
"learning_rate": 4.592632873084925e-05, |
|
"loss": 2.508, |
|
"theoretical_loss": 3.3636229670017808, |
|
"tokens_seen": 2546073600 |
|
}, |
|
{ |
|
"epoch": 0.91, |
|
"learning_rate": 4.533525628901078e-05, |
|
"loss": 2.503, |
|
"theoretical_loss": 3.3632792030519605, |
|
"tokens_seen": 2549350400 |
|
}, |
|
{ |
|
"epoch": 0.91, |
|
"learning_rate": 4.474418384717231e-05, |
|
"loss": 2.5047, |
|
"theoretical_loss": 3.362936004213045, |
|
"tokens_seen": 2552627200 |
|
}, |
|
{ |
|
"epoch": 0.91, |
|
"learning_rate": 4.415311140533384e-05, |
|
"loss": 2.4909, |
|
"theoretical_loss": 3.362593368832409, |
|
"tokens_seen": 2555904000 |
|
}, |
|
{ |
|
"epoch": 0.91, |
|
"learning_rate": 4.356203896349537e-05, |
|
"loss": 2.4579, |
|
"theoretical_loss": 3.36225129526437, |
|
"tokens_seen": 2559180800 |
|
}, |
|
{ |
|
"epoch": 0.92, |
|
"learning_rate": 4.2970966521656895e-05, |
|
"loss": 2.4927, |
|
"theoretical_loss": 3.3619097818701515, |
|
"tokens_seen": 2562457600 |
|
}, |
|
{ |
|
"epoch": 0.92, |
|
"learning_rate": 4.237989407981842e-05, |
|
"loss": 2.4774, |
|
"theoretical_loss": 3.3615688270178454, |
|
"tokens_seen": 2565734400 |
|
}, |
|
{ |
|
"epoch": 0.92, |
|
"learning_rate": 4.178882163797995e-05, |
|
"loss": 2.4833, |
|
"theoretical_loss": 3.3612284290823737, |
|
"tokens_seen": 2569011200 |
|
}, |
|
{ |
|
"epoch": 0.92, |
|
"learning_rate": 4.119774919614148e-05, |
|
"loss": 2.5097, |
|
"theoretical_loss": 3.3608885864454523, |
|
"tokens_seen": 2572288000 |
|
}, |
|
{ |
|
"epoch": 0.92, |
|
"learning_rate": 4.060667675430301e-05, |
|
"loss": 2.5231, |
|
"theoretical_loss": 3.360549297495553, |
|
"tokens_seen": 2575564800 |
|
}, |
|
{ |
|
"epoch": 0.92, |
|
"learning_rate": 4.001560431246454e-05, |
|
"loss": 2.5607, |
|
"theoretical_loss": 3.3602105606278663, |
|
"tokens_seen": 2578841600 |
|
}, |
|
{ |
|
"epoch": 0.92, |
|
"learning_rate": 3.942453187062606e-05, |
|
"loss": 2.5699, |
|
"theoretical_loss": 3.359872374244266, |
|
"tokens_seen": 2582118400 |
|
}, |
|
{ |
|
"epoch": 0.92, |
|
"learning_rate": 3.883345942878759e-05, |
|
"loss": 2.5091, |
|
"theoretical_loss": 3.3595347367532744, |
|
"tokens_seen": 2585395200 |
|
}, |
|
{ |
|
"epoch": 0.92, |
|
"learning_rate": 3.824238698694912e-05, |
|
"loss": 2.494, |
|
"theoretical_loss": 3.359197646570022, |
|
"tokens_seen": 2588672000 |
|
}, |
|
{ |
|
"epoch": 0.93, |
|
"learning_rate": 3.765131454511065e-05, |
|
"loss": 2.4356, |
|
"theoretical_loss": 3.3588611021162156, |
|
"tokens_seen": 2591948800 |
|
}, |
|
{ |
|
"epoch": 0.93, |
|
"learning_rate": 3.706024210327218e-05, |
|
"loss": 2.4527, |
|
"theoretical_loss": 3.3585251018201006, |
|
"tokens_seen": 2595225600 |
|
}, |
|
{ |
|
"epoch": 0.93, |
|
"learning_rate": 3.6469169661433705e-05, |
|
"loss": 2.4592, |
|
"theoretical_loss": 3.358189644116427, |
|
"tokens_seen": 2598502400 |
|
}, |
|
{ |
|
"epoch": 0.93, |
|
"learning_rate": 3.587809721959524e-05, |
|
"loss": 2.4722, |
|
"theoretical_loss": 3.3578547274464134, |
|
"tokens_seen": 2601779200 |
|
}, |
|
{ |
|
"epoch": 0.93, |
|
"learning_rate": 3.528702477775676e-05, |
|
"loss": 2.486, |
|
"theoretical_loss": 3.357520350257713, |
|
"tokens_seen": 2605056000 |
|
}, |
|
{ |
|
"epoch": 0.93, |
|
"learning_rate": 3.469595233591829e-05, |
|
"loss": 2.4389, |
|
"theoretical_loss": 3.357186511004378, |
|
"tokens_seen": 2608332800 |
|
}, |
|
{ |
|
"epoch": 0.93, |
|
"learning_rate": 3.410487989407982e-05, |
|
"loss": 2.4608, |
|
"theoretical_loss": 3.356853208146826, |
|
"tokens_seen": 2611609600 |
|
}, |
|
{ |
|
"epoch": 0.93, |
|
"learning_rate": 3.351380745224135e-05, |
|
"loss": 2.5044, |
|
"theoretical_loss": 3.356520440151805, |
|
"tokens_seen": 2614886400 |
|
}, |
|
{ |
|
"epoch": 0.94, |
|
"learning_rate": 3.292273501040288e-05, |
|
"loss": 2.4754, |
|
"theoretical_loss": 3.3561882054923613, |
|
"tokens_seen": 2618163200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.3773267789108808, |
|
"debugging/distinct-1-grams": 0.8169355025323173, |
|
"debugging/distinct-2-grams": 0.9636260389009043, |
|
"debugging/entropy-1-grams": 5.689604031966668, |
|
"debugging/entropy-2-grams": 6.3162845189678904, |
|
"debugging/length": 477.1111111111111, |
|
"debugging/num_segments": 9, |
|
"epoch": 0.94, |
|
"objective/train/docs_used": 1436780, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.696378231048584, |
|
"objective/train/theoretical_loss": 3.3558565026478027, |
|
"objective/train/tokens_used": 2641900000, |
|
"theoretical_loss": 3.3558565026478027, |
|
"tokens_seen": 2621440000 |
|
}, |
|
{ |
|
"epoch": 0.94, |
|
"learning_rate": 3.2331662568564405e-05, |
|
"loss": 2.4694, |
|
"theoretical_loss": 3.3558565026478027, |
|
"tokens_seen": 2621440000 |
|
}, |
|
{ |
|
"epoch": 0.94, |
|
"learning_rate": 3.174059012672593e-05, |
|
"loss": 2.4595, |
|
"theoretical_loss": 3.3555253301036685, |
|
"tokens_seen": 2624716800 |
|
}, |
|
{ |
|
"epoch": 0.94, |
|
"learning_rate": 3.114951768488746e-05, |
|
"loss": 2.5305, |
|
"theoretical_loss": 3.3551946863516937, |
|
"tokens_seen": 2627993600 |
|
}, |
|
{ |
|
"epoch": 0.94, |
|
"learning_rate": 3.055844524304899e-05, |
|
"loss": 2.5193, |
|
"theoretical_loss": 3.3548645698897777, |
|
"tokens_seen": 2631270400 |
|
}, |
|
{ |
|
"epoch": 0.94, |
|
"learning_rate": 2.996737280121052e-05, |
|
"loss": 2.4535, |
|
"theoretical_loss": 3.3545349792219494, |
|
"tokens_seen": 2634547200 |
|
}, |
|
{ |
|
"epoch": 0.94, |
|
"learning_rate": 2.9376300359372043e-05, |
|
"loss": 2.4608, |
|
"theoretical_loss": 3.354205912858337, |
|
"tokens_seen": 2637824000 |
|
}, |
|
{ |
|
"epoch": 0.94, |
|
"learning_rate": 2.8785227917533574e-05, |
|
"loss": 2.4699, |
|
"theoretical_loss": 3.353877369315133, |
|
"tokens_seen": 2641100800 |
|
}, |
|
{ |
|
"epoch": 0.94, |
|
"learning_rate": 2.81941554756951e-05, |
|
"loss": 2.4735, |
|
"theoretical_loss": 3.353549347114565, |
|
"tokens_seen": 2644377600 |
|
}, |
|
{ |
|
"epoch": 0.95, |
|
"learning_rate": 2.760308303385663e-05, |
|
"loss": 2.5041, |
|
"theoretical_loss": 3.3532218447848616, |
|
"tokens_seen": 2647654400 |
|
}, |
|
{ |
|
"epoch": 0.95, |
|
"learning_rate": 2.7012010592018156e-05, |
|
"loss": 2.4662, |
|
"theoretical_loss": 3.352894860860221, |
|
"tokens_seen": 2650931200 |
|
}, |
|
{ |
|
"epoch": 0.95, |
|
"learning_rate": 2.6420938150179687e-05, |
|
"loss": 2.4468, |
|
"theoretical_loss": 3.352568393880779, |
|
"tokens_seen": 2654208000 |
|
}, |
|
{ |
|
"epoch": 0.95, |
|
"learning_rate": 2.5829865708341215e-05, |
|
"loss": 2.5261, |
|
"theoretical_loss": 3.3522424423925794, |
|
"tokens_seen": 2657484800 |
|
}, |
|
{ |
|
"epoch": 0.95, |
|
"learning_rate": 2.5238793266502742e-05, |
|
"loss": 2.4736, |
|
"theoretical_loss": 3.3519170049475413, |
|
"tokens_seen": 2660761600 |
|
}, |
|
{ |
|
"epoch": 0.95, |
|
"learning_rate": 2.4647720824664273e-05, |
|
"loss": 2.4903, |
|
"theoretical_loss": 3.351592080103429, |
|
"tokens_seen": 2664038400 |
|
}, |
|
{ |
|
"epoch": 0.95, |
|
"learning_rate": 2.4056648382825798e-05, |
|
"loss": 2.523, |
|
"theoretical_loss": 3.3512676664238192, |
|
"tokens_seen": 2667315200 |
|
}, |
|
{ |
|
"epoch": 0.95, |
|
"learning_rate": 2.346557594098733e-05, |
|
"loss": 2.5285, |
|
"theoretical_loss": 3.350943762478075, |
|
"tokens_seen": 2670592000 |
|
}, |
|
{ |
|
"epoch": 0.95, |
|
"learning_rate": 2.2874503499148856e-05, |
|
"loss": 2.5645, |
|
"theoretical_loss": 3.3506203668413095, |
|
"tokens_seen": 2673868800 |
|
}, |
|
{ |
|
"epoch": 0.96, |
|
"learning_rate": 2.2283431057310387e-05, |
|
"loss": 2.5657, |
|
"theoretical_loss": 3.3502974780943613, |
|
"tokens_seen": 2677145600 |
|
}, |
|
{ |
|
"epoch": 0.96, |
|
"learning_rate": 2.169235861547191e-05, |
|
"loss": 2.5185, |
|
"theoretical_loss": 3.34997509482376, |
|
"tokens_seen": 2680422400 |
|
}, |
|
{ |
|
"epoch": 0.96, |
|
"learning_rate": 2.1101286173633442e-05, |
|
"loss": 2.563, |
|
"theoretical_loss": 3.3496532156217005, |
|
"tokens_seen": 2683699200 |
|
}, |
|
{ |
|
"epoch": 0.96, |
|
"learning_rate": 2.051021373179497e-05, |
|
"loss": 2.5605, |
|
"theoretical_loss": 3.34933183908601, |
|
"tokens_seen": 2686976000 |
|
}, |
|
{ |
|
"epoch": 0.96, |
|
"learning_rate": 1.9919141289956497e-05, |
|
"loss": 2.557, |
|
"theoretical_loss": 3.3490109638201195, |
|
"tokens_seen": 2690252800 |
|
}, |
|
{ |
|
"epoch": 0.96, |
|
"learning_rate": 1.9328068848118025e-05, |
|
"loss": 2.5404, |
|
"theoretical_loss": 3.348690588433038, |
|
"tokens_seen": 2693529600 |
|
}, |
|
{ |
|
"epoch": 0.96, |
|
"learning_rate": 1.8736996406279556e-05, |
|
"loss": 2.5472, |
|
"theoretical_loss": 3.3483707115393164, |
|
"tokens_seen": 2696806400 |
|
}, |
|
{ |
|
"epoch": 0.96, |
|
"learning_rate": 1.815774541327785e-05, |
|
"loss": 2.5388, |
|
"theoretical_loss": 3.3480513317590264, |
|
"tokens_seen": 2700083200 |
|
}, |
|
{ |
|
"epoch": 0.97, |
|
"learning_rate": 1.756667297143938e-05, |
|
"loss": 2.534, |
|
"theoretical_loss": 3.347732447717727, |
|
"tokens_seen": 2703360000 |
|
}, |
|
{ |
|
"epoch": 0.97, |
|
"learning_rate": 1.697560052960091e-05, |
|
"loss": 2.5488, |
|
"theoretical_loss": 3.347414058046437, |
|
"tokens_seen": 2706636800 |
|
}, |
|
{ |
|
"epoch": 0.97, |
|
"learning_rate": 1.6384528087762434e-05, |
|
"loss": 2.5082, |
|
"theoretical_loss": 3.3470961613816086, |
|
"tokens_seen": 2709913600 |
|
}, |
|
{ |
|
"epoch": 0.97, |
|
"learning_rate": 1.5793455645923965e-05, |
|
"loss": 2.5507, |
|
"theoretical_loss": 3.3467787563650955, |
|
"tokens_seen": 2713190400 |
|
}, |
|
{ |
|
"epoch": 0.97, |
|
"learning_rate": 1.5202383204085493e-05, |
|
"loss": 2.4918, |
|
"theoretical_loss": 3.34646184164413, |
|
"tokens_seen": 2716467200 |
|
}, |
|
{ |
|
"epoch": 0.97, |
|
"learning_rate": 1.4611310762247022e-05, |
|
"loss": 2.5285, |
|
"theoretical_loss": 3.346145415871291, |
|
"tokens_seen": 2719744000 |
|
}, |
|
{ |
|
"epoch": 0.97, |
|
"learning_rate": 1.402023832040855e-05, |
|
"loss": 2.5143, |
|
"theoretical_loss": 3.345829477704478, |
|
"tokens_seen": 2723020800 |
|
}, |
|
{ |
|
"epoch": 0.97, |
|
"learning_rate": 1.3429165878570077e-05, |
|
"loss": 2.5145, |
|
"theoretical_loss": 3.345514025806884, |
|
"tokens_seen": 2726297600 |
|
}, |
|
{ |
|
"epoch": 0.97, |
|
"learning_rate": 1.2838093436731606e-05, |
|
"loss": 2.5162, |
|
"theoretical_loss": 3.3451990588469687, |
|
"tokens_seen": 2729574400 |
|
}, |
|
{ |
|
"epoch": 0.98, |
|
"learning_rate": 1.2247020994893134e-05, |
|
"loss": 2.4997, |
|
"theoretical_loss": 3.3448845754984307, |
|
"tokens_seen": 2732851200 |
|
}, |
|
{ |
|
"epoch": 0.98, |
|
"learning_rate": 1.1655948553054663e-05, |
|
"loss": 2.5332, |
|
"theoretical_loss": 3.344570574440179, |
|
"tokens_seen": 2736128000 |
|
}, |
|
{ |
|
"epoch": 0.98, |
|
"learning_rate": 1.106487611121619e-05, |
|
"loss": 2.5341, |
|
"theoretical_loss": 3.3442570543563104, |
|
"tokens_seen": 2739404800 |
|
}, |
|
{ |
|
"epoch": 0.98, |
|
"learning_rate": 1.0473803669377718e-05, |
|
"loss": 2.5641, |
|
"theoretical_loss": 3.34394401393608, |
|
"tokens_seen": 2742681600 |
|
}, |
|
{ |
|
"epoch": 0.98, |
|
"learning_rate": 9.882731227539248e-06, |
|
"loss": 2.5486, |
|
"theoretical_loss": 3.343631451873874, |
|
"tokens_seen": 2745958400 |
|
}, |
|
{ |
|
"epoch": 0.98, |
|
"learning_rate": 9.291658785700775e-06, |
|
"loss": 2.5316, |
|
"theoretical_loss": 3.343319366869187, |
|
"tokens_seen": 2749235200 |
|
}, |
|
{ |
|
"epoch": 0.98, |
|
"learning_rate": 8.700586343862303e-06, |
|
"loss": 2.5689, |
|
"theoretical_loss": 3.343007757626593, |
|
"tokens_seen": 2752512000 |
|
}, |
|
{ |
|
"epoch": 0.98, |
|
"learning_rate": 8.109513902023834e-06, |
|
"loss": 2.5272, |
|
"theoretical_loss": 3.34269662285572, |
|
"tokens_seen": 2755788800 |
|
}, |
|
{ |
|
"epoch": 0.99, |
|
"learning_rate": 7.51844146018536e-06, |
|
"loss": 2.5221, |
|
"theoretical_loss": 3.342385961271226, |
|
"tokens_seen": 2759065600 |
|
}, |
|
{ |
|
"epoch": 0.99, |
|
"learning_rate": 6.927369018346889e-06, |
|
"loss": 2.5087, |
|
"theoretical_loss": 3.3420757715927705, |
|
"tokens_seen": 2762342400 |
|
}, |
|
{ |
|
"epoch": 0.99, |
|
"learning_rate": 6.336296576508416e-06, |
|
"loss": 2.5083, |
|
"theoretical_loss": 3.341766052544993, |
|
"tokens_seen": 2765619200 |
|
}, |
|
{ |
|
"epoch": 0.99, |
|
"learning_rate": 5.745224134669946e-06, |
|
"loss": 2.5091, |
|
"theoretical_loss": 3.341456802857484, |
|
"tokens_seen": 2768896000 |
|
}, |
|
{ |
|
"epoch": 0.99, |
|
"learning_rate": 5.154151692831474e-06, |
|
"loss": 2.476, |
|
"theoretical_loss": 3.341148021264762, |
|
"tokens_seen": 2772172800 |
|
}, |
|
{ |
|
"epoch": 0.99, |
|
"learning_rate": 4.563079250993002e-06, |
|
"loss": 2.4923, |
|
"theoretical_loss": 3.3408397065062485, |
|
"tokens_seen": 2775449600 |
|
}, |
|
{ |
|
"epoch": 0.99, |
|
"learning_rate": 3.97200680915453e-06, |
|
"loss": 2.5254, |
|
"theoretical_loss": 3.3405318573262424, |
|
"tokens_seen": 2778726400 |
|
}, |
|
{ |
|
"epoch": 0.99, |
|
"learning_rate": 3.3809343673160584e-06, |
|
"loss": 2.4967, |
|
"theoretical_loss": 3.3402244724738965, |
|
"tokens_seen": 2782003200 |
|
}, |
|
{ |
|
"epoch": 0.99, |
|
"objective/train/docs_used": 1527817, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.387871265411377, |
|
"objective/train/theoretical_loss": 3.3399175507031913, |
|
"objective/train/tokens_used": 2805740000, |
|
"theoretical_loss": 3.3399175507031913, |
|
"tokens_seen": 2785280000 |
|
}, |
|
{ |
|
"epoch": 0.99, |
|
"learning_rate": 2.789861925477587e-06, |
|
"loss": 2.4951, |
|
"theoretical_loss": 3.3399175507031913, |
|
"tokens_seen": 2785280000 |
|
}, |
|
{ |
|
"epoch": 1.0, |
|
"learning_rate": 2.198789483639115e-06, |
|
"loss": 2.5301, |
|
"theoretical_loss": 3.3396110907729133, |
|
"tokens_seen": 2788556800 |
|
}, |
|
{ |
|
"epoch": 1.0, |
|
"learning_rate": 1.607717041800643e-06, |
|
"loss": 2.5492, |
|
"theoretical_loss": 3.3393050914466285, |
|
"tokens_seen": 2791833600 |
|
}, |
|
{ |
|
"epoch": 1.0, |
|
"learning_rate": 1.0166445999621714e-06, |
|
"loss": 2.5805, |
|
"theoretical_loss": 3.33899955149266, |
|
"tokens_seen": 2795110400 |
|
}, |
|
{ |
|
"epoch": 1.0, |
|
"learning_rate": 4.255721581236997e-07, |
|
"loss": 2.5805, |
|
"theoretical_loss": 3.338694469684062, |
|
"tokens_seen": 2798387200 |
|
} |
|
], |
|
"max_steps": 42724, |
|
"num_train_epochs": 9223372036854775807, |
|
"total_flos": 1.428920419221504e+18, |
|
"trial_name": null, |
|
"trial_params": null |
|
} |
|
|