|
{ |
|
"best_metric": null, |
|
"best_model_checkpoint": null, |
|
"epoch": 0.5, |
|
"global_step": 21362, |
|
"is_hyper_param_search": false, |
|
"is_local_process_zero": true, |
|
"is_world_process_zero": true, |
|
"log_history": [ |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 1.1682242990654204e-06, |
|
"loss": 10.8813, |
|
"theoretical_loss": 20.81281176760504, |
|
"tokens_seen": 65536 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 5.841121495327103e-05, |
|
"loss": 8.9731, |
|
"theoretical_loss": 8.563476630668958, |
|
"tokens_seen": 3276800 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 0.00011682242990654206, |
|
"loss": 6.803, |
|
"theoretical_loss": 7.477752684105921, |
|
"tokens_seen": 6553600 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 0.00017523364485981307, |
|
"loss": 5.89, |
|
"theoretical_loss": 6.9337484549527915, |
|
"tokens_seen": 9830400 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 0.00023364485981308412, |
|
"loss": 5.4842, |
|
"theoretical_loss": 6.5835601944843045, |
|
"tokens_seen": 13107200 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.00029205607476635517, |
|
"loss": 5.2294, |
|
"theoretical_loss": 6.3307075311739744, |
|
"tokens_seen": 16384000 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.00035046728971962614, |
|
"loss": 5.0511, |
|
"theoretical_loss": 6.135523197998216, |
|
"tokens_seen": 19660800 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0004088785046728972, |
|
"loss": 4.9053, |
|
"theoretical_loss": 5.978095549927499, |
|
"tokens_seen": 22937600 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.00046728971962616824, |
|
"loss": 4.8028, |
|
"theoretical_loss": 5.847111292323815, |
|
"tokens_seen": 26214400 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.000499739928125591, |
|
"loss": 4.7268, |
|
"theoretical_loss": 5.735570781940016, |
|
"tokens_seen": 29491200 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0004991488556837526, |
|
"loss": 4.5965, |
|
"theoretical_loss": 5.638864110129244, |
|
"tokens_seen": 32768000 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0004985577832419141, |
|
"loss": 4.4507, |
|
"theoretical_loss": 5.553806347902798, |
|
"tokens_seen": 36044800 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0004979667108000757, |
|
"loss": 4.3796, |
|
"theoretical_loss": 5.478112046614329, |
|
"tokens_seen": 39321600 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004973756383582371, |
|
"loss": 4.2971, |
|
"theoretical_loss": 5.410089925637252, |
|
"tokens_seen": 42598400 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004967845659163987, |
|
"loss": 4.1952, |
|
"theoretical_loss": 5.348456049793725, |
|
"tokens_seen": 45875200 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004961934934745603, |
|
"loss": 4.1713, |
|
"theoretical_loss": 5.292214532995457, |
|
"tokens_seen": 49152000 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004956024210327218, |
|
"loss": 4.065, |
|
"theoretical_loss": 5.240578591827869, |
|
"tokens_seen": 52428800 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004950113485908833, |
|
"loss": 3.9725, |
|
"theoretical_loss": 5.192916690583679, |
|
"tokens_seen": 55705600 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004944202761490448, |
|
"loss": 3.9811, |
|
"theoretical_loss": 5.148714829414331, |
|
"tokens_seen": 58982400 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004938292037072064, |
|
"loss": 3.8862, |
|
"theoretical_loss": 5.107549528462992, |
|
"tokens_seen": 62259200 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004932381312653678, |
|
"loss": 3.7569, |
|
"theoretical_loss": 5.069068083201136, |
|
"tokens_seen": 65536000 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004926470588235294, |
|
"loss": 3.6979, |
|
"theoretical_loss": 5.032973875895897, |
|
"tokens_seen": 68812800 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.000492055986381691, |
|
"loss": 3.5973, |
|
"theoretical_loss": 4.999015274282555, |
|
"tokens_seen": 72089600 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004914649139398525, |
|
"loss": 3.5246, |
|
"theoretical_loss": 4.966977121409853, |
|
"tokens_seen": 75366400 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.000490873841498014, |
|
"loss": 3.5081, |
|
"theoretical_loss": 4.936674127683027, |
|
"tokens_seen": 78643200 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004902827690561755, |
|
"loss": 3.4698, |
|
"theoretical_loss": 4.907945679887972, |
|
"tokens_seen": 81920000 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004896916966143371, |
|
"loss": 3.4115, |
|
"theoretical_loss": 4.8806517198708175, |
|
"tokens_seen": 85196800 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004891006241724985, |
|
"loss": 3.4193, |
|
"theoretical_loss": 4.85466944053967, |
|
"tokens_seen": 88473600 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004885095517306601, |
|
"loss": 3.3934, |
|
"theoretical_loss": 4.829890613366704, |
|
"tokens_seen": 91750400 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004879184792888217, |
|
"loss": 3.35, |
|
"theoretical_loss": 4.806219408835812, |
|
"tokens_seen": 95027200 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0004873274068469832, |
|
"loss": 3.2861, |
|
"theoretical_loss": 4.783570605334148, |
|
"tokens_seen": 98304000 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.00048673633440514467, |
|
"loss": 3.2664, |
|
"theoretical_loss": 4.761868106830299, |
|
"tokens_seen": 101580800 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0004861452619633062, |
|
"loss": 3.3073, |
|
"theoretical_loss": 4.741043708020364, |
|
"tokens_seen": 104857600 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0004855541895214677, |
|
"loss": 3.2906, |
|
"theoretical_loss": 4.721036059306941, |
|
"tokens_seen": 108134400 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0004849631170796293, |
|
"loss": 3.2249, |
|
"theoretical_loss": 4.701789794289756, |
|
"tokens_seen": 111411200 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0004843720446377908, |
|
"loss": 3.2799, |
|
"theoretical_loss": 4.68325479029382, |
|
"tokens_seen": 114688000 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.00048378097219595233, |
|
"loss": 3.229, |
|
"theoretical_loss": 4.6653855384841725, |
|
"tokens_seen": 117964800 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.00048318989975411385, |
|
"loss": 3.2623, |
|
"theoretical_loss": 4.6481406047776295, |
|
"tokens_seen": 121241600 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.00048259882731227537, |
|
"loss": 3.2475, |
|
"theoretical_loss": 4.631482166397534, |
|
"tokens_seen": 124518400 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0004820077548704369, |
|
"loss": 3.1819, |
|
"theoretical_loss": 4.615375611773608, |
|
"tokens_seen": 127795200 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.00048141668242859847, |
|
"loss": 3.2083, |
|
"theoretical_loss": 4.5997891937483955, |
|
"tokens_seen": 131072000 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.00048082560998676, |
|
"loss": 3.189, |
|
"theoretical_loss": 4.584693727850565, |
|
"tokens_seen": 134348800 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0004802345375449215, |
|
"loss": 3.1967, |
|
"theoretical_loss": 4.570062328836407, |
|
"tokens_seen": 137625600 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.00047964346510308303, |
|
"loss": 3.1635, |
|
"theoretical_loss": 4.5558701798619285, |
|
"tokens_seen": 140902400 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.00047905239266124455, |
|
"loss": 3.1347, |
|
"theoretical_loss": 4.542094329588689, |
|
"tokens_seen": 144179200 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.00047846132021940607, |
|
"loss": 3.1606, |
|
"theoretical_loss": 4.528713513292708, |
|
"tokens_seen": 147456000 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0004778702477775676, |
|
"loss": 3.0966, |
|
"theoretical_loss": 4.515707994672887, |
|
"tokens_seen": 150732800 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.00047727917533572917, |
|
"loss": 3.1029, |
|
"theoretical_loss": 4.503059425571229, |
|
"tokens_seen": 154009600 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0004766881028938907, |
|
"loss": 3.1398, |
|
"theoretical_loss": 4.490750721243157, |
|
"tokens_seen": 157286400 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0004760970304520522, |
|
"loss": 3.0796, |
|
"theoretical_loss": 4.478765949169858, |
|
"tokens_seen": 160563200 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"objective/train/docs_used": 96518, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 3.2286458015441895, |
|
"objective/train/theoretical_loss": 4.46709022969911, |
|
"objective/train/tokens_used": 184300000, |
|
"theoretical_loss": 4.46709022969911, |
|
"tokens_seen": 163840000 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.00047550595801021373, |
|
"loss": 3.0725, |
|
"theoretical_loss": 4.46709022969911, |
|
"tokens_seen": 163840000 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.00047491488556837525, |
|
"loss": 3.0736, |
|
"theoretical_loss": 4.455709647047437, |
|
"tokens_seen": 167116800 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.00047432381312653677, |
|
"loss": 3.0957, |
|
"theoretical_loss": 4.444611169403321, |
|
"tokens_seen": 170393600 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.00047373274068469835, |
|
"loss": 3.0505, |
|
"theoretical_loss": 4.4337825770455375, |
|
"tokens_seen": 173670400 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.00047314166824285987, |
|
"loss": 3.0771, |
|
"theoretical_loss": 4.423212397538051, |
|
"tokens_seen": 176947200 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0004725505958010214, |
|
"loss": 3.0805, |
|
"theoretical_loss": 4.4128898471880325, |
|
"tokens_seen": 180224000 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0004719595233591829, |
|
"loss": 3.0562, |
|
"theoretical_loss": 4.40280477805997, |
|
"tokens_seen": 183500800 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.00047136845091734443, |
|
"loss": 3.0368, |
|
"theoretical_loss": 4.392947629929752, |
|
"tokens_seen": 186777600 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.00047077737847550595, |
|
"loss": 3.0456, |
|
"theoretical_loss": 4.383309386640423, |
|
"tokens_seen": 190054400 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0004701863060336675, |
|
"loss": 3.0098, |
|
"theoretical_loss": 4.373881536388167, |
|
"tokens_seen": 193331200 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.00046959523359182905, |
|
"loss": 3.0174, |
|
"theoretical_loss": 4.364656035524595, |
|
"tokens_seen": 196608000 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.00046900416114999057, |
|
"loss": 3.0129, |
|
"theoretical_loss": 4.355625275511174, |
|
"tokens_seen": 199884800 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0004684130887081521, |
|
"loss": 2.9953, |
|
"theoretical_loss": 4.346782052704563, |
|
"tokens_seen": 203161600 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0004678220162663136, |
|
"loss": 2.9867, |
|
"theoretical_loss": 4.338119540689052, |
|
"tokens_seen": 206438400 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.00046723094382447513, |
|
"loss": 3.0165, |
|
"theoretical_loss": 4.329631264904703, |
|
"tokens_seen": 209715200 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.00046663987138263665, |
|
"loss": 3.0017, |
|
"theoretical_loss": 4.321311079348144, |
|
"tokens_seen": 212992000 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0004660487989407982, |
|
"loss": 2.9951, |
|
"theoretical_loss": 4.313153145147683, |
|
"tokens_seen": 216268800 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.00046545772649895975, |
|
"loss": 2.9689, |
|
"theoretical_loss": 4.305151910836119, |
|
"tokens_seen": 219545600 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.00046486665405712127, |
|
"loss": 2.9826, |
|
"theoretical_loss": 4.2973020941635784, |
|
"tokens_seen": 222822400 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0004642755816152828, |
|
"loss": 2.9874, |
|
"theoretical_loss": 4.28959866530949, |
|
"tokens_seen": 226099200 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0004636845091734443, |
|
"loss": 2.9269, |
|
"theoretical_loss": 4.282036831367506, |
|
"tokens_seen": 229376000 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.00046309343673160583, |
|
"loss": 2.9501, |
|
"theoretical_loss": 4.274612021990189, |
|
"tokens_seen": 232652800 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0004625023642897674, |
|
"loss": 2.9073, |
|
"theoretical_loss": 4.267319876091788, |
|
"tokens_seen": 235929600 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004619112918479289, |
|
"loss": 2.9524, |
|
"theoretical_loss": 4.260156229517635, |
|
"tokens_seen": 239206400 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.00046132021940609044, |
|
"loss": 2.9937, |
|
"theoretical_loss": 4.253117103597704, |
|
"tokens_seen": 242483200 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.00046072914696425197, |
|
"loss": 2.9917, |
|
"theoretical_loss": 4.246198694509945, |
|
"tokens_seen": 245760000 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004601380745224135, |
|
"loss": 2.967, |
|
"theoretical_loss": 4.239397363386152, |
|
"tokens_seen": 249036800 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.000459547002080575, |
|
"loss": 2.9536, |
|
"theoretical_loss": 4.232709627099522, |
|
"tokens_seen": 252313600 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004589559296387366, |
|
"loss": 2.9099, |
|
"theoretical_loss": 4.226132149678757, |
|
"tokens_seen": 255590400 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004583648571968981, |
|
"loss": 2.9495, |
|
"theoretical_loss": 4.219661734298666, |
|
"tokens_seen": 258867200 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004577737847550596, |
|
"loss": 2.9228, |
|
"theoretical_loss": 4.213295315801815, |
|
"tokens_seen": 262144000 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004571827123132211, |
|
"loss": 2.914, |
|
"theoretical_loss": 4.207029953709861, |
|
"tokens_seen": 265420800 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0004565916398713826, |
|
"loss": 2.9065, |
|
"theoretical_loss": 4.200862825686893, |
|
"tokens_seen": 268697600 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.00045600056742954413, |
|
"loss": 2.9208, |
|
"theoretical_loss": 4.19479122142044, |
|
"tokens_seen": 271974400 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0004554094949877057, |
|
"loss": 2.8755, |
|
"theoretical_loss": 4.188812536888775, |
|
"tokens_seen": 275251200 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.00045481842254586723, |
|
"loss": 2.911, |
|
"theoretical_loss": 4.182924268985855, |
|
"tokens_seen": 278528000 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.00045422735010402875, |
|
"loss": 2.8592, |
|
"theoretical_loss": 4.177124010477671, |
|
"tokens_seen": 281804800 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.00045363627766219027, |
|
"loss": 2.9063, |
|
"theoretical_loss": 4.171409445265983, |
|
"tokens_seen": 285081600 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0004530452052203518, |
|
"loss": 2.8809, |
|
"theoretical_loss": 4.165778343937409, |
|
"tokens_seen": 288358400 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0004524541327785133, |
|
"loss": 2.8394, |
|
"theoretical_loss": 4.160228559577659, |
|
"tokens_seen": 291635200 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.00045186306033667483, |
|
"loss": 2.8306, |
|
"theoretical_loss": 4.15475802383233, |
|
"tokens_seen": 294912000 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0004512719878948364, |
|
"loss": 2.8102, |
|
"theoretical_loss": 4.149364743197177, |
|
"tokens_seen": 298188800 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.00045068091545299793, |
|
"loss": 2.8491, |
|
"theoretical_loss": 4.14404679552214, |
|
"tokens_seen": 301465600 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.00045008984301115945, |
|
"loss": 2.8152, |
|
"theoretical_loss": 4.138802326714632, |
|
"tokens_seen": 304742400 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.00044949877056932097, |
|
"loss": 2.8902, |
|
"theoretical_loss": 4.133629547628726, |
|
"tokens_seen": 308019200 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0004489076981274825, |
|
"loss": 2.8874, |
|
"theoretical_loss": 4.128526731127894, |
|
"tokens_seen": 311296000 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.000448316625685644, |
|
"loss": 2.8282, |
|
"theoretical_loss": 4.123492209309923, |
|
"tokens_seen": 314572800 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0004477255532438056, |
|
"loss": 2.8807, |
|
"theoretical_loss": 4.118524370883447, |
|
"tokens_seen": 317849600 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0004471344808019671, |
|
"loss": 2.8404, |
|
"theoretical_loss": 4.113621658686355, |
|
"tokens_seen": 321126400 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.00044654340836012863, |
|
"loss": 2.8468, |
|
"theoretical_loss": 4.108782567337039, |
|
"tokens_seen": 324403200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.5179236306635556, |
|
"debugging/distinct-1-grams": 0.7868809037747271, |
|
"debugging/distinct-2-grams": 0.95444323584591, |
|
"debugging/entropy-1-grams": 5.810297153205875, |
|
"debugging/entropy-2-grams": 6.5112113932041344, |
|
"debugging/length": 540.5, |
|
"debugging/num_segments": 12, |
|
"epoch": 0.12, |
|
"objective/train/docs_used": 182435, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.7323758602142334, |
|
"objective/train/theoretical_loss": 4.104005641010112, |
|
"objective/train/tokens_used": 348140000, |
|
"theoretical_loss": 4.104005641010112, |
|
"tokens_seen": 327680000 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.00044595233591829015, |
|
"loss": 2.8315, |
|
"theoretical_loss": 4.104005641010112, |
|
"tokens_seen": 327680000 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.00044536126347645167, |
|
"loss": 2.7719, |
|
"theoretical_loss": 4.099289471328812, |
|
"tokens_seen": 330956800 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0004447701910346132, |
|
"loss": 2.8239, |
|
"theoretical_loss": 4.094632695366921, |
|
"tokens_seen": 334233600 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.00044417911859277476, |
|
"loss": 2.7712, |
|
"theoretical_loss": 4.090033993753448, |
|
"tokens_seen": 337510400 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0004435880461509363, |
|
"loss": 2.7701, |
|
"theoretical_loss": 4.085492088873883, |
|
"tokens_seen": 340787200 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0004429969737090978, |
|
"loss": 2.7771, |
|
"theoretical_loss": 4.081005743162224, |
|
"tokens_seen": 344064000 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0004424059012672593, |
|
"loss": 2.7947, |
|
"theoretical_loss": 4.076573757478361, |
|
"tokens_seen": 347340800 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.00044181482882542085, |
|
"loss": 2.7985, |
|
"theoretical_loss": 4.072194969565807, |
|
"tokens_seen": 350617600 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.00044122375638358237, |
|
"loss": 2.8102, |
|
"theoretical_loss": 4.067868252585089, |
|
"tokens_seen": 353894400 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.0004406326839417439, |
|
"loss": 2.8096, |
|
"theoretical_loss": 4.063592513718411, |
|
"tokens_seen": 357171200 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.00044004161149990546, |
|
"loss": 2.7988, |
|
"theoretical_loss": 4.059366692841521, |
|
"tokens_seen": 360448000 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.000439450539058067, |
|
"loss": 2.8074, |
|
"theoretical_loss": 4.055189761258959, |
|
"tokens_seen": 363724800 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.0004388594666162285, |
|
"loss": 2.7982, |
|
"theoretical_loss": 4.051060720499127, |
|
"tokens_seen": 367001600 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.00043826839417439, |
|
"loss": 2.7748, |
|
"theoretical_loss": 4.046978601165831, |
|
"tokens_seen": 370278400 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.00043767732173255155, |
|
"loss": 2.7677, |
|
"theoretical_loss": 4.042942461843204, |
|
"tokens_seen": 373555200 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.00043708624929071307, |
|
"loss": 2.7828, |
|
"theoretical_loss": 4.038951388051044, |
|
"tokens_seen": 376832000 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.00043649517684887464, |
|
"loss": 2.7974, |
|
"theoretical_loss": 4.035004491247873, |
|
"tokens_seen": 380108800 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.00043590410440703616, |
|
"loss": 2.7766, |
|
"theoretical_loss": 4.031100907879109, |
|
"tokens_seen": 383385600 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0004353130319651977, |
|
"loss": 2.8236, |
|
"theoretical_loss": 4.02723979846797, |
|
"tokens_seen": 386662400 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0004347219595233592, |
|
"loss": 2.829, |
|
"theoretical_loss": 4.023420346746835, |
|
"tokens_seen": 389939200 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0004341308870815207, |
|
"loss": 2.7938, |
|
"theoretical_loss": 4.019641758826938, |
|
"tokens_seen": 393216000 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.00043353981463968225, |
|
"loss": 2.7822, |
|
"theoretical_loss": 4.015903262404413, |
|
"tokens_seen": 396492800 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0004329487421978438, |
|
"loss": 2.7694, |
|
"theoretical_loss": 4.012204106000786, |
|
"tokens_seen": 399769600 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.00043235766975600534, |
|
"loss": 2.7842, |
|
"theoretical_loss": 4.008543558236181, |
|
"tokens_seen": 403046400 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.00043176659731416686, |
|
"loss": 2.7663, |
|
"theoretical_loss": 4.004920907133565, |
|
"tokens_seen": 406323200 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0004311755248723284, |
|
"loss": 2.7278, |
|
"theoretical_loss": 4.001335459452449, |
|
"tokens_seen": 409600000 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0004305844524304899, |
|
"loss": 2.7589, |
|
"theoretical_loss": 3.997786540050617, |
|
"tokens_seen": 412876800 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0004299933799886514, |
|
"loss": 2.7719, |
|
"theoretical_loss": 3.9942734912724456, |
|
"tokens_seen": 416153600 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.00042940230754681295, |
|
"loss": 2.7241, |
|
"theoretical_loss": 3.9907956723625375, |
|
"tokens_seen": 419430400 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0004288112351049745, |
|
"loss": 2.7718, |
|
"theoretical_loss": 3.9873524589034224, |
|
"tokens_seen": 422707200 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.000428220162663136, |
|
"loss": 2.765, |
|
"theoretical_loss": 3.9839432422761556, |
|
"tokens_seen": 425984000 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0004276290902212975, |
|
"loss": 2.804, |
|
"theoretical_loss": 3.980567429142721, |
|
"tokens_seen": 429260800 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.00042703801777945903, |
|
"loss": 2.7606, |
|
"theoretical_loss": 3.977224440949197, |
|
"tokens_seen": 432537600 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.00042644694533762055, |
|
"loss": 2.772, |
|
"theoretical_loss": 3.9739137134486917, |
|
"tokens_seen": 435814400 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.00042585587289578207, |
|
"loss": 2.8015, |
|
"theoretical_loss": 3.9706346962431396, |
|
"tokens_seen": 439091200 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.00042526480045394365, |
|
"loss": 2.8007, |
|
"theoretical_loss": 3.9673868523430564, |
|
"tokens_seen": 442368000 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.00042467372801210517, |
|
"loss": 2.7835, |
|
"theoretical_loss": 3.9641696577444376, |
|
"tokens_seen": 445644800 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.0004240826555702667, |
|
"loss": 2.7811, |
|
"theoretical_loss": 3.9609826010220033, |
|
"tokens_seen": 448921600 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.0004234915831284282, |
|
"loss": 2.8235, |
|
"theoretical_loss": 3.9578251829380506, |
|
"tokens_seen": 452198400 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.00042290051068658973, |
|
"loss": 2.7819, |
|
"theoretical_loss": 3.954696916066199, |
|
"tokens_seen": 455475200 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.00042230943824475125, |
|
"loss": 2.7632, |
|
"theoretical_loss": 3.9515973244293643, |
|
"tokens_seen": 458752000 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0004217183658029128, |
|
"loss": 2.7585, |
|
"theoretical_loss": 3.948525943151326, |
|
"tokens_seen": 462028800 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.00042112729336107435, |
|
"loss": 2.7676, |
|
"theoretical_loss": 3.9454823181212815, |
|
"tokens_seen": 465305600 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.00042053622091923587, |
|
"loss": 2.7674, |
|
"theoretical_loss": 3.9424660056708167, |
|
"tokens_seen": 468582400 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0004199451484773974, |
|
"loss": 2.7598, |
|
"theoretical_loss": 3.939476572262754, |
|
"tokens_seen": 471859200 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0004193540760355589, |
|
"loss": 2.761, |
|
"theoretical_loss": 3.9365135941913563, |
|
"tokens_seen": 475136000 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.00041876300359372043, |
|
"loss": 2.7423, |
|
"theoretical_loss": 3.9335766572934023, |
|
"tokens_seen": 478412800 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.00041817193115188195, |
|
"loss": 2.7487, |
|
"theoretical_loss": 3.9306653566696603, |
|
"tokens_seen": 481689600 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0004175808587100435, |
|
"loss": 2.7297, |
|
"theoretical_loss": 3.927779296416332, |
|
"tokens_seen": 484966400 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.00041698978626820505, |
|
"loss": 2.7348, |
|
"theoretical_loss": 3.924918089366024, |
|
"tokens_seen": 488243200 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"objective/train/docs_used": 267104, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.6938159465789795, |
|
"objective/train/theoretical_loss": 3.9220813568378707, |
|
"objective/train/tokens_used": 511980000, |
|
"theoretical_loss": 3.9220813568378707, |
|
"tokens_seen": 491520000 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.00041639871382636657, |
|
"loss": 2.6909, |
|
"theoretical_loss": 3.9220813568378707, |
|
"tokens_seen": 491520000 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0004158076413845281, |
|
"loss": 2.6957, |
|
"theoretical_loss": 3.9192687283964096, |
|
"tokens_seen": 494796800 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0004152165689426896, |
|
"loss": 2.7163, |
|
"theoretical_loss": 3.9164798416188527, |
|
"tokens_seen": 498073600 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.00041462549650085113, |
|
"loss": 2.6846, |
|
"theoretical_loss": 3.913714341870409, |
|
"tokens_seen": 501350400 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0004140344240590127, |
|
"loss": 2.6939, |
|
"theoretical_loss": 3.9109718820873303, |
|
"tokens_seen": 504627200 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0004134433516171742, |
|
"loss": 2.7024, |
|
"theoretical_loss": 3.9082521225673625, |
|
"tokens_seen": 507904000 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.00041285227917533575, |
|
"loss": 2.6622, |
|
"theoretical_loss": 3.9055547307673075, |
|
"tokens_seen": 511180800 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.00041226120673349727, |
|
"loss": 2.7267, |
|
"theoretical_loss": 3.9028793811074056, |
|
"tokens_seen": 514457600 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0004116701342916588, |
|
"loss": 2.7517, |
|
"theoretical_loss": 3.900225754782274, |
|
"tokens_seen": 517734400 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0004110790618498203, |
|
"loss": 2.7289, |
|
"theoretical_loss": 3.897593539578138, |
|
"tokens_seen": 521011200 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0004104879894079819, |
|
"loss": 2.7181, |
|
"theoretical_loss": 3.8949824296961015, |
|
"tokens_seen": 524288000 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0004098969169661434, |
|
"loss": 2.7306, |
|
"theoretical_loss": 3.8923921255812353, |
|
"tokens_seen": 527564800 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0004093058445243049, |
|
"loss": 2.7015, |
|
"theoretical_loss": 3.8898223337572393, |
|
"tokens_seen": 530841600 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.00040871477208246645, |
|
"loss": 2.6978, |
|
"theoretical_loss": 3.88727276666648, |
|
"tokens_seen": 534118400 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.00040812369964062797, |
|
"loss": 2.6723, |
|
"theoretical_loss": 3.884743142515184, |
|
"tokens_seen": 537395200 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0004075326271987895, |
|
"loss": 2.6872, |
|
"theoretical_loss": 3.8822331851235985, |
|
"tokens_seen": 540672000 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.000406941554756951, |
|
"loss": 2.6913, |
|
"theoretical_loss": 3.87974262378093, |
|
"tokens_seen": 543948800 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0004063504823151126, |
|
"loss": 2.6513, |
|
"theoretical_loss": 3.877271193104873, |
|
"tokens_seen": 547225600 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0004057594098732741, |
|
"loss": 2.6908, |
|
"theoretical_loss": 3.8748186329055736, |
|
"tokens_seen": 550502400 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0004051801588802724, |
|
"loss": 2.7044, |
|
"theoretical_loss": 3.87238468805384, |
|
"tokens_seen": 553779200 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0004045890864384339, |
|
"loss": 2.6751, |
|
"theoretical_loss": 3.8699691083534633, |
|
"tokens_seen": 557056000 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0004039980139965954, |
|
"loss": 2.6641, |
|
"theoretical_loss": 3.8675716484174907, |
|
"tokens_seen": 560332800 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.00040340694155475694, |
|
"loss": 2.6713, |
|
"theoretical_loss": 3.8651920675482936, |
|
"tokens_seen": 563609600 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0004028158691129185, |
|
"loss": 2.6941, |
|
"theoretical_loss": 3.862830129621318, |
|
"tokens_seen": 566886400 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.00040222479667108003, |
|
"loss": 2.7425, |
|
"theoretical_loss": 3.8604856029723575, |
|
"tokens_seen": 570163200 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.00040163372422924155, |
|
"loss": 2.6816, |
|
"theoretical_loss": 3.8581582602882447, |
|
"tokens_seen": 573440000 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0004010426517874031, |
|
"loss": 2.7, |
|
"theoretical_loss": 3.8558478785008203, |
|
"tokens_seen": 576716800 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0004004515793455646, |
|
"loss": 2.7188, |
|
"theoretical_loss": 3.8535542386840778, |
|
"tokens_seen": 579993600 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0003998605069037261, |
|
"loss": 2.714, |
|
"theoretical_loss": 3.8512771259543586, |
|
"tokens_seen": 583270400 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0003992694344618877, |
|
"loss": 2.7144, |
|
"theoretical_loss": 3.8490163293735082, |
|
"tokens_seen": 586547200 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0003986783620200492, |
|
"loss": 2.7168, |
|
"theoretical_loss": 3.8467716418548648, |
|
"tokens_seen": 589824000 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.00039808728957821073, |
|
"loss": 2.7048, |
|
"theoretical_loss": 3.844542860072007, |
|
"tokens_seen": 593100800 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.00039749621713637225, |
|
"loss": 2.7227, |
|
"theoretical_loss": 3.8423297843701496, |
|
"tokens_seen": 596377600 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0003969051446945338, |
|
"loss": 2.7137, |
|
"theoretical_loss": 3.8401322186800995, |
|
"tokens_seen": 599654400 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0003963140722526953, |
|
"loss": 2.7505, |
|
"theoretical_loss": 3.83794997043469, |
|
"tokens_seen": 602931200 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.00039572299981085687, |
|
"loss": 2.7533, |
|
"theoretical_loss": 3.8357828504876004, |
|
"tokens_seen": 606208000 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0003951319273690184, |
|
"loss": 2.7222, |
|
"theoretical_loss": 3.833630673034487, |
|
"tokens_seen": 609484800 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0003945408549271799, |
|
"loss": 2.7234, |
|
"theoretical_loss": 3.831493255536345, |
|
"tokens_seen": 612761600 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.00039394978248534143, |
|
"loss": 2.7374, |
|
"theoretical_loss": 3.8293704186450253, |
|
"tokens_seen": 616038400 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.00039335871004350295, |
|
"loss": 2.7151, |
|
"theoretical_loss": 3.827261986130839, |
|
"tokens_seen": 619315200 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0003927676376016645, |
|
"loss": 2.6627, |
|
"theoretical_loss": 3.825167784812175, |
|
"tokens_seen": 622592000 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.000392176565159826, |
|
"loss": 2.69, |
|
"theoretical_loss": 3.823087644487069, |
|
"tokens_seen": 625868800 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.00039158549271798757, |
|
"loss": 2.6656, |
|
"theoretical_loss": 3.8210213978666565, |
|
"tokens_seen": 629145600 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0003909944202761491, |
|
"loss": 2.6679, |
|
"theoretical_loss": 3.8189688805104476, |
|
"tokens_seen": 632422400 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.00039040334783431056, |
|
"loss": 2.6709, |
|
"theoretical_loss": 3.816929930763374, |
|
"tokens_seen": 635699200 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0003898122753924721, |
|
"loss": 2.6848, |
|
"theoretical_loss": 3.8149043896945347, |
|
"tokens_seen": 638976000 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0003892212029506336, |
|
"loss": 2.6777, |
|
"theoretical_loss": 3.812892101037601, |
|
"tokens_seen": 642252800 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0003886301305087951, |
|
"loss": 2.6847, |
|
"theoretical_loss": 3.81089291113282, |
|
"tokens_seen": 645529600 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0003880390580669567, |
|
"loss": 2.6691, |
|
"theoretical_loss": 3.8089066688705673, |
|
"tokens_seen": 648806400 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0003874479856251182, |
|
"loss": 2.6569, |
|
"theoretical_loss": 3.8069332256363992, |
|
"tokens_seen": 652083200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.642861845651988, |
|
"debugging/distinct-1-grams": 0.8145672280225638, |
|
"debugging/distinct-2-grams": 0.980694270616048, |
|
"debugging/entropy-1-grams": 5.8621094281307204, |
|
"debugging/entropy-2-grams": 6.737099920464704, |
|
"debugging/length": 648.6666666666666, |
|
"debugging/num_segments": 12, |
|
"epoch": 0.23, |
|
"objective/train/docs_used": 351924, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.6956400871276855, |
|
"objective/train/theoretical_loss": 3.80497243525756, |
|
"objective/train/tokens_used": 675820000, |
|
"theoretical_loss": 3.80497243525756, |
|
"tokens_seen": 655360000 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.00038685691318327974, |
|
"loss": 2.6514, |
|
"theoretical_loss": 3.80497243525756, |
|
"tokens_seen": 655360000 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.00038626584074144126, |
|
"loss": 2.6215, |
|
"theoretical_loss": 3.8030241539508958, |
|
"tokens_seen": 658636800 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0003856747682996028, |
|
"loss": 2.619, |
|
"theoretical_loss": 3.8010882402721324, |
|
"tokens_seen": 661913600 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0003850836958577643, |
|
"loss": 2.656, |
|
"theoretical_loss": 3.7991645550664757, |
|
"tokens_seen": 665190400 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0003844926234159259, |
|
"loss": 2.6818, |
|
"theoretical_loss": 3.797252961420492, |
|
"tokens_seen": 668467200 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0003839015509740874, |
|
"loss": 2.6846, |
|
"theoretical_loss": 3.795353324615228, |
|
"tokens_seen": 671744000 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0003833104785322489, |
|
"loss": 2.7187, |
|
"theoretical_loss": 3.793465512080541, |
|
"tokens_seen": 675020800 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.00038271940609041044, |
|
"loss": 2.7152, |
|
"theoretical_loss": 3.791589393350587, |
|
"tokens_seen": 678297600 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.00038212833364857196, |
|
"loss": 2.6926, |
|
"theoretical_loss": 3.7897248400204475, |
|
"tokens_seen": 681574400 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0003815372612067335, |
|
"loss": 2.6837, |
|
"theoretical_loss": 3.7878717257038534, |
|
"tokens_seen": 684851200 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.000380946188764895, |
|
"loss": 2.6716, |
|
"theoretical_loss": 3.7860299259919685, |
|
"tokens_seen": 688128000 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0003803551163230566, |
|
"loss": 2.6557, |
|
"theoretical_loss": 3.7841993184132114, |
|
"tokens_seen": 691404800 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0003797640438812181, |
|
"loss": 2.6455, |
|
"theoretical_loss": 3.78237978239408, |
|
"tokens_seen": 694681600 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0003791729714393796, |
|
"loss": 2.6635, |
|
"theoretical_loss": 3.780571199220942, |
|
"tokens_seen": 697958400 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.00037858189899754114, |
|
"loss": 2.638, |
|
"theoretical_loss": 3.7787734520027803, |
|
"tokens_seen": 701235200 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.00037799082655570266, |
|
"loss": 2.6455, |
|
"theoretical_loss": 3.7769864256348455, |
|
"tokens_seen": 704512000 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0003773997541138642, |
|
"loss": 2.6937, |
|
"theoretical_loss": 3.775210006763202, |
|
"tokens_seen": 707788800 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.00037680868167202575, |
|
"loss": 2.6785, |
|
"theoretical_loss": 3.7734440837501406, |
|
"tokens_seen": 711065600 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0003762176092301873, |
|
"loss": 2.7156, |
|
"theoretical_loss": 3.7716885466404246, |
|
"tokens_seen": 714342400 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0003756265367883488, |
|
"loss": 2.7095, |
|
"theoretical_loss": 3.769943287128357, |
|
"tokens_seen": 717619200 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0003750354643465103, |
|
"loss": 2.7089, |
|
"theoretical_loss": 3.7682081985256364, |
|
"tokens_seen": 720896000 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.00037444439190467184, |
|
"loss": 2.7286, |
|
"theoretical_loss": 3.7664831757299795, |
|
"tokens_seen": 724172800 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.00037385331946283336, |
|
"loss": 2.6792, |
|
"theoretical_loss": 3.7647681151944976, |
|
"tokens_seen": 727449600 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.00037326224702099493, |
|
"loss": 2.6448, |
|
"theoretical_loss": 3.7630629148977937, |
|
"tokens_seen": 730726400 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.00037267117457915645, |
|
"loss": 2.6615, |
|
"theoretical_loss": 3.761367474314768, |
|
"tokens_seen": 734003200 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.000372080102137318, |
|
"loss": 2.6745, |
|
"theoretical_loss": 3.7596816943881084, |
|
"tokens_seen": 737280000 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0003714890296954795, |
|
"loss": 2.6448, |
|
"theoretical_loss": 3.758005477500451, |
|
"tokens_seen": 740556800 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.000370897957253641, |
|
"loss": 2.6424, |
|
"theoretical_loss": 3.756338727447186, |
|
"tokens_seen": 743833600 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.00037030688481180254, |
|
"loss": 2.6773, |
|
"theoretical_loss": 3.7546813494098945, |
|
"tokens_seen": 747110400 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.00036971581236996406, |
|
"loss": 2.6502, |
|
"theoretical_loss": 3.7530332499304007, |
|
"tokens_seen": 750387200 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.00036912473992812563, |
|
"loss": 2.673, |
|
"theoretical_loss": 3.7513943368854195, |
|
"tokens_seen": 753664000 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.00036853366748628715, |
|
"loss": 2.6322, |
|
"theoretical_loss": 3.7497645194617863, |
|
"tokens_seen": 756940800 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0003679425950444487, |
|
"loss": 2.602, |
|
"theoretical_loss": 3.748143708132246, |
|
"tokens_seen": 760217600 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0003673515226026102, |
|
"loss": 2.6187, |
|
"theoretical_loss": 3.7465318146317994, |
|
"tokens_seen": 763494400 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0003667604501607717, |
|
"loss": 2.587, |
|
"theoretical_loss": 3.7449287519345766, |
|
"tokens_seen": 766771200 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.00036616937771893324, |
|
"loss": 2.5946, |
|
"theoretical_loss": 3.7433344342312385, |
|
"tokens_seen": 770048000 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0003655783052770948, |
|
"loss": 2.6269, |
|
"theoretical_loss": 3.7417487769068756, |
|
"tokens_seen": 773324800 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.00036498723283525633, |
|
"loss": 2.6403, |
|
"theoretical_loss": 3.7401716965194076, |
|
"tokens_seen": 776601600 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.00036439616039341785, |
|
"loss": 2.6323, |
|
"theoretical_loss": 3.738603110778461, |
|
"tokens_seen": 779878400 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0003638050879515794, |
|
"loss": 2.6448, |
|
"theoretical_loss": 3.73704293852471, |
|
"tokens_seen": 783155200 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0003632140155097409, |
|
"loss": 2.6451, |
|
"theoretical_loss": 3.7354910997096793, |
|
"tokens_seen": 786432000 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0003626229430679024, |
|
"loss": 2.656, |
|
"theoretical_loss": 3.7339475153759825, |
|
"tokens_seen": 789708800 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.000362031870626064, |
|
"loss": 2.611, |
|
"theoretical_loss": 3.732412107638, |
|
"tokens_seen": 792985600 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.00036144079818422546, |
|
"loss": 2.6138, |
|
"theoretical_loss": 3.7308847996629724, |
|
"tokens_seen": 796262400 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.000360849725742387, |
|
"loss": 2.6362, |
|
"theoretical_loss": 3.7293655156525043, |
|
"tokens_seen": 799539200 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0003602586533005485, |
|
"loss": 2.6161, |
|
"theoretical_loss": 3.727854180824469, |
|
"tokens_seen": 802816000 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.00035966758085871, |
|
"loss": 2.5601, |
|
"theoretical_loss": 3.7263507213952978, |
|
"tokens_seen": 806092800 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.00035907650841687154, |
|
"loss": 2.584, |
|
"theoretical_loss": 3.724855064562658, |
|
"tokens_seen": 809369600 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.00035848543597503306, |
|
"loss": 2.6023, |
|
"theoretical_loss": 3.723367138488488, |
|
"tokens_seen": 812646400 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.00035789436353319464, |
|
"loss": 2.6156, |
|
"theoretical_loss": 3.7218868722824014, |
|
"tokens_seen": 815923200 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"objective/train/docs_used": 436767, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.712568759918213, |
|
"objective/train/theoretical_loss": 3.7204141959854384, |
|
"objective/train/tokens_used": 839660000, |
|
"theoretical_loss": 3.7204141959854384, |
|
"tokens_seen": 819200000 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.00035730329109135616, |
|
"loss": 2.6219, |
|
"theoretical_loss": 3.7204141959854384, |
|
"tokens_seen": 819200000 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0003567122186495177, |
|
"loss": 2.6477, |
|
"theoretical_loss": 3.718949040554162, |
|
"tokens_seen": 822476800 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0003561211462076792, |
|
"loss": 2.6576, |
|
"theoretical_loss": 3.7174913378450833, |
|
"tokens_seen": 825753600 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0003555300737658407, |
|
"loss": 2.6582, |
|
"theoretical_loss": 3.7160410205994183, |
|
"tokens_seen": 829030400 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.00035493900132400224, |
|
"loss": 2.6759, |
|
"theoretical_loss": 3.7145980224281585, |
|
"tokens_seen": 832307200 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0003543597503310006, |
|
"loss": 2.6768, |
|
"theoretical_loss": 3.713162277797449, |
|
"tokens_seen": 835584000 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.00035376867788916214, |
|
"loss": 2.6422, |
|
"theoretical_loss": 3.7117337220142748, |
|
"tokens_seen": 838860800 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0003531894268961604, |
|
"loss": 2.6515, |
|
"theoretical_loss": 3.7103122912124364, |
|
"tokens_seen": 842137600 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0003526101759031587, |
|
"loss": 2.6762, |
|
"theoretical_loss": 3.7088979223388128, |
|
"tokens_seen": 845414400 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0003520191034613202, |
|
"loss": 2.6563, |
|
"theoretical_loss": 3.70749055313991, |
|
"tokens_seen": 848691200 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0003514280310194817, |
|
"loss": 2.6461, |
|
"theoretical_loss": 3.7060901221486766, |
|
"tokens_seen": 851968000 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0003508369585776433, |
|
"loss": 2.6557, |
|
"theoretical_loss": 3.704696568671591, |
|
"tokens_seen": 855244800 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0003502458861358048, |
|
"loss": 2.6539, |
|
"theoretical_loss": 3.7033098327760063, |
|
"tokens_seen": 858521600 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00034965481369396634, |
|
"loss": 2.632, |
|
"theoretical_loss": 3.7019298552777533, |
|
"tokens_seen": 861798400 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00034906374125212786, |
|
"loss": 2.619, |
|
"theoretical_loss": 3.700556577728988, |
|
"tokens_seen": 865075200 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0003484726688102894, |
|
"loss": 2.629, |
|
"theoretical_loss": 3.6991899424062815, |
|
"tokens_seen": 868352000 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0003478815963684509, |
|
"loss": 2.609, |
|
"theoretical_loss": 3.697829892298951, |
|
"tokens_seen": 871628800 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0003472905239266125, |
|
"loss": 2.6334, |
|
"theoretical_loss": 3.696476371097618, |
|
"tokens_seen": 874905600 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.000346699451484774, |
|
"loss": 2.6014, |
|
"theoretical_loss": 3.695129323182993, |
|
"tokens_seen": 878182400 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0003461083790429355, |
|
"loss": 2.5987, |
|
"theoretical_loss": 3.693788693614879, |
|
"tokens_seen": 881459200 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.00034551730660109704, |
|
"loss": 2.5732, |
|
"theoretical_loss": 3.6924544281213967, |
|
"tokens_seen": 884736000 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.00034492623415925856, |
|
"loss": 2.5476, |
|
"theoretical_loss": 3.691126473088412, |
|
"tokens_seen": 888012800 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0003443351617174201, |
|
"loss": 2.5573, |
|
"theoretical_loss": 3.689804775549173, |
|
"tokens_seen": 891289600 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0003437440892755816, |
|
"loss": 2.5655, |
|
"theoretical_loss": 3.688489283174146, |
|
"tokens_seen": 894566400 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0003431530168337432, |
|
"loss": 2.6253, |
|
"theoretical_loss": 3.6871799442610538, |
|
"tokens_seen": 897843200 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0003425619443919047, |
|
"loss": 2.6272, |
|
"theoretical_loss": 3.685876707725093, |
|
"tokens_seen": 901120000 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0003419708719500662, |
|
"loss": 2.5663, |
|
"theoretical_loss": 3.6845795230893517, |
|
"tokens_seen": 904396800 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.00034137979950822774, |
|
"loss": 2.5763, |
|
"theoretical_loss": 3.6832883404754035, |
|
"tokens_seen": 907673600 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00034078872706638926, |
|
"loss": 2.5661, |
|
"theoretical_loss": 3.6820031105940796, |
|
"tokens_seen": 910950400 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.0003401976546245508, |
|
"loss": 2.565, |
|
"theoretical_loss": 3.6807237847364176, |
|
"tokens_seen": 914227200 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00033960658218271236, |
|
"loss": 2.6169, |
|
"theoretical_loss": 3.6794503147647846, |
|
"tokens_seen": 917504000 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.0003390155097408739, |
|
"loss": 2.6209, |
|
"theoretical_loss": 3.67818265310416, |
|
"tokens_seen": 920780800 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.0003384244372990354, |
|
"loss": 2.602, |
|
"theoretical_loss": 3.6769207527335888, |
|
"tokens_seen": 924057600 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.0003378333648571969, |
|
"loss": 2.597, |
|
"theoretical_loss": 3.675664567177787, |
|
"tokens_seen": 927334400 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00033724229241535844, |
|
"loss": 2.596, |
|
"theoretical_loss": 3.674414050498913, |
|
"tokens_seen": 930611200 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00033665121997351996, |
|
"loss": 2.5945, |
|
"theoretical_loss": 3.6731691572884824, |
|
"tokens_seen": 933888000 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00033606014753168154, |
|
"loss": 2.5671, |
|
"theoretical_loss": 3.671929842659438, |
|
"tokens_seen": 937164800 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.00033546907508984306, |
|
"loss": 2.5612, |
|
"theoretical_loss": 3.6706960622383624, |
|
"tokens_seen": 940441600 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0003348780026480046, |
|
"loss": 2.5934, |
|
"theoretical_loss": 3.6694677721578377, |
|
"tokens_seen": 943718400 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0003342869302061661, |
|
"loss": 2.59, |
|
"theoretical_loss": 3.66824492904894, |
|
"tokens_seen": 946995200 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.00033369585776432757, |
|
"loss": 2.5596, |
|
"theoretical_loss": 3.667027490033874, |
|
"tokens_seen": 950272000 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0003331047853224891, |
|
"loss": 2.5686, |
|
"theoretical_loss": 3.6658154127187412, |
|
"tokens_seen": 953548800 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0003325137128806506, |
|
"loss": 2.5853, |
|
"theoretical_loss": 3.664608655186437, |
|
"tokens_seen": 956825600 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0003319226404388122, |
|
"loss": 2.5926, |
|
"theoretical_loss": 3.663407175989679, |
|
"tokens_seen": 960102400 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0003313315679969737, |
|
"loss": 2.5611, |
|
"theoretical_loss": 3.662210934144158, |
|
"tokens_seen": 963379200 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0003307404955551352, |
|
"loss": 2.587, |
|
"theoretical_loss": 3.661019889121812, |
|
"tokens_seen": 966656000 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00033014942311329675, |
|
"loss": 2.6005, |
|
"theoretical_loss": 3.6598340008442234, |
|
"tokens_seen": 969932800 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00032955835067145827, |
|
"loss": 2.5902, |
|
"theoretical_loss": 3.6586532296761285, |
|
"tokens_seen": 973209600 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0003289672782296198, |
|
"loss": 2.6145, |
|
"theoretical_loss": 3.657477536419047, |
|
"tokens_seen": 976486400 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00032837620578778136, |
|
"loss": 2.6437, |
|
"theoretical_loss": 3.656306882305022, |
|
"tokens_seen": 979763200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.6198785821735879, |
|
"debugging/distinct-1-grams": 0.7707728209964196, |
|
"debugging/distinct-2-grams": 0.9604165709732011, |
|
"debugging/entropy-1-grams": 5.836498792012382, |
|
"debugging/entropy-2-grams": 6.692653651879306, |
|
"debugging/length": 640.8333333333334, |
|
"debugging/num_segments": 12, |
|
"epoch": 0.35, |
|
"objective/train/docs_used": 522194, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.5617499351501465, |
|
"objective/train/theoretical_loss": 3.6551412289904697, |
|
"objective/train/tokens_used": 1003500000, |
|
"theoretical_loss": 3.6551412289904697, |
|
"tokens_seen": 983040000 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0003277851333459429, |
|
"loss": 2.5923, |
|
"theoretical_loss": 3.6551412289904697, |
|
"tokens_seen": 983040000 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0003271940609041044, |
|
"loss": 2.5912, |
|
"theoretical_loss": 3.6539805385501376, |
|
"tokens_seen": 986316800 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0003266029884622659, |
|
"loss": 2.6037, |
|
"theoretical_loss": 3.652824773471171, |
|
"tokens_seen": 989593600 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00032601191602042745, |
|
"loss": 2.5777, |
|
"theoretical_loss": 3.651673896647277, |
|
"tokens_seen": 992870400 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.00032542084357858897, |
|
"loss": 2.5642, |
|
"theoretical_loss": 3.6505278713729985, |
|
"tokens_seen": 996147200 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.00032482977113675054, |
|
"loss": 2.5653, |
|
"theoretical_loss": 3.6493866613380774, |
|
"tokens_seen": 999424000 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.00032423869869491206, |
|
"loss": 2.5584, |
|
"theoretical_loss": 3.648250230621924, |
|
"tokens_seen": 1002700800 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0003236476262530736, |
|
"loss": 2.5586, |
|
"theoretical_loss": 3.647118543688179, |
|
"tokens_seen": 1005977600 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0003230565538112351, |
|
"loss": 2.5333, |
|
"theoretical_loss": 3.6459915653793633, |
|
"tokens_seen": 1009254400 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0003224654813693966, |
|
"loss": 2.5645, |
|
"theoretical_loss": 3.644869260911628, |
|
"tokens_seen": 1012531200 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.00032187440892755814, |
|
"loss": 2.5094, |
|
"theoretical_loss": 3.64375159586959, |
|
"tokens_seen": 1015808000 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0003212833364857197, |
|
"loss": 2.5334, |
|
"theoretical_loss": 3.642638536201252, |
|
"tokens_seen": 1019084800 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00032069226404388124, |
|
"loss": 2.5607, |
|
"theoretical_loss": 3.6415300482130135, |
|
"tokens_seen": 1022361600 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00032010119160204276, |
|
"loss": 2.5815, |
|
"theoretical_loss": 3.6404260985647667, |
|
"tokens_seen": 1025638400 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0003195101191602043, |
|
"loss": 2.531, |
|
"theoretical_loss": 3.6393266542650684, |
|
"tokens_seen": 1028915200 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0003189190467183658, |
|
"loss": 2.5341, |
|
"theoretical_loss": 3.638231682666401, |
|
"tokens_seen": 1032192000 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0003183279742765273, |
|
"loss": 2.5545, |
|
"theoretical_loss": 3.637141151460505, |
|
"tokens_seen": 1035468800 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00031773690183468884, |
|
"loss": 2.5548, |
|
"theoretical_loss": 3.636055028673799, |
|
"tokens_seen": 1038745600 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0003171458293928504, |
|
"loss": 2.5728, |
|
"theoretical_loss": 3.634973282662864, |
|
"tokens_seen": 1042022400 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00031655475695101194, |
|
"loss": 2.6221, |
|
"theoretical_loss": 3.6338958821100107, |
|
"tokens_seen": 1045299200 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00031596368450917346, |
|
"loss": 2.6254, |
|
"theoretical_loss": 3.63282279601892, |
|
"tokens_seen": 1048576000 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.000315372612067335, |
|
"loss": 2.6667, |
|
"theoretical_loss": 3.631753993710352, |
|
"tokens_seen": 1051852800 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0003147815396254965, |
|
"loss": 2.6569, |
|
"theoretical_loss": 3.630689444817925, |
|
"tokens_seen": 1055129600 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.000314190467183658, |
|
"loss": 2.648, |
|
"theoretical_loss": 3.629629119283967, |
|
"tokens_seen": 1058406400 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0003135993947418196, |
|
"loss": 2.668, |
|
"theoretical_loss": 3.628572987355434, |
|
"tokens_seen": 1061683200 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0003130083222999811, |
|
"loss": 2.6364, |
|
"theoretical_loss": 3.6275210195798913, |
|
"tokens_seen": 1064960000 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.00031241724985814264, |
|
"loss": 2.6346, |
|
"theoretical_loss": 3.626473186801564, |
|
"tokens_seen": 1068236800 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.00031182617741630416, |
|
"loss": 2.6531, |
|
"theoretical_loss": 3.6254294601574495, |
|
"tokens_seen": 1071513600 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0003112351049744657, |
|
"loss": 2.628, |
|
"theoretical_loss": 3.624389811073493, |
|
"tokens_seen": 1074790400 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0003106440325326272, |
|
"loss": 2.6233, |
|
"theoretical_loss": 3.6233542112608257, |
|
"tokens_seen": 1078067200 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0003100529600907888, |
|
"loss": 2.6271, |
|
"theoretical_loss": 3.6223226327120592, |
|
"tokens_seen": 1081344000 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0003094618876489503, |
|
"loss": 2.6132, |
|
"theoretical_loss": 3.621295047697644, |
|
"tokens_seen": 1084620800 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0003088708152071118, |
|
"loss": 2.6227, |
|
"theoretical_loss": 3.6202714287622833, |
|
"tokens_seen": 1087897600 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.00030827974276527334, |
|
"loss": 2.6147, |
|
"theoretical_loss": 3.6192517487214038, |
|
"tokens_seen": 1091174400 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.00030768867032343486, |
|
"loss": 2.5954, |
|
"theoretical_loss": 3.6182359806576834, |
|
"tokens_seen": 1094451200 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0003070975978815964, |
|
"loss": 2.6142, |
|
"theoretical_loss": 3.6172240979176333, |
|
"tokens_seen": 1097728000 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0003065065254397579, |
|
"loss": 2.5959, |
|
"theoretical_loss": 3.616216074108232, |
|
"tokens_seen": 1101004800 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0003059154529979195, |
|
"loss": 2.6174, |
|
"theoretical_loss": 3.6152118830936164, |
|
"tokens_seen": 1104281600 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.000305324380556081, |
|
"loss": 2.6236, |
|
"theoretical_loss": 3.6142114989918195, |
|
"tokens_seen": 1107558400 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00030473330811424246, |
|
"loss": 2.6293, |
|
"theoretical_loss": 3.6132148961715624, |
|
"tokens_seen": 1110835200 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.000304142235672404, |
|
"loss": 2.5852, |
|
"theoretical_loss": 3.6122220492490964, |
|
"tokens_seen": 1114112000 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0003035511632305655, |
|
"loss": 2.5782, |
|
"theoretical_loss": 3.6112329330850894, |
|
"tokens_seen": 1117388800 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.000302960090788727, |
|
"loss": 2.607, |
|
"theoretical_loss": 3.61024752278157, |
|
"tokens_seen": 1120665600 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0003023690183468886, |
|
"loss": 2.56, |
|
"theoretical_loss": 3.6092657936789054, |
|
"tokens_seen": 1123942400 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0003017779459050501, |
|
"loss": 2.5852, |
|
"theoretical_loss": 3.6082877213528377, |
|
"tokens_seen": 1127219200 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00030118687346321164, |
|
"loss": 2.5555, |
|
"theoretical_loss": 3.60731328161156, |
|
"tokens_seen": 1130496000 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00030059580102137316, |
|
"loss": 2.5653, |
|
"theoretical_loss": 3.6063424504928365, |
|
"tokens_seen": 1133772800 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.0003000047285795347, |
|
"loss": 2.5872, |
|
"theoretical_loss": 3.60537520426117, |
|
"tokens_seen": 1137049600 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.0002994136561376962, |
|
"loss": 2.5925, |
|
"theoretical_loss": 3.6044115194050086, |
|
"tokens_seen": 1140326400 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.0002988225836958578, |
|
"loss": 2.5761, |
|
"theoretical_loss": 3.603451372633997, |
|
"tokens_seen": 1143603200 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"objective/train/docs_used": 604425, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.565331220626831, |
|
"objective/train/theoretical_loss": 3.6024947408762698, |
|
"objective/train/tokens_used": 1167340000, |
|
"theoretical_loss": 3.6024947408762698, |
|
"tokens_seen": 1146880000 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.0002982315112540193, |
|
"loss": 2.6043, |
|
"theoretical_loss": 3.6024947408762698, |
|
"tokens_seen": 1146880000 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.0002976404388121808, |
|
"loss": 2.5725, |
|
"theoretical_loss": 3.601541601275783, |
|
"tokens_seen": 1150156800 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00029704936637034234, |
|
"loss": 2.548, |
|
"theoretical_loss": 3.6005919311896886, |
|
"tokens_seen": 1153433600 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00029647011537734067, |
|
"loss": 2.5699, |
|
"theoretical_loss": 3.5996457081857454, |
|
"tokens_seen": 1156710400 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00029587904293550214, |
|
"loss": 2.5709, |
|
"theoretical_loss": 3.598702910039772, |
|
"tokens_seen": 1159987200 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00029528797049366366, |
|
"loss": 2.5726, |
|
"theoretical_loss": 3.597763514733133, |
|
"tokens_seen": 1163264000 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00029469689805182523, |
|
"loss": 2.5542, |
|
"theoretical_loss": 3.59682750045027, |
|
"tokens_seen": 1166540800 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00029410582560998675, |
|
"loss": 2.5631, |
|
"theoretical_loss": 3.5958948455762583, |
|
"tokens_seen": 1169817600 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0002935147531681483, |
|
"loss": 2.5865, |
|
"theoretical_loss": 3.594965528694412, |
|
"tokens_seen": 1173094400 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0002929236807263098, |
|
"loss": 2.5499, |
|
"theoretical_loss": 3.594039528583913, |
|
"tokens_seen": 1176371200 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0002923326082844713, |
|
"loss": 2.559, |
|
"theoretical_loss": 3.5931168242174847, |
|
"tokens_seen": 1179648000 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00029174153584263284, |
|
"loss": 2.5468, |
|
"theoretical_loss": 3.59219739475909, |
|
"tokens_seen": 1182924800 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0002911504634007944, |
|
"loss": 2.5555, |
|
"theoretical_loss": 3.5912812195616732, |
|
"tokens_seen": 1186201600 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00029055939095895593, |
|
"loss": 2.5321, |
|
"theoretical_loss": 3.590368278164926, |
|
"tokens_seen": 1189478400 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.00028996831851711745, |
|
"loss": 2.5672, |
|
"theoretical_loss": 3.5894585502930902, |
|
"tokens_seen": 1192755200 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.000289377246075279, |
|
"loss": 2.5742, |
|
"theoretical_loss": 3.588552015852793, |
|
"tokens_seen": 1196032000 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.0002887861736334405, |
|
"loss": 2.5502, |
|
"theoretical_loss": 3.5876486549309097, |
|
"tokens_seen": 1199308800 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.000288195101191602, |
|
"loss": 2.5514, |
|
"theoretical_loss": 3.586748447792462, |
|
"tokens_seen": 1202585600 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.0002876040287497636, |
|
"loss": 2.5578, |
|
"theoretical_loss": 3.5858513748785423, |
|
"tokens_seen": 1205862400 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.0002870129563079251, |
|
"loss": 2.6184, |
|
"theoretical_loss": 3.5849574168042704, |
|
"tokens_seen": 1209139200 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.00028642188386608663, |
|
"loss": 2.5991, |
|
"theoretical_loss": 3.5840665543567782, |
|
"tokens_seen": 1212416000 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.00028583081142424815, |
|
"loss": 2.5757, |
|
"theoretical_loss": 3.583178768493222, |
|
"tokens_seen": 1215692800 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00028523973898240967, |
|
"loss": 2.5659, |
|
"theoretical_loss": 3.5822940403388284, |
|
"tokens_seen": 1218969600 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0002846486665405712, |
|
"loss": 2.5729, |
|
"theoretical_loss": 3.581412351184958, |
|
"tokens_seen": 1222246400 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0002840575940987327, |
|
"loss": 2.5652, |
|
"theoretical_loss": 3.580533682487208, |
|
"tokens_seen": 1225523200 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0002834665216568943, |
|
"loss": 2.5672, |
|
"theoretical_loss": 3.579658015863532, |
|
"tokens_seen": 1228800000 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0002828754492150558, |
|
"loss": 2.5836, |
|
"theoretical_loss": 3.5787853330923927, |
|
"tokens_seen": 1232076800 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00028228437677321733, |
|
"loss": 2.5493, |
|
"theoretical_loss": 3.577915616110936, |
|
"tokens_seen": 1235353600 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00028169330433137885, |
|
"loss": 2.5739, |
|
"theoretical_loss": 3.577048847013194, |
|
"tokens_seen": 1238630400 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00028110223188954037, |
|
"loss": 2.5689, |
|
"theoretical_loss": 3.57618500804831, |
|
"tokens_seen": 1241907200 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0002805111594477019, |
|
"loss": 2.5232, |
|
"theoretical_loss": 3.575324081618793, |
|
"tokens_seen": 1245184000 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00027992008700586347, |
|
"loss": 2.5686, |
|
"theoretical_loss": 3.5744660502787875, |
|
"tokens_seen": 1248460800 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.000279329014564025, |
|
"loss": 2.5236, |
|
"theoretical_loss": 3.5736108967323794, |
|
"tokens_seen": 1251737600 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.0002787379421221865, |
|
"loss": 2.5925, |
|
"theoretical_loss": 3.5727586038319155, |
|
"tokens_seen": 1255014400 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00027814686968034803, |
|
"loss": 2.5437, |
|
"theoretical_loss": 3.571909154576348, |
|
"tokens_seen": 1258291200 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00027755579723850955, |
|
"loss": 2.5461, |
|
"theoretical_loss": 3.5710625321096074, |
|
"tokens_seen": 1261568000 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00027696472479667107, |
|
"loss": 2.5288, |
|
"theoretical_loss": 3.570218719718989, |
|
"tokens_seen": 1264844800 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00027637365235483265, |
|
"loss": 2.5546, |
|
"theoretical_loss": 3.569377700833569, |
|
"tokens_seen": 1268121600 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00027578257991299417, |
|
"loss": 2.5354, |
|
"theoretical_loss": 3.568539459022639, |
|
"tokens_seen": 1271398400 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0002751915074711557, |
|
"loss": 2.5518, |
|
"theoretical_loss": 3.5677039779941584, |
|
"tokens_seen": 1274675200 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0002746004350293172, |
|
"loss": 2.5598, |
|
"theoretical_loss": 3.566871241593236, |
|
"tokens_seen": 1277952000 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00027400936258747873, |
|
"loss": 2.5445, |
|
"theoretical_loss": 3.5660412338006235, |
|
"tokens_seen": 1281228800 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00027341829014564025, |
|
"loss": 2.5946, |
|
"theoretical_loss": 3.565213938731236, |
|
"tokens_seen": 1284505600 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0002728272177038018, |
|
"loss": 2.5831, |
|
"theoretical_loss": 3.5643893406326868, |
|
"tokens_seen": 1287782400 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00027223614526196335, |
|
"loss": 2.5568, |
|
"theoretical_loss": 3.5635674238838466, |
|
"tokens_seen": 1291059200 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00027164507282012487, |
|
"loss": 2.5429, |
|
"theoretical_loss": 3.5627481729934196, |
|
"tokens_seen": 1294336000 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0002710540003782864, |
|
"loss": 2.5785, |
|
"theoretical_loss": 3.561931572598538, |
|
"tokens_seen": 1297612800 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0002704629279364479, |
|
"loss": 2.5493, |
|
"theoretical_loss": 3.5611176074633777, |
|
"tokens_seen": 1300889600 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00026987185549460943, |
|
"loss": 2.5602, |
|
"theoretical_loss": 3.5603062624777895, |
|
"tokens_seen": 1304166400 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00026928078305277095, |
|
"loss": 2.6058, |
|
"theoretical_loss": 3.559497522655951, |
|
"tokens_seen": 1307443200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.4840744728399488, |
|
"debugging/distinct-1-grams": 0.7704641680561449, |
|
"debugging/distinct-2-grams": 0.9548886056073045, |
|
"debugging/entropy-1-grams": 5.354809677980487, |
|
"debugging/entropy-2-grams": 6.077984561009924, |
|
"debugging/length": 581.0, |
|
"debugging/num_segments": 7, |
|
"epoch": 0.47, |
|
"objective/train/docs_used": 687328, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.7656729221343994, |
|
"objective/train/theoretical_loss": 3.5586913731350327, |
|
"objective/train/tokens_used": 1331180000, |
|
"theoretical_loss": 3.5586913731350327, |
|
"tokens_seen": 1310720000 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.0002686897106109325, |
|
"loss": 2.6199, |
|
"theoretical_loss": 3.5586913731350327, |
|
"tokens_seen": 1310720000 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00026809863816909405, |
|
"loss": 2.6217, |
|
"theoretical_loss": 3.557887799173889, |
|
"tokens_seen": 1313996800 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00026750756572725557, |
|
"loss": 2.5979, |
|
"theoretical_loss": 3.557086786151754, |
|
"tokens_seen": 1317273600 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.0002669164932854171, |
|
"loss": 2.5856, |
|
"theoretical_loss": 3.5562883195669697, |
|
"tokens_seen": 1320550400 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00026632542084357855, |
|
"loss": 2.5917, |
|
"theoretical_loss": 3.555492385035719, |
|
"tokens_seen": 1323827200 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.0002657343484017401, |
|
"loss": 2.5804, |
|
"theoretical_loss": 3.5546989682907784, |
|
"tokens_seen": 1327104000 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026514327595990165, |
|
"loss": 2.5405, |
|
"theoretical_loss": 3.5539080551802895, |
|
"tokens_seen": 1330380800 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026455220351806317, |
|
"loss": 2.5578, |
|
"theoretical_loss": 3.553119631666546, |
|
"tokens_seen": 1333657600 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.0002639611310762247, |
|
"loss": 2.5811, |
|
"theoretical_loss": 3.5523336838247914, |
|
"tokens_seen": 1336934400 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.0002633700586343862, |
|
"loss": 2.5485, |
|
"theoretical_loss": 3.55155019784204, |
|
"tokens_seen": 1340211200 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026277898619254773, |
|
"loss": 2.5404, |
|
"theoretical_loss": 3.5507691600159053, |
|
"tokens_seen": 1343488000 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026218791375070925, |
|
"loss": 2.5348, |
|
"theoretical_loss": 3.5499905567534515, |
|
"tokens_seen": 1346764800 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026159684130887083, |
|
"loss": 2.5136, |
|
"theoretical_loss": 3.549214374570052, |
|
"tokens_seen": 1350041600 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026100576886703235, |
|
"loss": 2.5227, |
|
"theoretical_loss": 3.5484406000882665, |
|
"tokens_seen": 1353318400 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026041469642519387, |
|
"loss": 2.4971, |
|
"theoretical_loss": 3.5476692200367346, |
|
"tokens_seen": 1356595200 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.0002598236239833554, |
|
"loss": 2.5241, |
|
"theoretical_loss": 3.546900221249076, |
|
"tokens_seen": 1359872000 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.0002592325515415169, |
|
"loss": 2.5345, |
|
"theoretical_loss": 3.5461335906628157, |
|
"tokens_seen": 1363148800 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00025864147909967843, |
|
"loss": 2.5072, |
|
"theoretical_loss": 3.54536931531831, |
|
"tokens_seen": 1366425600 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00025805040665783995, |
|
"loss": 2.5089, |
|
"theoretical_loss": 3.5446073823576985, |
|
"tokens_seen": 1369702400 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00025745933421600153, |
|
"loss": 2.5158, |
|
"theoretical_loss": 3.543847779023859, |
|
"tokens_seen": 1372979200 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00025686826177416305, |
|
"loss": 2.5466, |
|
"theoretical_loss": 3.543090492659384, |
|
"tokens_seen": 1376256000 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00025627718933232457, |
|
"loss": 2.5079, |
|
"theoretical_loss": 3.542335510705562, |
|
"tokens_seen": 1379532800 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.0002556861168904861, |
|
"loss": 2.5099, |
|
"theoretical_loss": 3.541582820701378, |
|
"tokens_seen": 1382809600 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.0002550950444486476, |
|
"loss": 2.4877, |
|
"theoretical_loss": 3.5408324102825253, |
|
"tokens_seen": 1386086400 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.00025450397200680913, |
|
"loss": 2.4898, |
|
"theoretical_loss": 3.5400842671804265, |
|
"tokens_seen": 1389363200 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.0002539128995649707, |
|
"loss": 2.4993, |
|
"theoretical_loss": 3.5393383792212676, |
|
"tokens_seen": 1392640000 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.00025332182712313223, |
|
"loss": 2.4967, |
|
"theoretical_loss": 3.5385947343250486, |
|
"tokens_seen": 1395916800 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.00025273075468129375, |
|
"loss": 2.5021, |
|
"theoretical_loss": 3.53785332050464, |
|
"tokens_seen": 1399193600 |
|
} |
|
], |
|
"max_steps": 42724, |
|
"num_train_epochs": 9223372036854775807, |
|
"total_flos": 7.14460209610752e+17, |
|
"trial_name": null, |
|
"trial_params": null |
|
} |
|
|