|
{ |
|
"best_metric": null, |
|
"best_model_checkpoint": null, |
|
"epoch": 0.5, |
|
"global_step": 21362, |
|
"is_hyper_param_search": false, |
|
"is_local_process_zero": true, |
|
"is_world_process_zero": true, |
|
"log_history": [ |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 1.1682242990654204e-06, |
|
"loss": 10.8704, |
|
"theoretical_loss": 20.812814784551147, |
|
"tokens_seen": 65536 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 5.841121495327103e-05, |
|
"loss": 9.0023, |
|
"theoretical_loss": 8.563479647615063, |
|
"tokens_seen": 3276800 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 0.00011682242990654206, |
|
"loss": 6.8915, |
|
"theoretical_loss": 7.4777557010520255, |
|
"tokens_seen": 6553600 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 0.00017523364485981307, |
|
"loss": 6.1075, |
|
"theoretical_loss": 6.933751471898896, |
|
"tokens_seen": 9830400 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 0.00023364485981308412, |
|
"loss": 5.7212, |
|
"theoretical_loss": 6.583563211430409, |
|
"tokens_seen": 13107200 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.00029205607476635517, |
|
"loss": 5.4534, |
|
"theoretical_loss": 6.330710548120079, |
|
"tokens_seen": 16384000 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.00035046728971962614, |
|
"loss": 5.2364, |
|
"theoretical_loss": 6.135526214944321, |
|
"tokens_seen": 19660800 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0004088785046728972, |
|
"loss": 5.123, |
|
"theoretical_loss": 5.978098566873603, |
|
"tokens_seen": 22937600 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.00046728971962616824, |
|
"loss": 4.9823, |
|
"theoretical_loss": 5.847114309269919, |
|
"tokens_seen": 26214400 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.000499739928125591, |
|
"loss": 4.8312, |
|
"theoretical_loss": 5.73557379888612, |
|
"tokens_seen": 29491200 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0004991488556837526, |
|
"loss": 4.7245, |
|
"theoretical_loss": 5.638867127075349, |
|
"tokens_seen": 32768000 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0004985577832419141, |
|
"loss": 4.6219, |
|
"theoretical_loss": 5.553809364848902, |
|
"tokens_seen": 36044800 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0004979667108000757, |
|
"loss": 4.5325, |
|
"theoretical_loss": 5.478115063560433, |
|
"tokens_seen": 39321600 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004973756383582371, |
|
"loss": 4.45, |
|
"theoretical_loss": 5.410092942583357, |
|
"tokens_seen": 42598400 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004967845659163987, |
|
"loss": 4.4207, |
|
"theoretical_loss": 5.34845906673983, |
|
"tokens_seen": 45875200 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004961934934745603, |
|
"loss": 4.3088, |
|
"theoretical_loss": 5.292217549941562, |
|
"tokens_seen": 49152000 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004956024210327218, |
|
"loss": 4.2258, |
|
"theoretical_loss": 5.240581608773973, |
|
"tokens_seen": 52428800 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004950113485908833, |
|
"loss": 4.18, |
|
"theoretical_loss": 5.192919707529784, |
|
"tokens_seen": 55705600 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004944202761490448, |
|
"loss": 4.0541, |
|
"theoretical_loss": 5.1487178463604355, |
|
"tokens_seen": 58982400 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004938292037072064, |
|
"loss": 3.9835, |
|
"theoretical_loss": 5.107552545409097, |
|
"tokens_seen": 62259200 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004932381312653678, |
|
"loss": 3.8648, |
|
"theoretical_loss": 5.069071100147241, |
|
"tokens_seen": 65536000 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004926470588235294, |
|
"loss": 3.8072, |
|
"theoretical_loss": 5.032976892842002, |
|
"tokens_seen": 68812800 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.000492055986381691, |
|
"loss": 3.7248, |
|
"theoretical_loss": 4.999018291228659, |
|
"tokens_seen": 72089600 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004914649139398525, |
|
"loss": 3.6863, |
|
"theoretical_loss": 4.966980138355957, |
|
"tokens_seen": 75366400 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.000490873841498014, |
|
"loss": 3.626, |
|
"theoretical_loss": 4.936677144629131, |
|
"tokens_seen": 78643200 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004902827690561755, |
|
"loss": 3.5795, |
|
"theoretical_loss": 4.907948696834077, |
|
"tokens_seen": 81920000 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004896916966143371, |
|
"loss": 3.5699, |
|
"theoretical_loss": 4.880654736816922, |
|
"tokens_seen": 85196800 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004891006241724985, |
|
"loss": 3.5148, |
|
"theoretical_loss": 4.8546724574857745, |
|
"tokens_seen": 88473600 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004885095517306601, |
|
"loss": 3.5161, |
|
"theoretical_loss": 4.829893630312808, |
|
"tokens_seen": 91750400 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004879184792888217, |
|
"loss": 3.456, |
|
"theoretical_loss": 4.806222425781916, |
|
"tokens_seen": 95027200 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0004873274068469832, |
|
"loss": 3.4477, |
|
"theoretical_loss": 4.783573622280253, |
|
"tokens_seen": 98304000 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.00048673633440514467, |
|
"loss": 3.4544, |
|
"theoretical_loss": 4.7618711237764035, |
|
"tokens_seen": 101580800 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0004861452619633062, |
|
"loss": 3.4465, |
|
"theoretical_loss": 4.741046724966468, |
|
"tokens_seen": 104857600 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0004855541895214677, |
|
"loss": 3.3894, |
|
"theoretical_loss": 4.721039076253046, |
|
"tokens_seen": 108134400 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0004849631170796293, |
|
"loss": 3.4326, |
|
"theoretical_loss": 4.701792811235861, |
|
"tokens_seen": 111411200 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0004843720446377908, |
|
"loss": 3.4057, |
|
"theoretical_loss": 4.683257807239925, |
|
"tokens_seen": 114688000 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.00048378097219595233, |
|
"loss": 3.4303, |
|
"theoretical_loss": 4.665388555430277, |
|
"tokens_seen": 117964800 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.00048318989975411385, |
|
"loss": 3.3601, |
|
"theoretical_loss": 4.648143621723734, |
|
"tokens_seen": 121241600 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.00048259882731227537, |
|
"loss": 3.3686, |
|
"theoretical_loss": 4.631485183343639, |
|
"tokens_seen": 124518400 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0004820077548704369, |
|
"loss": 3.3747, |
|
"theoretical_loss": 4.6153786287197125, |
|
"tokens_seen": 127795200 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.00048141668242859847, |
|
"loss": 3.3412, |
|
"theoretical_loss": 4.5997922106945, |
|
"tokens_seen": 131072000 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.00048082560998676, |
|
"loss": 3.3429, |
|
"theoretical_loss": 4.58469674479667, |
|
"tokens_seen": 134348800 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0004802345375449215, |
|
"loss": 3.3472, |
|
"theoretical_loss": 4.570065345782512, |
|
"tokens_seen": 137625600 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.00047964346510308303, |
|
"loss": 3.28, |
|
"theoretical_loss": 4.555873196808033, |
|
"tokens_seen": 140902400 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.00047905239266124455, |
|
"loss": 3.2929, |
|
"theoretical_loss": 4.542097346534794, |
|
"tokens_seen": 144179200 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.00047846132021940607, |
|
"loss": 3.2618, |
|
"theoretical_loss": 4.528716530238812, |
|
"tokens_seen": 147456000 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0004778702477775676, |
|
"loss": 3.2718, |
|
"theoretical_loss": 4.515711011618992, |
|
"tokens_seen": 150732800 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.00047727917533572917, |
|
"loss": 3.2578, |
|
"theoretical_loss": 4.503062442517334, |
|
"tokens_seen": 154009600 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0004766881028938907, |
|
"loss": 3.2513, |
|
"theoretical_loss": 4.4907537381892615, |
|
"tokens_seen": 157286400 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0004760970304520522, |
|
"loss": 3.2565, |
|
"theoretical_loss": 4.478768966115963, |
|
"tokens_seen": 160563200 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"objective/train/docs_used": 101104, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 3.2651968002319336, |
|
"objective/train/theoretical_loss": 4.467093246645215, |
|
"objective/train/tokens_used": 184300000, |
|
"theoretical_loss": 4.467093246645215, |
|
"tokens_seen": 163840000 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.00047550595801021373, |
|
"loss": 3.2099, |
|
"theoretical_loss": 4.467093246645215, |
|
"tokens_seen": 163840000 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.00047491488556837525, |
|
"loss": 3.2339, |
|
"theoretical_loss": 4.455712663993541, |
|
"tokens_seen": 167116800 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.00047432381312653677, |
|
"loss": 3.2285, |
|
"theoretical_loss": 4.444614186349425, |
|
"tokens_seen": 170393600 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.00047373274068469835, |
|
"loss": 3.2265, |
|
"theoretical_loss": 4.433785593991642, |
|
"tokens_seen": 173670400 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.00047314166824285987, |
|
"loss": 3.1712, |
|
"theoretical_loss": 4.423215414484155, |
|
"tokens_seen": 176947200 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0004725505958010214, |
|
"loss": 3.1556, |
|
"theoretical_loss": 4.412892864134137, |
|
"tokens_seen": 180224000 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0004719595233591829, |
|
"loss": 3.1573, |
|
"theoretical_loss": 4.402807795006074, |
|
"tokens_seen": 183500800 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.00047136845091734443, |
|
"loss": 3.1409, |
|
"theoretical_loss": 4.392950646875857, |
|
"tokens_seen": 186777600 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.00047077737847550595, |
|
"loss": 3.1509, |
|
"theoretical_loss": 4.383312403586528, |
|
"tokens_seen": 190054400 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0004701863060336675, |
|
"loss": 3.1539, |
|
"theoretical_loss": 4.373884553334271, |
|
"tokens_seen": 193331200 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.00046959523359182905, |
|
"loss": 3.1413, |
|
"theoretical_loss": 4.364659052470699, |
|
"tokens_seen": 196608000 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.00046900416114999057, |
|
"loss": 3.1588, |
|
"theoretical_loss": 4.355628292457278, |
|
"tokens_seen": 199884800 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0004684130887081521, |
|
"loss": 3.1403, |
|
"theoretical_loss": 4.346785069650667, |
|
"tokens_seen": 203161600 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0004678220162663136, |
|
"loss": 3.1111, |
|
"theoretical_loss": 4.338122557635156, |
|
"tokens_seen": 206438400 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.00046723094382447513, |
|
"loss": 3.1383, |
|
"theoretical_loss": 4.329634281850807, |
|
"tokens_seen": 209715200 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.00046663987138263665, |
|
"loss": 3.1209, |
|
"theoretical_loss": 4.321314096294248, |
|
"tokens_seen": 212992000 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0004660487989407982, |
|
"loss": 3.0778, |
|
"theoretical_loss": 4.3131561620937875, |
|
"tokens_seen": 216268800 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.00046545772649895975, |
|
"loss": 3.0869, |
|
"theoretical_loss": 4.305154927782223, |
|
"tokens_seen": 219545600 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.00046486665405712127, |
|
"loss": 3.0298, |
|
"theoretical_loss": 4.297305111109683, |
|
"tokens_seen": 222822400 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0004642755816152828, |
|
"loss": 3.1051, |
|
"theoretical_loss": 4.2896016822555945, |
|
"tokens_seen": 226099200 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0004636845091734443, |
|
"loss": 3.1019, |
|
"theoretical_loss": 4.282039848313611, |
|
"tokens_seen": 229376000 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.00046309343673160583, |
|
"loss": 3.1333, |
|
"theoretical_loss": 4.274615038936293, |
|
"tokens_seen": 232652800 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0004625023642897674, |
|
"loss": 3.1352, |
|
"theoretical_loss": 4.267322893037893, |
|
"tokens_seen": 235929600 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004619112918479289, |
|
"loss": 3.1255, |
|
"theoretical_loss": 4.26015924646374, |
|
"tokens_seen": 239206400 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.00046132021940609044, |
|
"loss": 3.1133, |
|
"theoretical_loss": 4.253120120543809, |
|
"tokens_seen": 242483200 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.00046072914696425197, |
|
"loss": 3.0263, |
|
"theoretical_loss": 4.24620171145605, |
|
"tokens_seen": 245760000 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004601380745224135, |
|
"loss": 3.0439, |
|
"theoretical_loss": 4.239400380332256, |
|
"tokens_seen": 249036800 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.000459547002080575, |
|
"loss": 3.0767, |
|
"theoretical_loss": 4.232712644045627, |
|
"tokens_seen": 252313600 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004589559296387366, |
|
"loss": 3.0505, |
|
"theoretical_loss": 4.226135166624862, |
|
"tokens_seen": 255590400 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004583648571968981, |
|
"loss": 3.103, |
|
"theoretical_loss": 4.21966475124477, |
|
"tokens_seen": 258867200 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004577737847550596, |
|
"loss": 3.0753, |
|
"theoretical_loss": 4.21329833274792, |
|
"tokens_seen": 262144000 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004571827123132211, |
|
"loss": 3.0362, |
|
"theoretical_loss": 4.207032970655965, |
|
"tokens_seen": 265420800 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0004565916398713826, |
|
"loss": 2.9727, |
|
"theoretical_loss": 4.2008658426329974, |
|
"tokens_seen": 268697600 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.00045600056742954413, |
|
"loss": 2.9651, |
|
"theoretical_loss": 4.194794238366544, |
|
"tokens_seen": 271974400 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0004554094949877057, |
|
"loss": 2.9526, |
|
"theoretical_loss": 4.188815553834879, |
|
"tokens_seen": 275251200 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.00045481842254586723, |
|
"loss": 2.9446, |
|
"theoretical_loss": 4.182927285931959, |
|
"tokens_seen": 278528000 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.00045422735010402875, |
|
"loss": 2.9296, |
|
"theoretical_loss": 4.177127027423776, |
|
"tokens_seen": 281804800 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.00045363627766219027, |
|
"loss": 2.9973, |
|
"theoretical_loss": 4.171412462212087, |
|
"tokens_seen": 285081600 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0004530452052203518, |
|
"loss": 2.9539, |
|
"theoretical_loss": 4.165781360883513, |
|
"tokens_seen": 288358400 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0004524541327785133, |
|
"loss": 2.9667, |
|
"theoretical_loss": 4.160231576523763, |
|
"tokens_seen": 291635200 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.00045186306033667483, |
|
"loss": 3.0032, |
|
"theoretical_loss": 4.1547610407784346, |
|
"tokens_seen": 294912000 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0004512719878948364, |
|
"loss": 2.9912, |
|
"theoretical_loss": 4.1493677601432815, |
|
"tokens_seen": 298188800 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.00045068091545299793, |
|
"loss": 2.986, |
|
"theoretical_loss": 4.1440498124682446, |
|
"tokens_seen": 301465600 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.00045008984301115945, |
|
"loss": 2.9958, |
|
"theoretical_loss": 4.138805343660737, |
|
"tokens_seen": 304742400 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.00044949877056932097, |
|
"loss": 3.0062, |
|
"theoretical_loss": 4.133632564574831, |
|
"tokens_seen": 308019200 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0004489076981274825, |
|
"loss": 2.9213, |
|
"theoretical_loss": 4.128529748073999, |
|
"tokens_seen": 311296000 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.000448316625685644, |
|
"loss": 2.9256, |
|
"theoretical_loss": 4.123495226256027, |
|
"tokens_seen": 314572800 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0004477255532438056, |
|
"loss": 2.9407, |
|
"theoretical_loss": 4.118527387829552, |
|
"tokens_seen": 317849600 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0004471344808019671, |
|
"loss": 2.9414, |
|
"theoretical_loss": 4.11362467563246, |
|
"tokens_seen": 321126400 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.00044654340836012863, |
|
"loss": 2.9174, |
|
"theoretical_loss": 4.108785584283144, |
|
"tokens_seen": 324403200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.6415514729857165, |
|
"debugging/distinct-1-grams": 0.7808741793632828, |
|
"debugging/distinct-2-grams": 0.9670734237074065, |
|
"debugging/entropy-1-grams": 6.5137716497370866, |
|
"debugging/entropy-2-grams": 7.60777283477522, |
|
"debugging/length": 573.1071428571429, |
|
"debugging/num_segments": 28, |
|
"epoch": 0.12, |
|
"objective/train/docs_used": 191543, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 3.150941848754883, |
|
"objective/train/theoretical_loss": 4.104008657956216, |
|
"objective/train/tokens_used": 348140000, |
|
"theoretical_loss": 4.104008657956216, |
|
"tokens_seen": 327680000 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.00044595233591829015, |
|
"loss": 2.9292, |
|
"theoretical_loss": 4.104008657956216, |
|
"tokens_seen": 327680000 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.00044536126347645167, |
|
"loss": 2.9323, |
|
"theoretical_loss": 4.099292488274917, |
|
"tokens_seen": 330956800 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0004447701910346132, |
|
"loss": 2.9439, |
|
"theoretical_loss": 4.094635712313026, |
|
"tokens_seen": 334233600 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.00044417911859277476, |
|
"loss": 2.9369, |
|
"theoretical_loss": 4.090037010699552, |
|
"tokens_seen": 337510400 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0004435880461509363, |
|
"loss": 2.9069, |
|
"theoretical_loss": 4.085495105819987, |
|
"tokens_seen": 340787200 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0004429969737090978, |
|
"loss": 2.9343, |
|
"theoretical_loss": 4.081008760108329, |
|
"tokens_seen": 344064000 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0004424059012672593, |
|
"loss": 2.9185, |
|
"theoretical_loss": 4.076576774424465, |
|
"tokens_seen": 347340800 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.00044181482882542085, |
|
"loss": 2.9285, |
|
"theoretical_loss": 4.072197986511911, |
|
"tokens_seen": 350617600 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.00044122375638358237, |
|
"loss": 2.8793, |
|
"theoretical_loss": 4.067871269531193, |
|
"tokens_seen": 353894400 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.0004406326839417439, |
|
"loss": 2.9095, |
|
"theoretical_loss": 4.063595530664515, |
|
"tokens_seen": 357171200 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.00044004161149990546, |
|
"loss": 2.9123, |
|
"theoretical_loss": 4.059369709787625, |
|
"tokens_seen": 360448000 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.000439450539058067, |
|
"loss": 2.9597, |
|
"theoretical_loss": 4.055192778205064, |
|
"tokens_seen": 363724800 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.0004388594666162285, |
|
"loss": 2.938, |
|
"theoretical_loss": 4.051063737445231, |
|
"tokens_seen": 367001600 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.00043826839417439, |
|
"loss": 2.9016, |
|
"theoretical_loss": 4.046981618111936, |
|
"tokens_seen": 370278400 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.00043767732173255155, |
|
"loss": 2.9464, |
|
"theoretical_loss": 4.042945478789308, |
|
"tokens_seen": 373555200 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.00043708624929071307, |
|
"loss": 2.9239, |
|
"theoretical_loss": 4.038954404997148, |
|
"tokens_seen": 376832000 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.00043649517684887464, |
|
"loss": 2.8813, |
|
"theoretical_loss": 4.035007508193978, |
|
"tokens_seen": 380108800 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.00043590410440703616, |
|
"loss": 2.9102, |
|
"theoretical_loss": 4.031103924825214, |
|
"tokens_seen": 383385600 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0004353130319651977, |
|
"loss": 2.8759, |
|
"theoretical_loss": 4.027242815414074, |
|
"tokens_seen": 386662400 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0004347219595233592, |
|
"loss": 2.8722, |
|
"theoretical_loss": 4.023423363692939, |
|
"tokens_seen": 389939200 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0004341308870815207, |
|
"loss": 2.9149, |
|
"theoretical_loss": 4.019644775773043, |
|
"tokens_seen": 393216000 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.00043353981463968225, |
|
"loss": 2.912, |
|
"theoretical_loss": 4.015906279350517, |
|
"tokens_seen": 396492800 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0004329487421978438, |
|
"loss": 2.8936, |
|
"theoretical_loss": 4.01220712294689, |
|
"tokens_seen": 399769600 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.00043235766975600534, |
|
"loss": 2.8953, |
|
"theoretical_loss": 4.008546575182286, |
|
"tokens_seen": 403046400 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.00043176659731416686, |
|
"loss": 2.8929, |
|
"theoretical_loss": 4.0049239240796695, |
|
"tokens_seen": 406323200 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0004311755248723284, |
|
"loss": 2.9099, |
|
"theoretical_loss": 4.001338476398553, |
|
"tokens_seen": 409600000 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0004305844524304899, |
|
"loss": 2.8873, |
|
"theoretical_loss": 3.9977895569967217, |
|
"tokens_seen": 412876800 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0004299933799886514, |
|
"loss": 2.9155, |
|
"theoretical_loss": 3.99427650821855, |
|
"tokens_seen": 416153600 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.00042940230754681295, |
|
"loss": 2.9272, |
|
"theoretical_loss": 3.990798689308642, |
|
"tokens_seen": 419430400 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0004288112351049745, |
|
"loss": 2.9085, |
|
"theoretical_loss": 3.987355475849527, |
|
"tokens_seen": 422707200 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.000428220162663136, |
|
"loss": 2.9369, |
|
"theoretical_loss": 3.98394625922226, |
|
"tokens_seen": 425984000 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0004276290902212975, |
|
"loss": 2.9425, |
|
"theoretical_loss": 3.9805704460888256, |
|
"tokens_seen": 429260800 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.00042703801777945903, |
|
"loss": 2.9333, |
|
"theoretical_loss": 3.9772274578953013, |
|
"tokens_seen": 432537600 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.00042644694533762055, |
|
"loss": 2.899, |
|
"theoretical_loss": 3.973916730394796, |
|
"tokens_seen": 435814400 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.00042585587289578207, |
|
"loss": 2.9102, |
|
"theoretical_loss": 3.970637713189244, |
|
"tokens_seen": 439091200 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.00042526480045394365, |
|
"loss": 2.9236, |
|
"theoretical_loss": 3.967389869289161, |
|
"tokens_seen": 442368000 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.00042467372801210517, |
|
"loss": 2.9171, |
|
"theoretical_loss": 3.964172674690542, |
|
"tokens_seen": 445644800 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.0004240826555702667, |
|
"loss": 2.9071, |
|
"theoretical_loss": 3.9609856179681078, |
|
"tokens_seen": 448921600 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.0004234915831284282, |
|
"loss": 2.8795, |
|
"theoretical_loss": 3.957828199884155, |
|
"tokens_seen": 452198400 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.00042290051068658973, |
|
"loss": 2.858, |
|
"theoretical_loss": 3.9546999330123036, |
|
"tokens_seen": 455475200 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.00042230943824475125, |
|
"loss": 2.871, |
|
"theoretical_loss": 3.951600341375469, |
|
"tokens_seen": 458752000 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0004217183658029128, |
|
"loss": 2.8548, |
|
"theoretical_loss": 3.9485289600974305, |
|
"tokens_seen": 462028800 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.00042112729336107435, |
|
"loss": 2.8076, |
|
"theoretical_loss": 3.945485335067386, |
|
"tokens_seen": 465305600 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.00042053622091923587, |
|
"loss": 2.8602, |
|
"theoretical_loss": 3.9424690226169212, |
|
"tokens_seen": 468582400 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0004199451484773974, |
|
"loss": 2.8829, |
|
"theoretical_loss": 3.9394795892088585, |
|
"tokens_seen": 471859200 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0004193540760355589, |
|
"loss": 2.8454, |
|
"theoretical_loss": 3.9365166111374608, |
|
"tokens_seen": 475136000 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.00041876300359372043, |
|
"loss": 2.8048, |
|
"theoretical_loss": 3.933579674239507, |
|
"tokens_seen": 478412800 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.00041817193115188195, |
|
"loss": 2.8377, |
|
"theoretical_loss": 3.930668373615765, |
|
"tokens_seen": 481689600 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0004175808587100435, |
|
"loss": 2.8013, |
|
"theoretical_loss": 3.9277823133624366, |
|
"tokens_seen": 484966400 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.00041698978626820505, |
|
"loss": 2.8266, |
|
"theoretical_loss": 3.9249211063121283, |
|
"tokens_seen": 488243200 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"objective/train/docs_used": 280311, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.8735318183898926, |
|
"objective/train/theoretical_loss": 3.9220843737839752, |
|
"objective/train/tokens_used": 511980000, |
|
"theoretical_loss": 3.9220843737839752, |
|
"tokens_seen": 491520000 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.00041639871382636657, |
|
"loss": 2.8674, |
|
"theoretical_loss": 3.9220843737839752, |
|
"tokens_seen": 491520000 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0004158076413845281, |
|
"loss": 2.9015, |
|
"theoretical_loss": 3.919271745342514, |
|
"tokens_seen": 494796800 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0004152165689426896, |
|
"loss": 2.8371, |
|
"theoretical_loss": 3.916482858564957, |
|
"tokens_seen": 498073600 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.00041462549650085113, |
|
"loss": 2.7965, |
|
"theoretical_loss": 3.9137173588165135, |
|
"tokens_seen": 501350400 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0004140344240590127, |
|
"loss": 2.8257, |
|
"theoretical_loss": 3.9109748990334348, |
|
"tokens_seen": 504627200 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0004134433516171742, |
|
"loss": 2.8352, |
|
"theoretical_loss": 3.908255139513467, |
|
"tokens_seen": 507904000 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.00041285227917533575, |
|
"loss": 2.8285, |
|
"theoretical_loss": 3.905557747713412, |
|
"tokens_seen": 511180800 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.00041226120673349727, |
|
"loss": 2.8056, |
|
"theoretical_loss": 3.90288239805351, |
|
"tokens_seen": 514457600 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0004116701342916588, |
|
"loss": 2.7956, |
|
"theoretical_loss": 3.9002287717283783, |
|
"tokens_seen": 517734400 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0004110790618498203, |
|
"loss": 2.7715, |
|
"theoretical_loss": 3.8975965565242423, |
|
"tokens_seen": 521011200 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0004104879894079819, |
|
"loss": 2.7886, |
|
"theoretical_loss": 3.894985446642206, |
|
"tokens_seen": 524288000 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0004098969169661434, |
|
"loss": 2.7978, |
|
"theoretical_loss": 3.8923951425273398, |
|
"tokens_seen": 527564800 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0004093058445243049, |
|
"loss": 2.7747, |
|
"theoretical_loss": 3.889825350703344, |
|
"tokens_seen": 530841600 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.00040871477208246645, |
|
"loss": 2.8062, |
|
"theoretical_loss": 3.8872757836125844, |
|
"tokens_seen": 534118400 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.00040812369964062797, |
|
"loss": 2.8084, |
|
"theoretical_loss": 3.8847461594612884, |
|
"tokens_seen": 537395200 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0004075326271987895, |
|
"loss": 2.8153, |
|
"theoretical_loss": 3.882236202069703, |
|
"tokens_seen": 540672000 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.000406941554756951, |
|
"loss": 2.8584, |
|
"theoretical_loss": 3.8797456407270343, |
|
"tokens_seen": 543948800 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0004063504823151126, |
|
"loss": 2.8143, |
|
"theoretical_loss": 3.8772742100509774, |
|
"tokens_seen": 547225600 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0004057594098732741, |
|
"loss": 2.7896, |
|
"theoretical_loss": 3.874821649851678, |
|
"tokens_seen": 550502400 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0004051683374314356, |
|
"loss": 2.8271, |
|
"theoretical_loss": 3.8723877049999444, |
|
"tokens_seen": 553779200 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.00040457726498959715, |
|
"loss": 2.822, |
|
"theoretical_loss": 3.869972125299568, |
|
"tokens_seen": 557056000 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.00040398619254775867, |
|
"loss": 2.8469, |
|
"theoretical_loss": 3.867574665363595, |
|
"tokens_seen": 560332800 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0004033951201059202, |
|
"loss": 2.8082, |
|
"theoretical_loss": 3.865195084494398, |
|
"tokens_seen": 563609600 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.00040280404766408176, |
|
"loss": 2.845, |
|
"theoretical_loss": 3.8628331465674224, |
|
"tokens_seen": 566886400 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0004022129752222433, |
|
"loss": 2.8963, |
|
"theoretical_loss": 3.860488619918462, |
|
"tokens_seen": 570163200 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0004016219027804048, |
|
"loss": 2.8779, |
|
"theoretical_loss": 3.858161277234349, |
|
"tokens_seen": 573440000 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0004010308303385663, |
|
"loss": 2.8456, |
|
"theoretical_loss": 3.855850895446925, |
|
"tokens_seen": 576716800 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.00040043975789672785, |
|
"loss": 2.8734, |
|
"theoretical_loss": 3.8535572556301823, |
|
"tokens_seen": 579993600 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0003998486854548893, |
|
"loss": 2.8319, |
|
"theoretical_loss": 3.851280142900463, |
|
"tokens_seen": 583270400 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0003992576130130509, |
|
"loss": 2.8623, |
|
"theoretical_loss": 3.8490193463196127, |
|
"tokens_seen": 586547200 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0003986665405712124, |
|
"loss": 2.8307, |
|
"theoretical_loss": 3.8467746588009692, |
|
"tokens_seen": 589824000 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.00039807546812937393, |
|
"loss": 2.8052, |
|
"theoretical_loss": 3.8445458770181116, |
|
"tokens_seen": 593100800 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.00039748439568753545, |
|
"loss": 2.7865, |
|
"theoretical_loss": 3.842332801316254, |
|
"tokens_seen": 596377600 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.00039689332324569697, |
|
"loss": 2.8027, |
|
"theoretical_loss": 3.840135235626204, |
|
"tokens_seen": 599654400 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0003963022508038585, |
|
"loss": 2.7811, |
|
"theoretical_loss": 3.8379529873807945, |
|
"tokens_seen": 602931200 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.00039571117836202, |
|
"loss": 2.7893, |
|
"theoretical_loss": 3.835785867433705, |
|
"tokens_seen": 606208000 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0003951201059201816, |
|
"loss": 2.7681, |
|
"theoretical_loss": 3.8336336899805916, |
|
"tokens_seen": 609484800 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0003945290334783431, |
|
"loss": 2.8031, |
|
"theoretical_loss": 3.8314962724824495, |
|
"tokens_seen": 612761600 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.00039393796103650463, |
|
"loss": 2.7924, |
|
"theoretical_loss": 3.8293734355911297, |
|
"tokens_seen": 616038400 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.00039334688859466615, |
|
"loss": 2.767, |
|
"theoretical_loss": 3.8272650030769433, |
|
"tokens_seen": 619315200 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.00039275581615282767, |
|
"loss": 2.7884, |
|
"theoretical_loss": 3.8251708017582793, |
|
"tokens_seen": 622592000 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0003921647437109892, |
|
"loss": 2.7508, |
|
"theoretical_loss": 3.8230906614331737, |
|
"tokens_seen": 625868800 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.00039157367126915077, |
|
"loss": 2.7596, |
|
"theoretical_loss": 3.821024414812761, |
|
"tokens_seen": 629145600 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0003909825988273123, |
|
"loss": 2.7717, |
|
"theoretical_loss": 3.818971897456552, |
|
"tokens_seen": 632422400 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0003903915263854738, |
|
"loss": 2.8154, |
|
"theoretical_loss": 3.8169329477094784, |
|
"tokens_seen": 635699200 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.00038980045394363533, |
|
"loss": 2.8062, |
|
"theoretical_loss": 3.814907406640639, |
|
"tokens_seen": 638976000 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.00038920938150179685, |
|
"loss": 2.8738, |
|
"theoretical_loss": 3.8128951179837056, |
|
"tokens_seen": 642252800 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.00038861830905995837, |
|
"loss": 2.8178, |
|
"theoretical_loss": 3.8108959280789243, |
|
"tokens_seen": 645529600 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.00038802723661811994, |
|
"loss": 2.8169, |
|
"theoretical_loss": 3.8089096858166718, |
|
"tokens_seen": 648806400 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.00038743616417628147, |
|
"loss": 2.7933, |
|
"theoretical_loss": 3.8069362425825037, |
|
"tokens_seen": 652083200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.5192820648589896, |
|
"debugging/distinct-1-grams": 0.829841073421897, |
|
"debugging/distinct-2-grams": 0.9832158471819804, |
|
"debugging/entropy-1-grams": 5.99510879346684, |
|
"debugging/entropy-2-grams": 6.734429878009261, |
|
"debugging/length": 553.6666666666666, |
|
"debugging/num_segments": 12, |
|
"epoch": 0.23, |
|
"objective/train/docs_used": 369170, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.8842573165893555, |
|
"objective/train/theoretical_loss": 3.8049754522036645, |
|
"objective/train/tokens_used": 675820000, |
|
"theoretical_loss": 3.8049754522036645, |
|
"tokens_seen": 655360000 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.000386845091734443, |
|
"loss": 2.7845, |
|
"theoretical_loss": 3.8049754522036645, |
|
"tokens_seen": 655360000 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0003862540192926045, |
|
"loss": 2.8021, |
|
"theoretical_loss": 3.8030271708970003, |
|
"tokens_seen": 658636800 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.00038566294685076603, |
|
"loss": 2.7696, |
|
"theoretical_loss": 3.801091257218237, |
|
"tokens_seen": 661913600 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.00038507187440892755, |
|
"loss": 2.7535, |
|
"theoretical_loss": 3.79916757201258, |
|
"tokens_seen": 665190400 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0003844808019670891, |
|
"loss": 2.7957, |
|
"theoretical_loss": 3.7972559783665965, |
|
"tokens_seen": 668467200 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.00038388972952525064, |
|
"loss": 2.7915, |
|
"theoretical_loss": 3.7953563415613325, |
|
"tokens_seen": 671744000 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.00038329865708341216, |
|
"loss": 2.8197, |
|
"theoretical_loss": 3.7934685290266454, |
|
"tokens_seen": 675020800 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0003827075846415737, |
|
"loss": 2.8175, |
|
"theoretical_loss": 3.7915924102966914, |
|
"tokens_seen": 678297600 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0003821165121997352, |
|
"loss": 2.8248, |
|
"theoretical_loss": 3.789727856966552, |
|
"tokens_seen": 681574400 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.00038152543975789673, |
|
"loss": 2.8221, |
|
"theoretical_loss": 3.787874742649958, |
|
"tokens_seen": 684851200 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.00038093436731605825, |
|
"loss": 2.8231, |
|
"theoretical_loss": 3.786032942938073, |
|
"tokens_seen": 688128000 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0003803432948742198, |
|
"loss": 2.8081, |
|
"theoretical_loss": 3.784202335359316, |
|
"tokens_seen": 691404800 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0003797640438812181, |
|
"loss": 2.814, |
|
"theoretical_loss": 3.7823827993401844, |
|
"tokens_seen": 694681600 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0003791729714393796, |
|
"loss": 2.7961, |
|
"theoretical_loss": 3.7805742161670466, |
|
"tokens_seen": 697958400 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.00037858189899754114, |
|
"loss": 2.7582, |
|
"theoretical_loss": 3.7787764689488847, |
|
"tokens_seen": 701235200 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.00037799082655570266, |
|
"loss": 2.7534, |
|
"theoretical_loss": 3.77698944258095, |
|
"tokens_seen": 704512000 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0003773997541138642, |
|
"loss": 2.7708, |
|
"theoretical_loss": 3.7752130237093064, |
|
"tokens_seen": 707788800 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.00037680868167202575, |
|
"loss": 2.7862, |
|
"theoretical_loss": 3.773447100696245, |
|
"tokens_seen": 711065600 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0003762176092301873, |
|
"loss": 2.8106, |
|
"theoretical_loss": 3.771691563586529, |
|
"tokens_seen": 714342400 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0003756265367883488, |
|
"loss": 2.7718, |
|
"theoretical_loss": 3.7699463040744616, |
|
"tokens_seen": 717619200 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0003750354643465103, |
|
"loss": 2.7397, |
|
"theoretical_loss": 3.768211215471741, |
|
"tokens_seen": 720896000 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.00037444439190467184, |
|
"loss": 2.7226, |
|
"theoretical_loss": 3.766486192676084, |
|
"tokens_seen": 724172800 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.00037385331946283336, |
|
"loss": 2.6997, |
|
"theoretical_loss": 3.764771132140602, |
|
"tokens_seen": 727449600 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.00037326224702099493, |
|
"loss": 2.7344, |
|
"theoretical_loss": 3.763065931843898, |
|
"tokens_seen": 730726400 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.00037267117457915645, |
|
"loss": 2.7288, |
|
"theoretical_loss": 3.7613704912608723, |
|
"tokens_seen": 734003200 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.000372080102137318, |
|
"loss": 2.7765, |
|
"theoretical_loss": 3.759684711334213, |
|
"tokens_seen": 737280000 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0003714890296954795, |
|
"loss": 2.786, |
|
"theoretical_loss": 3.7580084944465555, |
|
"tokens_seen": 740556800 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.000370897957253641, |
|
"loss": 2.789, |
|
"theoretical_loss": 3.7563417443932905, |
|
"tokens_seen": 743833600 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.00037030688481180254, |
|
"loss": 2.7933, |
|
"theoretical_loss": 3.754684366355999, |
|
"tokens_seen": 747110400 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.00036971581236996406, |
|
"loss": 2.7431, |
|
"theoretical_loss": 3.753036266876505, |
|
"tokens_seen": 750387200 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.00036912473992812563, |
|
"loss": 2.7615, |
|
"theoretical_loss": 3.751397353831524, |
|
"tokens_seen": 753664000 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.00036853366748628715, |
|
"loss": 2.7141, |
|
"theoretical_loss": 3.749767536407891, |
|
"tokens_seen": 756940800 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0003679425950444487, |
|
"loss": 2.7317, |
|
"theoretical_loss": 3.7481467250783504, |
|
"tokens_seen": 760217600 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0003673515226026102, |
|
"loss": 2.737, |
|
"theoretical_loss": 3.746534831577904, |
|
"tokens_seen": 763494400 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0003667604501607717, |
|
"loss": 2.6981, |
|
"theoretical_loss": 3.744931768880681, |
|
"tokens_seen": 766771200 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.00036616937771893324, |
|
"loss": 2.7117, |
|
"theoretical_loss": 3.743337451177343, |
|
"tokens_seen": 770048000 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0003655783052770948, |
|
"loss": 2.6921, |
|
"theoretical_loss": 3.74175179385298, |
|
"tokens_seen": 773324800 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.00036498723283525633, |
|
"loss": 2.783, |
|
"theoretical_loss": 3.740174713465512, |
|
"tokens_seen": 776601600 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.00036439616039341785, |
|
"loss": 2.7398, |
|
"theoretical_loss": 3.7386061277245655, |
|
"tokens_seen": 779878400 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0003638050879515794, |
|
"loss": 2.7741, |
|
"theoretical_loss": 3.7370459554708146, |
|
"tokens_seen": 783155200 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0003632140155097409, |
|
"loss": 2.803, |
|
"theoretical_loss": 3.735494116655784, |
|
"tokens_seen": 786432000 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0003626229430679024, |
|
"loss": 2.7697, |
|
"theoretical_loss": 3.733950532322087, |
|
"tokens_seen": 789708800 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.000362031870626064, |
|
"loss": 2.7877, |
|
"theoretical_loss": 3.7324151245841044, |
|
"tokens_seen": 792985600 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.00036144079818422546, |
|
"loss": 2.7828, |
|
"theoretical_loss": 3.730887816609077, |
|
"tokens_seen": 796262400 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.000360849725742387, |
|
"loss": 2.7742, |
|
"theoretical_loss": 3.729368532598609, |
|
"tokens_seen": 799539200 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0003602586533005485, |
|
"loss": 2.7669, |
|
"theoretical_loss": 3.7278571977705734, |
|
"tokens_seen": 802816000 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.00035966758085871, |
|
"loss": 2.783, |
|
"theoretical_loss": 3.7263537383414023, |
|
"tokens_seen": 806092800 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.00035907650841687154, |
|
"loss": 2.7676, |
|
"theoretical_loss": 3.7248580815087626, |
|
"tokens_seen": 809369600 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.00035848543597503306, |
|
"loss": 2.7941, |
|
"theoretical_loss": 3.7233701554345924, |
|
"tokens_seen": 812646400 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.00035789436353319464, |
|
"loss": 2.7726, |
|
"theoretical_loss": 3.721889889228506, |
|
"tokens_seen": 815923200 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"objective/train/docs_used": 458185, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 3.146359443664551, |
|
"objective/train/theoretical_loss": 3.720417212931543, |
|
"objective/train/tokens_used": 839660000, |
|
"theoretical_loss": 3.720417212931543, |
|
"tokens_seen": 819200000 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.00035730329109135616, |
|
"loss": 2.7644, |
|
"theoretical_loss": 3.720417212931543, |
|
"tokens_seen": 819200000 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0003567122186495177, |
|
"loss": 2.7402, |
|
"theoretical_loss": 3.7189520575002666, |
|
"tokens_seen": 822476800 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0003561211462076792, |
|
"loss": 2.7263, |
|
"theoretical_loss": 3.717494354791188, |
|
"tokens_seen": 825753600 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0003555418952146775, |
|
"loss": 2.7261, |
|
"theoretical_loss": 3.716044037545523, |
|
"tokens_seen": 829030400 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.00035495082277283904, |
|
"loss": 2.7537, |
|
"theoretical_loss": 3.714601039374263, |
|
"tokens_seen": 832307200 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0003543597503310006, |
|
"loss": 2.6884, |
|
"theoretical_loss": 3.7131652947435536, |
|
"tokens_seen": 835584000 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.00035376867788916214, |
|
"loss": 2.6958, |
|
"theoretical_loss": 3.7117367389603793, |
|
"tokens_seen": 838860800 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.00035317760544732366, |
|
"loss": 2.7246, |
|
"theoretical_loss": 3.710315308158541, |
|
"tokens_seen": 842137600 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0003525865330054852, |
|
"loss": 2.7493, |
|
"theoretical_loss": 3.7089009392849173, |
|
"tokens_seen": 845414400 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.00035199546056364665, |
|
"loss": 2.6882, |
|
"theoretical_loss": 3.7074935700860143, |
|
"tokens_seen": 848691200 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.00035140438812180817, |
|
"loss": 2.7039, |
|
"theoretical_loss": 3.706093139094781, |
|
"tokens_seen": 851968000 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00035082513712880655, |
|
"loss": 2.7232, |
|
"theoretical_loss": 3.7046995856176954, |
|
"tokens_seen": 855244800 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00035023406468696807, |
|
"loss": 2.7433, |
|
"theoretical_loss": 3.703312849722111, |
|
"tokens_seen": 858521600 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0003496429922451296, |
|
"loss": 2.7097, |
|
"theoretical_loss": 3.701932872223858, |
|
"tokens_seen": 861798400 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0003490519198032911, |
|
"loss": 2.6895, |
|
"theoretical_loss": 3.7005595946750924, |
|
"tokens_seen": 865075200 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00034846084736145263, |
|
"loss": 2.7131, |
|
"theoretical_loss": 3.699192959352386, |
|
"tokens_seen": 868352000 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00034786977491961415, |
|
"loss": 2.6754, |
|
"theoretical_loss": 3.6978329092450557, |
|
"tokens_seen": 871628800 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00034727870247777573, |
|
"loss": 2.7166, |
|
"theoretical_loss": 3.6964793880437226, |
|
"tokens_seen": 874905600 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00034668763003593725, |
|
"loss": 2.7085, |
|
"theoretical_loss": 3.6951323401290974, |
|
"tokens_seen": 878182400 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00034609655759409877, |
|
"loss": 2.7027, |
|
"theoretical_loss": 3.6937917105609834, |
|
"tokens_seen": 881459200 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0003455054851522603, |
|
"loss": 2.7318, |
|
"theoretical_loss": 3.692457445067501, |
|
"tokens_seen": 884736000 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0003449144127104218, |
|
"loss": 2.7378, |
|
"theoretical_loss": 3.6911294900345166, |
|
"tokens_seen": 888012800 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.00034432334026858333, |
|
"loss": 2.6772, |
|
"theoretical_loss": 3.6898077924952775, |
|
"tokens_seen": 891289600 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.00034373226782674485, |
|
"loss": 2.7122, |
|
"theoretical_loss": 3.6884923001202505, |
|
"tokens_seen": 894566400 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.00034314119538490643, |
|
"loss": 2.6732, |
|
"theoretical_loss": 3.6871829612071583, |
|
"tokens_seen": 897843200 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0003425501229430679, |
|
"loss": 2.6777, |
|
"theoretical_loss": 3.6858797246711976, |
|
"tokens_seen": 901120000 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0003419590505012294, |
|
"loss": 2.6763, |
|
"theoretical_loss": 3.684582540035456, |
|
"tokens_seen": 904396800 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.00034136797805939094, |
|
"loss": 2.6968, |
|
"theoretical_loss": 3.683291357421508, |
|
"tokens_seen": 907673600 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00034078872706638926, |
|
"loss": 2.7399, |
|
"theoretical_loss": 3.682006127540184, |
|
"tokens_seen": 910950400 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.0003401976546245508, |
|
"loss": 2.7273, |
|
"theoretical_loss": 3.680726801682522, |
|
"tokens_seen": 914227200 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00033960658218271236, |
|
"loss": 2.7208, |
|
"theoretical_loss": 3.679453331710889, |
|
"tokens_seen": 917504000 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.0003390155097408739, |
|
"loss": 2.7396, |
|
"theoretical_loss": 3.6781856700502646, |
|
"tokens_seen": 920780800 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.0003384244372990354, |
|
"loss": 2.7219, |
|
"theoretical_loss": 3.6769237696796933, |
|
"tokens_seen": 924057600 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.0003378333648571969, |
|
"loss": 2.7145, |
|
"theoretical_loss": 3.6756675841238913, |
|
"tokens_seen": 927334400 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00033724229241535844, |
|
"loss": 2.7389, |
|
"theoretical_loss": 3.6744170674450176, |
|
"tokens_seen": 930611200 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00033665121997351996, |
|
"loss": 2.7275, |
|
"theoretical_loss": 3.673172174234587, |
|
"tokens_seen": 933888000 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00033606014753168154, |
|
"loss": 2.7496, |
|
"theoretical_loss": 3.6719328596055423, |
|
"tokens_seen": 937164800 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.00033546907508984306, |
|
"loss": 2.6951, |
|
"theoretical_loss": 3.670699079184467, |
|
"tokens_seen": 940441600 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0003348780026480046, |
|
"loss": 2.7434, |
|
"theoretical_loss": 3.669470789103942, |
|
"tokens_seen": 943718400 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0003342869302061661, |
|
"loss": 2.7029, |
|
"theoretical_loss": 3.6682479459950446, |
|
"tokens_seen": 946995200 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.00033369585776432757, |
|
"loss": 2.6727, |
|
"theoretical_loss": 3.6670305069799785, |
|
"tokens_seen": 950272000 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0003331047853224891, |
|
"loss": 2.6661, |
|
"theoretical_loss": 3.6658184296648457, |
|
"tokens_seen": 953548800 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0003325137128806506, |
|
"loss": 2.6413, |
|
"theoretical_loss": 3.6646116721325415, |
|
"tokens_seen": 956825600 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0003319226404388122, |
|
"loss": 2.6338, |
|
"theoretical_loss": 3.6634101929357836, |
|
"tokens_seen": 960102400 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0003313315679969737, |
|
"loss": 2.6128, |
|
"theoretical_loss": 3.6622139510902625, |
|
"tokens_seen": 963379200 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0003307404955551352, |
|
"loss": 2.6289, |
|
"theoretical_loss": 3.6610229060679167, |
|
"tokens_seen": 966656000 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00033014942311329675, |
|
"loss": 2.68, |
|
"theoretical_loss": 3.659837017790328, |
|
"tokens_seen": 969932800 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00032955835067145827, |
|
"loss": 2.6894, |
|
"theoretical_loss": 3.658656246622233, |
|
"tokens_seen": 973209600 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0003289672782296198, |
|
"loss": 2.6932, |
|
"theoretical_loss": 3.6574805533651515, |
|
"tokens_seen": 976486400 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00032837620578778136, |
|
"loss": 2.6035, |
|
"theoretical_loss": 3.6563098992511267, |
|
"tokens_seen": 979763200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.4766946989185522, |
|
"debugging/distinct-1-grams": 0.7725611660367836, |
|
"debugging/distinct-2-grams": 0.9530799205344639, |
|
"debugging/entropy-1-grams": 5.710381177727143, |
|
"debugging/entropy-2-grams": 6.414420510766599, |
|
"debugging/length": 521.8, |
|
"debugging/num_segments": 10, |
|
"epoch": 0.35, |
|
"objective/train/docs_used": 548781, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.4019718170166016, |
|
"objective/train/theoretical_loss": 3.655144245936574, |
|
"objective/train/tokens_used": 1003500000, |
|
"theoretical_loss": 3.655144245936574, |
|
"tokens_seen": 983040000 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0003277851333459429, |
|
"loss": 2.6843, |
|
"theoretical_loss": 3.655144245936574, |
|
"tokens_seen": 983040000 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0003271940609041044, |
|
"loss": 2.6468, |
|
"theoretical_loss": 3.653983555496242, |
|
"tokens_seen": 986316800 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0003266029884622659, |
|
"loss": 2.6824, |
|
"theoretical_loss": 3.6528277904172755, |
|
"tokens_seen": 989593600 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00032601191602042745, |
|
"loss": 2.6955, |
|
"theoretical_loss": 3.6516769135933815, |
|
"tokens_seen": 992870400 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.00032542084357858897, |
|
"loss": 2.7883, |
|
"theoretical_loss": 3.650530888319103, |
|
"tokens_seen": 996147200 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.00032482977113675054, |
|
"loss": 2.7914, |
|
"theoretical_loss": 3.649389678284182, |
|
"tokens_seen": 999424000 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.00032423869869491206, |
|
"loss": 2.7986, |
|
"theoretical_loss": 3.6482532475680287, |
|
"tokens_seen": 1002700800 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0003236476262530736, |
|
"loss": 2.7822, |
|
"theoretical_loss": 3.6471215606342833, |
|
"tokens_seen": 1005977600 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0003230565538112351, |
|
"loss": 2.7971, |
|
"theoretical_loss": 3.645994582325468, |
|
"tokens_seen": 1009254400 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0003224654813693966, |
|
"loss": 2.7829, |
|
"theoretical_loss": 3.6448722778577327, |
|
"tokens_seen": 1012531200 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.00032187440892755814, |
|
"loss": 2.7909, |
|
"theoretical_loss": 3.6437546128156946, |
|
"tokens_seen": 1015808000 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0003212833364857197, |
|
"loss": 2.7787, |
|
"theoretical_loss": 3.6426415531473566, |
|
"tokens_seen": 1019084800 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00032069226404388124, |
|
"loss": 2.7566, |
|
"theoretical_loss": 3.641533065159118, |
|
"tokens_seen": 1022361600 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00032010119160204276, |
|
"loss": 2.742, |
|
"theoretical_loss": 3.6404291155108712, |
|
"tokens_seen": 1025638400 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0003195101191602043, |
|
"loss": 2.7427, |
|
"theoretical_loss": 3.639329671211173, |
|
"tokens_seen": 1028915200 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0003189190467183658, |
|
"loss": 2.7428, |
|
"theoretical_loss": 3.6382346996125055, |
|
"tokens_seen": 1032192000 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0003183279742765273, |
|
"loss": 2.7289, |
|
"theoretical_loss": 3.6371441684066097, |
|
"tokens_seen": 1035468800 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00031773690183468884, |
|
"loss": 2.7481, |
|
"theoretical_loss": 3.6360580456199036, |
|
"tokens_seen": 1038745600 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0003171458293928504, |
|
"loss": 2.7757, |
|
"theoretical_loss": 3.6349762996089683, |
|
"tokens_seen": 1042022400 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00031655475695101194, |
|
"loss": 2.7142, |
|
"theoretical_loss": 3.633898899056115, |
|
"tokens_seen": 1045299200 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00031596368450917346, |
|
"loss": 2.7581, |
|
"theoretical_loss": 3.6328258129650246, |
|
"tokens_seen": 1048576000 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.000315372612067335, |
|
"loss": 2.7326, |
|
"theoretical_loss": 3.6317570106564565, |
|
"tokens_seen": 1051852800 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0003147815396254965, |
|
"loss": 2.7498, |
|
"theoretical_loss": 3.6306924617640295, |
|
"tokens_seen": 1055129600 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.000314190467183658, |
|
"loss": 2.7604, |
|
"theoretical_loss": 3.6296321362300716, |
|
"tokens_seen": 1058406400 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0003135993947418196, |
|
"loss": 2.7208, |
|
"theoretical_loss": 3.6285760043015385, |
|
"tokens_seen": 1061683200 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0003130083222999811, |
|
"loss": 2.7355, |
|
"theoretical_loss": 3.6275240365259958, |
|
"tokens_seen": 1064960000 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.00031241724985814264, |
|
"loss": 2.7018, |
|
"theoretical_loss": 3.6264762037476683, |
|
"tokens_seen": 1068236800 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.00031182617741630416, |
|
"loss": 2.6897, |
|
"theoretical_loss": 3.625432477103554, |
|
"tokens_seen": 1071513600 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0003112351049744657, |
|
"loss": 2.6915, |
|
"theoretical_loss": 3.6243928280195976, |
|
"tokens_seen": 1074790400 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0003106440325326272, |
|
"loss": 2.6801, |
|
"theoretical_loss": 3.62335722820693, |
|
"tokens_seen": 1078067200 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0003100529600907888, |
|
"loss": 2.7026, |
|
"theoretical_loss": 3.6223256496581637, |
|
"tokens_seen": 1081344000 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0003094618876489503, |
|
"loss": 2.7034, |
|
"theoretical_loss": 3.6212980646437485, |
|
"tokens_seen": 1084620800 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0003088708152071118, |
|
"loss": 2.6787, |
|
"theoretical_loss": 3.6202744457083877, |
|
"tokens_seen": 1087897600 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.00030827974276527334, |
|
"loss": 2.7251, |
|
"theoretical_loss": 3.6192547656675083, |
|
"tokens_seen": 1091174400 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.00030768867032343486, |
|
"loss": 2.708, |
|
"theoretical_loss": 3.618238997603788, |
|
"tokens_seen": 1094451200 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0003070975978815964, |
|
"loss": 2.6985, |
|
"theoretical_loss": 3.617227114863738, |
|
"tokens_seen": 1097728000 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.00030651834688859465, |
|
"loss": 2.7094, |
|
"theoretical_loss": 3.6162190910543366, |
|
"tokens_seen": 1101004800 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.00030592727444675623, |
|
"loss": 2.6993, |
|
"theoretical_loss": 3.615214900039721, |
|
"tokens_seen": 1104281600 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00030533620200491775, |
|
"loss": 2.6727, |
|
"theoretical_loss": 3.614214515937924, |
|
"tokens_seen": 1107558400 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00030474512956307927, |
|
"loss": 2.6647, |
|
"theoretical_loss": 3.613217913117667, |
|
"tokens_seen": 1110835200 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0003041540571212408, |
|
"loss": 2.7036, |
|
"theoretical_loss": 3.612225066195201, |
|
"tokens_seen": 1114112000 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0003035629846794023, |
|
"loss": 2.6909, |
|
"theoretical_loss": 3.611235950031194, |
|
"tokens_seen": 1117388800 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00030297191223756383, |
|
"loss": 2.6698, |
|
"theoretical_loss": 3.6102505397276743, |
|
"tokens_seen": 1120665600 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0003023808397957254, |
|
"loss": 2.6833, |
|
"theoretical_loss": 3.60926881062501, |
|
"tokens_seen": 1123942400 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00030178976735388693, |
|
"loss": 2.6548, |
|
"theoretical_loss": 3.608290738298942, |
|
"tokens_seen": 1127219200 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00030119869491204845, |
|
"loss": 2.6415, |
|
"theoretical_loss": 3.6073162985576643, |
|
"tokens_seen": 1130496000 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00030060762247020997, |
|
"loss": 2.6656, |
|
"theoretical_loss": 3.606345467438941, |
|
"tokens_seen": 1133772800 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.0003000165500283715, |
|
"loss": 2.6773, |
|
"theoretical_loss": 3.6053782212072747, |
|
"tokens_seen": 1137049600 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.000299425477586533, |
|
"loss": 2.6801, |
|
"theoretical_loss": 3.604414536351113, |
|
"tokens_seen": 1140326400 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.0002988344051446946, |
|
"loss": 2.6656, |
|
"theoretical_loss": 3.6034543895801017, |
|
"tokens_seen": 1143603200 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"objective/train/docs_used": 634175, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.45387601852417, |
|
"objective/train/theoretical_loss": 3.6024977578223742, |
|
"objective/train/tokens_used": 1167340000, |
|
"theoretical_loss": 3.6024977578223742, |
|
"tokens_seen": 1146880000 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.0002982433327028561, |
|
"loss": 2.6558, |
|
"theoretical_loss": 3.6024977578223742, |
|
"tokens_seen": 1146880000 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00029765226026101763, |
|
"loss": 2.7031, |
|
"theoretical_loss": 3.6015446182218875, |
|
"tokens_seen": 1150156800 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00029706118781917915, |
|
"loss": 2.7284, |
|
"theoretical_loss": 3.600594948135793, |
|
"tokens_seen": 1153433600 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00029647011537734067, |
|
"loss": 2.7, |
|
"theoretical_loss": 3.59964872513185, |
|
"tokens_seen": 1156710400 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00029587904293550214, |
|
"loss": 2.7097, |
|
"theoretical_loss": 3.5987059269858763, |
|
"tokens_seen": 1159987200 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00029528797049366366, |
|
"loss": 2.6929, |
|
"theoretical_loss": 3.5977665316792375, |
|
"tokens_seen": 1163264000 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00029469689805182523, |
|
"loss": 2.6813, |
|
"theoretical_loss": 3.5968305173963744, |
|
"tokens_seen": 1166540800 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00029410582560998675, |
|
"loss": 2.6756, |
|
"theoretical_loss": 3.5958978625223628, |
|
"tokens_seen": 1169817600 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0002935147531681483, |
|
"loss": 2.6742, |
|
"theoretical_loss": 3.5949685456405165, |
|
"tokens_seen": 1173094400 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0002929236807263098, |
|
"loss": 2.7111, |
|
"theoretical_loss": 3.5940425455300176, |
|
"tokens_seen": 1176371200 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0002923326082844713, |
|
"loss": 2.683, |
|
"theoretical_loss": 3.593119841163589, |
|
"tokens_seen": 1179648000 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00029174153584263284, |
|
"loss": 2.6908, |
|
"theoretical_loss": 3.5922004117051944, |
|
"tokens_seen": 1182924800 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0002911504634007944, |
|
"loss": 2.6789, |
|
"theoretical_loss": 3.5912842365077777, |
|
"tokens_seen": 1186201600 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00029055939095895593, |
|
"loss": 2.6923, |
|
"theoretical_loss": 3.5903712951110305, |
|
"tokens_seen": 1189478400 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.00028996831851711745, |
|
"loss": 2.6713, |
|
"theoretical_loss": 3.5894615672391947, |
|
"tokens_seen": 1192755200 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.000289377246075279, |
|
"loss": 2.692, |
|
"theoretical_loss": 3.5885550327988973, |
|
"tokens_seen": 1196032000 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.0002887861736334405, |
|
"loss": 2.6788, |
|
"theoretical_loss": 3.587651671877014, |
|
"tokens_seen": 1199308800 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.000288195101191602, |
|
"loss": 2.691, |
|
"theoretical_loss": 3.5867514647385663, |
|
"tokens_seen": 1202585600 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.0002876040287497636, |
|
"loss": 2.7057, |
|
"theoretical_loss": 3.585854391824647, |
|
"tokens_seen": 1205862400 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.0002870129563079251, |
|
"loss": 2.6776, |
|
"theoretical_loss": 3.584960433750375, |
|
"tokens_seen": 1209139200 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.00028642188386608663, |
|
"loss": 2.7006, |
|
"theoretical_loss": 3.5840695713028827, |
|
"tokens_seen": 1212416000 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.00028583081142424815, |
|
"loss": 2.6434, |
|
"theoretical_loss": 3.5831817854393266, |
|
"tokens_seen": 1215692800 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00028523973898240967, |
|
"loss": 2.6768, |
|
"theoretical_loss": 3.582297057284933, |
|
"tokens_seen": 1218969600 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0002846486665405712, |
|
"loss": 2.6751, |
|
"theoretical_loss": 3.5814153681310623, |
|
"tokens_seen": 1222246400 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0002840575940987327, |
|
"loss": 2.7175, |
|
"theoretical_loss": 3.5805366994333125, |
|
"tokens_seen": 1225523200 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0002834665216568943, |
|
"loss": 2.6869, |
|
"theoretical_loss": 3.5796610328096365, |
|
"tokens_seen": 1228800000 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0002828754492150558, |
|
"loss": 2.7104, |
|
"theoretical_loss": 3.578788350038497, |
|
"tokens_seen": 1232076800 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00028228437677321733, |
|
"loss": 2.6871, |
|
"theoretical_loss": 3.5779186330570405, |
|
"tokens_seen": 1235353600 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00028169330433137885, |
|
"loss": 2.6961, |
|
"theoretical_loss": 3.5770518639592983, |
|
"tokens_seen": 1238630400 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00028110223188954037, |
|
"loss": 2.6979, |
|
"theoretical_loss": 3.5761880249944147, |
|
"tokens_seen": 1241907200 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0002805111594477019, |
|
"loss": 2.7091, |
|
"theoretical_loss": 3.5753270985648973, |
|
"tokens_seen": 1245184000 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00027992008700586347, |
|
"loss": 2.7407, |
|
"theoretical_loss": 3.574469067224892, |
|
"tokens_seen": 1248460800 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.000279329014564025, |
|
"loss": 2.7418, |
|
"theoretical_loss": 3.573613913678484, |
|
"tokens_seen": 1251737600 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.0002787379421221865, |
|
"loss": 2.7307, |
|
"theoretical_loss": 3.57276162077802, |
|
"tokens_seen": 1255014400 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00027814686968034803, |
|
"loss": 2.6796, |
|
"theoretical_loss": 3.5719121715224524, |
|
"tokens_seen": 1258291200 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00027755579723850955, |
|
"loss": 2.685, |
|
"theoretical_loss": 3.571065549055712, |
|
"tokens_seen": 1261568000 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00027696472479667107, |
|
"loss": 2.6801, |
|
"theoretical_loss": 3.5702217366650935, |
|
"tokens_seen": 1264844800 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00027637365235483265, |
|
"loss": 2.6468, |
|
"theoretical_loss": 3.5693807177796737, |
|
"tokens_seen": 1268121600 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00027578257991299417, |
|
"loss": 2.6318, |
|
"theoretical_loss": 3.5685424759687434, |
|
"tokens_seen": 1271398400 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0002751915074711557, |
|
"loss": 2.6674, |
|
"theoretical_loss": 3.567706994940263, |
|
"tokens_seen": 1274675200 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0002746004350293172, |
|
"loss": 2.7201, |
|
"theoretical_loss": 3.5668742585393405, |
|
"tokens_seen": 1277952000 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00027400936258747873, |
|
"loss": 2.6202, |
|
"theoretical_loss": 3.566044250746728, |
|
"tokens_seen": 1281228800 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00027341829014564025, |
|
"loss": 2.6826, |
|
"theoretical_loss": 3.5652169556773403, |
|
"tokens_seen": 1284505600 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0002728272177038018, |
|
"loss": 2.598, |
|
"theoretical_loss": 3.5643923575787912, |
|
"tokens_seen": 1287782400 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00027223614526196335, |
|
"loss": 2.654, |
|
"theoretical_loss": 3.563570440829951, |
|
"tokens_seen": 1291059200 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00027164507282012487, |
|
"loss": 2.6348, |
|
"theoretical_loss": 3.562751189939524, |
|
"tokens_seen": 1294336000 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0002710540003782864, |
|
"loss": 2.6582, |
|
"theoretical_loss": 3.5619345895446424, |
|
"tokens_seen": 1297612800 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0002704629279364479, |
|
"loss": 2.6563, |
|
"theoretical_loss": 3.561120624409482, |
|
"tokens_seen": 1300889600 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00026987185549460943, |
|
"loss": 2.6318, |
|
"theoretical_loss": 3.560309279423894, |
|
"tokens_seen": 1304166400 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00026928078305277095, |
|
"loss": 2.6117, |
|
"theoretical_loss": 3.5595005396020554, |
|
"tokens_seen": 1307443200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.5611916397279219, |
|
"debugging/distinct-1-grams": 0.7925379500087467, |
|
"debugging/distinct-2-grams": 0.9770296872052382, |
|
"debugging/entropy-1-grams": 6.151978161681489, |
|
"debugging/entropy-2-grams": 7.198376973101845, |
|
"debugging/length": 525.15, |
|
"debugging/num_segments": 20, |
|
"epoch": 0.47, |
|
"objective/train/docs_used": 722407, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.8044486045837402, |
|
"objective/train/theoretical_loss": 3.558694390081137, |
|
"objective/train/tokens_used": 1331180000, |
|
"theoretical_loss": 3.558694390081137, |
|
"tokens_seen": 1310720000 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.0002686897106109325, |
|
"loss": 2.621, |
|
"theoretical_loss": 3.558694390081137, |
|
"tokens_seen": 1310720000 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00026809863816909405, |
|
"loss": 2.6119, |
|
"theoretical_loss": 3.5578908161199934, |
|
"tokens_seen": 1313996800 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00026750756572725557, |
|
"loss": 2.5984, |
|
"theoretical_loss": 3.5570898030978584, |
|
"tokens_seen": 1317273600 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.0002669164932854171, |
|
"loss": 2.618, |
|
"theoretical_loss": 3.556291336513074, |
|
"tokens_seen": 1320550400 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00026632542084357855, |
|
"loss": 2.6012, |
|
"theoretical_loss": 3.5554954019818235, |
|
"tokens_seen": 1323827200 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.0002657343484017401, |
|
"loss": 2.6034, |
|
"theoretical_loss": 3.554701985236883, |
|
"tokens_seen": 1327104000 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026514327595990165, |
|
"loss": 2.6416, |
|
"theoretical_loss": 3.553911072126394, |
|
"tokens_seen": 1330380800 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026455220351806317, |
|
"loss": 2.6386, |
|
"theoretical_loss": 3.5531226486126504, |
|
"tokens_seen": 1333657600 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.0002639611310762247, |
|
"loss": 2.6568, |
|
"theoretical_loss": 3.552336700770896, |
|
"tokens_seen": 1336934400 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.0002633700586343862, |
|
"loss": 2.6236, |
|
"theoretical_loss": 3.5515532147881443, |
|
"tokens_seen": 1340211200 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026277898619254773, |
|
"loss": 2.6408, |
|
"theoretical_loss": 3.5507721769620098, |
|
"tokens_seen": 1343488000 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026218791375070925, |
|
"loss": 2.6688, |
|
"theoretical_loss": 3.549993573699556, |
|
"tokens_seen": 1346764800 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026159684130887083, |
|
"loss": 2.6196, |
|
"theoretical_loss": 3.5492173915161565, |
|
"tokens_seen": 1350041600 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026100576886703235, |
|
"loss": 2.6396, |
|
"theoretical_loss": 3.548443617034371, |
|
"tokens_seen": 1353318400 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026041469642519387, |
|
"loss": 2.6577, |
|
"theoretical_loss": 3.547672236982839, |
|
"tokens_seen": 1356595200 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.0002598236239833554, |
|
"loss": 2.6491, |
|
"theoretical_loss": 3.5469032381951804, |
|
"tokens_seen": 1359872000 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.0002592325515415169, |
|
"loss": 2.6643, |
|
"theoretical_loss": 3.5461366076089202, |
|
"tokens_seen": 1363148800 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00025864147909967843, |
|
"loss": 2.648, |
|
"theoretical_loss": 3.5453723322644146, |
|
"tokens_seen": 1366425600 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00025805040665783995, |
|
"loss": 2.6345, |
|
"theoretical_loss": 3.544610399303803, |
|
"tokens_seen": 1369702400 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00025745933421600153, |
|
"loss": 2.6438, |
|
"theoretical_loss": 3.5438507959699637, |
|
"tokens_seen": 1372979200 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00025686826177416305, |
|
"loss": 2.6408, |
|
"theoretical_loss": 3.5430935096054883, |
|
"tokens_seen": 1376256000 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00025627718933232457, |
|
"loss": 2.6286, |
|
"theoretical_loss": 3.5423385276516663, |
|
"tokens_seen": 1379532800 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.0002556861168904861, |
|
"loss": 2.6113, |
|
"theoretical_loss": 3.5415858376474825, |
|
"tokens_seen": 1382809600 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.0002550950444486476, |
|
"loss": 2.6155, |
|
"theoretical_loss": 3.5408354272286298, |
|
"tokens_seen": 1386086400 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.00025450397200680913, |
|
"loss": 2.6191, |
|
"theoretical_loss": 3.540087284126531, |
|
"tokens_seen": 1389363200 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.0002539128995649707, |
|
"loss": 2.6304, |
|
"theoretical_loss": 3.539341396167372, |
|
"tokens_seen": 1392640000 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.00025332182712313223, |
|
"loss": 2.654, |
|
"theoretical_loss": 3.538597751271153, |
|
"tokens_seen": 1395916800 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.00025273075468129375, |
|
"loss": 2.6814, |
|
"theoretical_loss": 3.5378563374507443, |
|
"tokens_seen": 1399193600 |
|
} |
|
], |
|
"max_steps": 42724, |
|
"num_train_epochs": 9223372036854775807, |
|
"total_flos": 7.14460209610752e+17, |
|
"trial_name": null, |
|
"trial_params": null |
|
} |
|
|