|
{ |
|
"best_metric": null, |
|
"best_model_checkpoint": null, |
|
"epoch": 1.0, |
|
"global_step": 42724, |
|
"is_hyper_param_search": false, |
|
"is_local_process_zero": true, |
|
"is_world_process_zero": true, |
|
"log_history": [ |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 1.1682242990654204e-06, |
|
"loss": 10.8108, |
|
"theoretical_loss": 20.812814784551147, |
|
"tokens_seen": 65536 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 5.841121495327103e-05, |
|
"loss": 8.8846, |
|
"theoretical_loss": 8.563479647615063, |
|
"tokens_seen": 3276800 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 0.00011682242990654206, |
|
"loss": 6.7892, |
|
"theoretical_loss": 7.4777557010520255, |
|
"tokens_seen": 6553600 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 0.00017523364485981307, |
|
"loss": 5.8537, |
|
"theoretical_loss": 6.933751471898896, |
|
"tokens_seen": 9830400 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 0.00023364485981308412, |
|
"loss": 5.4837, |
|
"theoretical_loss": 6.583563211430409, |
|
"tokens_seen": 13107200 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.00029205607476635517, |
|
"loss": 5.2188, |
|
"theoretical_loss": 6.330710548120079, |
|
"tokens_seen": 16384000 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.00035046728971962614, |
|
"loss": 5.0585, |
|
"theoretical_loss": 6.135526214944321, |
|
"tokens_seen": 19660800 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0004088785046728972, |
|
"loss": 4.8994, |
|
"theoretical_loss": 5.978098566873603, |
|
"tokens_seen": 22937600 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.00046728971962616824, |
|
"loss": 4.7997, |
|
"theoretical_loss": 5.847114309269919, |
|
"tokens_seen": 26214400 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.000499739928125591, |
|
"loss": 4.699, |
|
"theoretical_loss": 5.73557379888612, |
|
"tokens_seen": 29491200 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0004991488556837526, |
|
"loss": 4.5763, |
|
"theoretical_loss": 5.638867127075349, |
|
"tokens_seen": 32768000 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0004985577832419141, |
|
"loss": 4.4151, |
|
"theoretical_loss": 5.553809364848902, |
|
"tokens_seen": 36044800 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0004979667108000757, |
|
"loss": 4.3352, |
|
"theoretical_loss": 5.478115063560433, |
|
"tokens_seen": 39321600 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004973756383582371, |
|
"loss": 4.2843, |
|
"theoretical_loss": 5.410092942583357, |
|
"tokens_seen": 42598400 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004967845659163987, |
|
"loss": 4.1876, |
|
"theoretical_loss": 5.34845906673983, |
|
"tokens_seen": 45875200 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004961934934745603, |
|
"loss": 4.121, |
|
"theoretical_loss": 5.292217549941562, |
|
"tokens_seen": 49152000 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004956024210327218, |
|
"loss": 4.0357, |
|
"theoretical_loss": 5.240581608773973, |
|
"tokens_seen": 52428800 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004950113485908833, |
|
"loss": 3.9509, |
|
"theoretical_loss": 5.192919707529784, |
|
"tokens_seen": 55705600 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004944202761490448, |
|
"loss": 3.9396, |
|
"theoretical_loss": 5.1487178463604355, |
|
"tokens_seen": 58982400 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004938292037072064, |
|
"loss": 3.8453, |
|
"theoretical_loss": 5.107552545409097, |
|
"tokens_seen": 62259200 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004932381312653678, |
|
"loss": 3.7222, |
|
"theoretical_loss": 5.069071100147241, |
|
"tokens_seen": 65536000 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004926470588235294, |
|
"loss": 3.6644, |
|
"theoretical_loss": 5.032976892842002, |
|
"tokens_seen": 68812800 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.000492055986381691, |
|
"loss": 3.5734, |
|
"theoretical_loss": 4.999018291228659, |
|
"tokens_seen": 72089600 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004914649139398525, |
|
"loss": 3.5035, |
|
"theoretical_loss": 4.966980138355957, |
|
"tokens_seen": 75366400 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.000490873841498014, |
|
"loss": 3.4903, |
|
"theoretical_loss": 4.936677144629131, |
|
"tokens_seen": 78643200 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004902827690561755, |
|
"loss": 3.4628, |
|
"theoretical_loss": 4.907948696834077, |
|
"tokens_seen": 81920000 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004896916966143371, |
|
"loss": 3.4025, |
|
"theoretical_loss": 4.880654736816922, |
|
"tokens_seen": 85196800 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004891006241724985, |
|
"loss": 3.3946, |
|
"theoretical_loss": 4.8546724574857745, |
|
"tokens_seen": 88473600 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004885095517306601, |
|
"loss": 3.3608, |
|
"theoretical_loss": 4.829893630312808, |
|
"tokens_seen": 91750400 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004879184792888217, |
|
"loss": 3.3419, |
|
"theoretical_loss": 4.806222425781916, |
|
"tokens_seen": 95027200 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0004873274068469832, |
|
"loss": 3.2802, |
|
"theoretical_loss": 4.783573622280253, |
|
"tokens_seen": 98304000 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.00048673633440514467, |
|
"loss": 3.2453, |
|
"theoretical_loss": 4.7618711237764035, |
|
"tokens_seen": 101580800 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0004861452619633062, |
|
"loss": 3.3002, |
|
"theoretical_loss": 4.741046724966468, |
|
"tokens_seen": 104857600 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0004855541895214677, |
|
"loss": 3.2674, |
|
"theoretical_loss": 4.721039076253046, |
|
"tokens_seen": 108134400 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0004849631170796293, |
|
"loss": 3.201, |
|
"theoretical_loss": 4.701792811235861, |
|
"tokens_seen": 111411200 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0004843720446377908, |
|
"loss": 3.2566, |
|
"theoretical_loss": 4.683257807239925, |
|
"tokens_seen": 114688000 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.00048378097219595233, |
|
"loss": 3.2271, |
|
"theoretical_loss": 4.665388555430277, |
|
"tokens_seen": 117964800 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.00048318989975411385, |
|
"loss": 3.2303, |
|
"theoretical_loss": 4.648143621723734, |
|
"tokens_seen": 121241600 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.00048259882731227537, |
|
"loss": 3.2211, |
|
"theoretical_loss": 4.631485183343639, |
|
"tokens_seen": 124518400 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0004820077548704369, |
|
"loss": 3.1683, |
|
"theoretical_loss": 4.6153786287197125, |
|
"tokens_seen": 127795200 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.00048141668242859847, |
|
"loss": 3.1829, |
|
"theoretical_loss": 4.5997922106945, |
|
"tokens_seen": 131072000 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.00048082560998676, |
|
"loss": 3.1919, |
|
"theoretical_loss": 4.58469674479667, |
|
"tokens_seen": 134348800 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0004802345375449215, |
|
"loss": 3.2002, |
|
"theoretical_loss": 4.570065345782512, |
|
"tokens_seen": 137625600 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.00047964346510308303, |
|
"loss": 3.1367, |
|
"theoretical_loss": 4.555873196808033, |
|
"tokens_seen": 140902400 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.00047905239266124455, |
|
"loss": 3.1157, |
|
"theoretical_loss": 4.542097346534794, |
|
"tokens_seen": 144179200 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.00047846132021940607, |
|
"loss": 3.1215, |
|
"theoretical_loss": 4.528716530238812, |
|
"tokens_seen": 147456000 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0004778702477775676, |
|
"loss": 3.092, |
|
"theoretical_loss": 4.515711011618992, |
|
"tokens_seen": 150732800 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.00047727917533572917, |
|
"loss": 3.0751, |
|
"theoretical_loss": 4.503062442517334, |
|
"tokens_seen": 154009600 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0004766881028938907, |
|
"loss": 3.1182, |
|
"theoretical_loss": 4.4907537381892615, |
|
"tokens_seen": 157286400 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0004760970304520522, |
|
"loss": 3.078, |
|
"theoretical_loss": 4.478768966115963, |
|
"tokens_seen": 160563200 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"objective/train/docs_used": 96518, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 3.2037177085876465, |
|
"objective/train/theoretical_loss": 4.467093246645215, |
|
"objective/train/tokens_used": 184300000, |
|
"theoretical_loss": 4.467093246645215, |
|
"tokens_seen": 163840000 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.00047550595801021373, |
|
"loss": 3.0499, |
|
"theoretical_loss": 4.467093246645215, |
|
"tokens_seen": 163840000 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.00047491488556837525, |
|
"loss": 3.0773, |
|
"theoretical_loss": 4.455712663993541, |
|
"tokens_seen": 167116800 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.00047432381312653677, |
|
"loss": 3.0735, |
|
"theoretical_loss": 4.444614186349425, |
|
"tokens_seen": 170393600 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.00047373274068469835, |
|
"loss": 3.02, |
|
"theoretical_loss": 4.433785593991642, |
|
"tokens_seen": 173670400 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.00047314166824285987, |
|
"loss": 3.0629, |
|
"theoretical_loss": 4.423215414484155, |
|
"tokens_seen": 176947200 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0004725505958010214, |
|
"loss": 3.0337, |
|
"theoretical_loss": 4.412892864134137, |
|
"tokens_seen": 180224000 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0004719595233591829, |
|
"loss": 3.0301, |
|
"theoretical_loss": 4.402807795006074, |
|
"tokens_seen": 183500800 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.00047136845091734443, |
|
"loss": 2.9966, |
|
"theoretical_loss": 4.392950646875857, |
|
"tokens_seen": 186777600 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.00047077737847550595, |
|
"loss": 3.0129, |
|
"theoretical_loss": 4.383312403586528, |
|
"tokens_seen": 190054400 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0004701863060336675, |
|
"loss": 2.9981, |
|
"theoretical_loss": 4.373884553334271, |
|
"tokens_seen": 193331200 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.00046959523359182905, |
|
"loss": 3.0304, |
|
"theoretical_loss": 4.364659052470699, |
|
"tokens_seen": 196608000 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.00046900416114999057, |
|
"loss": 2.9986, |
|
"theoretical_loss": 4.355628292457278, |
|
"tokens_seen": 199884800 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0004684130887081521, |
|
"loss": 2.9998, |
|
"theoretical_loss": 4.346785069650667, |
|
"tokens_seen": 203161600 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0004678220162663136, |
|
"loss": 2.9699, |
|
"theoretical_loss": 4.338122557635156, |
|
"tokens_seen": 206438400 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.00046723094382447513, |
|
"loss": 3.0122, |
|
"theoretical_loss": 4.329634281850807, |
|
"tokens_seen": 209715200 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.00046663987138263665, |
|
"loss": 2.9991, |
|
"theoretical_loss": 4.321314096294248, |
|
"tokens_seen": 212992000 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0004660487989407982, |
|
"loss": 2.9779, |
|
"theoretical_loss": 4.3131561620937875, |
|
"tokens_seen": 216268800 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.00046545772649895975, |
|
"loss": 2.9469, |
|
"theoretical_loss": 4.305154927782223, |
|
"tokens_seen": 219545600 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.00046486665405712127, |
|
"loss": 2.9714, |
|
"theoretical_loss": 4.297305111109683, |
|
"tokens_seen": 222822400 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0004642755816152828, |
|
"loss": 2.971, |
|
"theoretical_loss": 4.2896016822555945, |
|
"tokens_seen": 226099200 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0004636845091734443, |
|
"loss": 2.9227, |
|
"theoretical_loss": 4.282039848313611, |
|
"tokens_seen": 229376000 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.00046309343673160583, |
|
"loss": 2.9236, |
|
"theoretical_loss": 4.274615038936293, |
|
"tokens_seen": 232652800 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0004625023642897674, |
|
"loss": 2.9169, |
|
"theoretical_loss": 4.267322893037893, |
|
"tokens_seen": 235929600 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004619112918479289, |
|
"loss": 2.93, |
|
"theoretical_loss": 4.26015924646374, |
|
"tokens_seen": 239206400 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.00046132021940609044, |
|
"loss": 2.9526, |
|
"theoretical_loss": 4.253120120543809, |
|
"tokens_seen": 242483200 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.00046072914696425197, |
|
"loss": 2.9707, |
|
"theoretical_loss": 4.24620171145605, |
|
"tokens_seen": 245760000 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004601380745224135, |
|
"loss": 2.9652, |
|
"theoretical_loss": 4.239400380332256, |
|
"tokens_seen": 249036800 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.000459547002080575, |
|
"loss": 2.9379, |
|
"theoretical_loss": 4.232712644045627, |
|
"tokens_seen": 252313600 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004589559296387366, |
|
"loss": 2.9114, |
|
"theoretical_loss": 4.226135166624862, |
|
"tokens_seen": 255590400 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004583648571968981, |
|
"loss": 2.929, |
|
"theoretical_loss": 4.21966475124477, |
|
"tokens_seen": 258867200 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004577737847550596, |
|
"loss": 2.9192, |
|
"theoretical_loss": 4.21329833274792, |
|
"tokens_seen": 262144000 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004571827123132211, |
|
"loss": 2.8887, |
|
"theoretical_loss": 4.207032970655965, |
|
"tokens_seen": 265420800 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0004565916398713826, |
|
"loss": 2.8974, |
|
"theoretical_loss": 4.2008658426329974, |
|
"tokens_seen": 268697600 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.00045600056742954413, |
|
"loss": 2.8986, |
|
"theoretical_loss": 4.194794238366544, |
|
"tokens_seen": 271974400 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0004554094949877057, |
|
"loss": 2.8746, |
|
"theoretical_loss": 4.188815553834879, |
|
"tokens_seen": 275251200 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.00045481842254586723, |
|
"loss": 2.8823, |
|
"theoretical_loss": 4.182927285931959, |
|
"tokens_seen": 278528000 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.00045422735010402875, |
|
"loss": 2.8312, |
|
"theoretical_loss": 4.177127027423776, |
|
"tokens_seen": 281804800 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.00045363627766219027, |
|
"loss": 2.839, |
|
"theoretical_loss": 4.171412462212087, |
|
"tokens_seen": 285081600 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0004530452052203518, |
|
"loss": 2.9863, |
|
"theoretical_loss": 4.165781360883513, |
|
"tokens_seen": 288358400 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0004524541327785133, |
|
"loss": 2.8758, |
|
"theoretical_loss": 4.160231576523763, |
|
"tokens_seen": 291635200 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.00045186306033667483, |
|
"loss": 2.8114, |
|
"theoretical_loss": 4.1547610407784346, |
|
"tokens_seen": 294912000 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0004512719878948364, |
|
"loss": 2.8251, |
|
"theoretical_loss": 4.1493677601432815, |
|
"tokens_seen": 298188800 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.00045068091545299793, |
|
"loss": 2.8408, |
|
"theoretical_loss": 4.1440498124682446, |
|
"tokens_seen": 301465600 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.00045008984301115945, |
|
"loss": 2.8312, |
|
"theoretical_loss": 4.138805343660737, |
|
"tokens_seen": 304742400 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.00044949877056932097, |
|
"loss": 2.8875, |
|
"theoretical_loss": 4.133632564574831, |
|
"tokens_seen": 308019200 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0004489076981274825, |
|
"loss": 2.8759, |
|
"theoretical_loss": 4.128529748073999, |
|
"tokens_seen": 311296000 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.000448316625685644, |
|
"loss": 2.8609, |
|
"theoretical_loss": 4.123495226256027, |
|
"tokens_seen": 314572800 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0004477255532438056, |
|
"loss": 2.8392, |
|
"theoretical_loss": 4.118527387829552, |
|
"tokens_seen": 317849600 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0004471344808019671, |
|
"loss": 2.8094, |
|
"theoretical_loss": 4.11362467563246, |
|
"tokens_seen": 321126400 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.00044654340836012863, |
|
"loss": 2.8147, |
|
"theoretical_loss": 4.108785584283144, |
|
"tokens_seen": 324403200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.5473251668236196, |
|
"debugging/distinct-1-grams": 0.778067561286293, |
|
"debugging/distinct-2-grams": 0.9667833105719122, |
|
"debugging/entropy-1-grams": 5.872420866822687, |
|
"debugging/entropy-2-grams": 6.645619306539771, |
|
"debugging/length": 629.4166666666666, |
|
"debugging/num_segments": 12, |
|
"epoch": 0.12, |
|
"objective/train/docs_used": 182434, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.7209677696228027, |
|
"objective/train/theoretical_loss": 4.104008657956216, |
|
"objective/train/tokens_used": 348140000, |
|
"theoretical_loss": 4.104008657956216, |
|
"tokens_seen": 327680000 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.00044595233591829015, |
|
"loss": 2.808, |
|
"theoretical_loss": 4.104008657956216, |
|
"tokens_seen": 327680000 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.00044536126347645167, |
|
"loss": 2.8024, |
|
"theoretical_loss": 4.099292488274917, |
|
"tokens_seen": 330956800 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0004447701910346132, |
|
"loss": 2.8006, |
|
"theoretical_loss": 4.094635712313026, |
|
"tokens_seen": 334233600 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.00044417911859277476, |
|
"loss": 2.7996, |
|
"theoretical_loss": 4.090037010699552, |
|
"tokens_seen": 337510400 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0004435880461509363, |
|
"loss": 2.7717, |
|
"theoretical_loss": 4.085495105819987, |
|
"tokens_seen": 340787200 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0004429969737090978, |
|
"loss": 2.7623, |
|
"theoretical_loss": 4.081008760108329, |
|
"tokens_seen": 344064000 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0004424059012672593, |
|
"loss": 2.7872, |
|
"theoretical_loss": 4.076576774424465, |
|
"tokens_seen": 347340800 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.00044181482882542085, |
|
"loss": 2.8093, |
|
"theoretical_loss": 4.072197986511911, |
|
"tokens_seen": 350617600 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.00044122375638358237, |
|
"loss": 2.8166, |
|
"theoretical_loss": 4.067871269531193, |
|
"tokens_seen": 353894400 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.0004406326839417439, |
|
"loss": 2.7933, |
|
"theoretical_loss": 4.063595530664515, |
|
"tokens_seen": 357171200 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.00044004161149990546, |
|
"loss": 2.799, |
|
"theoretical_loss": 4.059369709787625, |
|
"tokens_seen": 360448000 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.000439450539058067, |
|
"loss": 2.7816, |
|
"theoretical_loss": 4.055192778205064, |
|
"tokens_seen": 363724800 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.0004388594666162285, |
|
"loss": 2.7722, |
|
"theoretical_loss": 4.051063737445231, |
|
"tokens_seen": 367001600 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.00043826839417439, |
|
"loss": 2.7627, |
|
"theoretical_loss": 4.046981618111936, |
|
"tokens_seen": 370278400 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.00043767732173255155, |
|
"loss": 2.7495, |
|
"theoretical_loss": 4.042945478789308, |
|
"tokens_seen": 373555200 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.00043708624929071307, |
|
"loss": 2.7597, |
|
"theoretical_loss": 4.038954404997148, |
|
"tokens_seen": 376832000 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.00043649517684887464, |
|
"loss": 2.7771, |
|
"theoretical_loss": 4.035007508193978, |
|
"tokens_seen": 380108800 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.00043590410440703616, |
|
"loss": 2.7474, |
|
"theoretical_loss": 4.031103924825214, |
|
"tokens_seen": 383385600 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0004353130319651977, |
|
"loss": 2.8124, |
|
"theoretical_loss": 4.027242815414074, |
|
"tokens_seen": 386662400 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0004347219595233592, |
|
"loss": 2.7814, |
|
"theoretical_loss": 4.023423363692939, |
|
"tokens_seen": 389939200 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0004341308870815207, |
|
"loss": 2.7909, |
|
"theoretical_loss": 4.019644775773043, |
|
"tokens_seen": 393216000 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.00043353981463968225, |
|
"loss": 2.7537, |
|
"theoretical_loss": 4.015906279350517, |
|
"tokens_seen": 396492800 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0004329487421978438, |
|
"loss": 2.7388, |
|
"theoretical_loss": 4.01220712294689, |
|
"tokens_seen": 399769600 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.00043235766975600534, |
|
"loss": 2.76, |
|
"theoretical_loss": 4.008546575182286, |
|
"tokens_seen": 403046400 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.00043176659731416686, |
|
"loss": 2.7373, |
|
"theoretical_loss": 4.0049239240796695, |
|
"tokens_seen": 406323200 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0004311755248723284, |
|
"loss": 2.705, |
|
"theoretical_loss": 4.001338476398553, |
|
"tokens_seen": 409600000 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0004305844524304899, |
|
"loss": 2.7386, |
|
"theoretical_loss": 3.9977895569967217, |
|
"tokens_seen": 412876800 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0004299933799886514, |
|
"loss": 2.7393, |
|
"theoretical_loss": 3.99427650821855, |
|
"tokens_seen": 416153600 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.00042940230754681295, |
|
"loss": 2.718, |
|
"theoretical_loss": 3.990798689308642, |
|
"tokens_seen": 419430400 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0004288112351049745, |
|
"loss": 2.745, |
|
"theoretical_loss": 3.987355475849527, |
|
"tokens_seen": 422707200 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.000428220162663136, |
|
"loss": 2.7552, |
|
"theoretical_loss": 3.98394625922226, |
|
"tokens_seen": 425984000 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0004276290902212975, |
|
"loss": 2.7915, |
|
"theoretical_loss": 3.9805704460888256, |
|
"tokens_seen": 429260800 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.00042703801777945903, |
|
"loss": 2.7639, |
|
"theoretical_loss": 3.9772274578953013, |
|
"tokens_seen": 432537600 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.00042644694533762055, |
|
"loss": 2.7386, |
|
"theoretical_loss": 3.973916730394796, |
|
"tokens_seen": 435814400 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.00042585587289578207, |
|
"loss": 2.7708, |
|
"theoretical_loss": 3.970637713189244, |
|
"tokens_seen": 439091200 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.00042526480045394365, |
|
"loss": 2.793, |
|
"theoretical_loss": 3.967389869289161, |
|
"tokens_seen": 442368000 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.00042467372801210517, |
|
"loss": 2.7812, |
|
"theoretical_loss": 3.964172674690542, |
|
"tokens_seen": 445644800 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.0004240826555702667, |
|
"loss": 2.7581, |
|
"theoretical_loss": 3.9609856179681078, |
|
"tokens_seen": 448921600 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.0004234915831284282, |
|
"loss": 2.8062, |
|
"theoretical_loss": 3.957828199884155, |
|
"tokens_seen": 452198400 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.00042290051068658973, |
|
"loss": 2.7805, |
|
"theoretical_loss": 3.9546999330123036, |
|
"tokens_seen": 455475200 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.00042230943824475125, |
|
"loss": 2.7614, |
|
"theoretical_loss": 3.951600341375469, |
|
"tokens_seen": 458752000 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0004217183658029128, |
|
"loss": 2.751, |
|
"theoretical_loss": 3.9485289600974305, |
|
"tokens_seen": 462028800 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.00042112729336107435, |
|
"loss": 2.763, |
|
"theoretical_loss": 3.945485335067386, |
|
"tokens_seen": 465305600 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.00042053622091923587, |
|
"loss": 2.7535, |
|
"theoretical_loss": 3.9424690226169212, |
|
"tokens_seen": 468582400 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0004199451484773974, |
|
"loss": 2.7463, |
|
"theoretical_loss": 3.9394795892088585, |
|
"tokens_seen": 471859200 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0004193540760355589, |
|
"loss": 2.7594, |
|
"theoretical_loss": 3.9365166111374608, |
|
"tokens_seen": 475136000 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.00041876300359372043, |
|
"loss": 2.7323, |
|
"theoretical_loss": 3.933579674239507, |
|
"tokens_seen": 478412800 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.00041817193115188195, |
|
"loss": 2.7369, |
|
"theoretical_loss": 3.930668373615765, |
|
"tokens_seen": 481689600 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0004175808587100435, |
|
"loss": 2.6955, |
|
"theoretical_loss": 3.9277823133624366, |
|
"tokens_seen": 484966400 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.00041698978626820505, |
|
"loss": 2.7195, |
|
"theoretical_loss": 3.9249211063121283, |
|
"tokens_seen": 488243200 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"objective/train/docs_used": 267097, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.437924861907959, |
|
"objective/train/theoretical_loss": 3.9220843737839752, |
|
"objective/train/tokens_used": 511980000, |
|
"theoretical_loss": 3.9220843737839752, |
|
"tokens_seen": 491520000 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.00041639871382636657, |
|
"loss": 2.6773, |
|
"theoretical_loss": 3.9220843737839752, |
|
"tokens_seen": 491520000 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0004158076413845281, |
|
"loss": 2.6983, |
|
"theoretical_loss": 3.919271745342514, |
|
"tokens_seen": 494796800 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0004152165689426896, |
|
"loss": 2.7126, |
|
"theoretical_loss": 3.916482858564957, |
|
"tokens_seen": 498073600 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.00041462549650085113, |
|
"loss": 2.6827, |
|
"theoretical_loss": 3.9137173588165135, |
|
"tokens_seen": 501350400 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0004140344240590127, |
|
"loss": 2.6784, |
|
"theoretical_loss": 3.9109748990334348, |
|
"tokens_seen": 504627200 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0004134433516171742, |
|
"loss": 2.6749, |
|
"theoretical_loss": 3.908255139513467, |
|
"tokens_seen": 507904000 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.00041285227917533575, |
|
"loss": 2.6522, |
|
"theoretical_loss": 3.905557747713412, |
|
"tokens_seen": 511180800 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.00041226120673349727, |
|
"loss": 2.6944, |
|
"theoretical_loss": 3.90288239805351, |
|
"tokens_seen": 514457600 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0004116701342916588, |
|
"loss": 2.7296, |
|
"theoretical_loss": 3.9002287717283783, |
|
"tokens_seen": 517734400 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0004110790618498203, |
|
"loss": 2.7132, |
|
"theoretical_loss": 3.8975965565242423, |
|
"tokens_seen": 521011200 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0004104879894079819, |
|
"loss": 2.6974, |
|
"theoretical_loss": 3.894985446642206, |
|
"tokens_seen": 524288000 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0004098969169661434, |
|
"loss": 2.7223, |
|
"theoretical_loss": 3.8923951425273398, |
|
"tokens_seen": 527564800 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0004093176659731417, |
|
"loss": 2.7323, |
|
"theoretical_loss": 3.889825350703344, |
|
"tokens_seen": 530841600 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0004087265935313032, |
|
"loss": 2.7031, |
|
"theoretical_loss": 3.8872757836125844, |
|
"tokens_seen": 534118400 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0004081355210894647, |
|
"loss": 2.6896, |
|
"theoretical_loss": 3.8847461594612884, |
|
"tokens_seen": 537395200 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.00040754444864762624, |
|
"loss": 2.6779, |
|
"theoretical_loss": 3.882236202069703, |
|
"tokens_seen": 540672000 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0004069533762057878, |
|
"loss": 2.6749, |
|
"theoretical_loss": 3.8797456407270343, |
|
"tokens_seen": 543948800 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.00040636230376394933, |
|
"loss": 2.6434, |
|
"theoretical_loss": 3.8772742100509774, |
|
"tokens_seen": 547225600 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.00040577123132211085, |
|
"loss": 2.6778, |
|
"theoretical_loss": 3.874821649851678, |
|
"tokens_seen": 550502400 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0004051801588802724, |
|
"loss": 2.6636, |
|
"theoretical_loss": 3.8723877049999444, |
|
"tokens_seen": 553779200 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0004045890864384339, |
|
"loss": 2.6414, |
|
"theoretical_loss": 3.869972125299568, |
|
"tokens_seen": 557056000 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0004039980139965954, |
|
"loss": 2.6586, |
|
"theoretical_loss": 3.867574665363595, |
|
"tokens_seen": 560332800 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.00040340694155475694, |
|
"loss": 2.6388, |
|
"theoretical_loss": 3.865195084494398, |
|
"tokens_seen": 563609600 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0004028158691129185, |
|
"loss": 2.703, |
|
"theoretical_loss": 3.8628331465674224, |
|
"tokens_seen": 566886400 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.00040222479667108003, |
|
"loss": 2.7167, |
|
"theoretical_loss": 3.860488619918462, |
|
"tokens_seen": 570163200 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.00040163372422924155, |
|
"loss": 2.6639, |
|
"theoretical_loss": 3.858161277234349, |
|
"tokens_seen": 573440000 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0004010426517874031, |
|
"loss": 2.6878, |
|
"theoretical_loss": 3.855850895446925, |
|
"tokens_seen": 576716800 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0004004515793455646, |
|
"loss": 2.6932, |
|
"theoretical_loss": 3.8535572556301823, |
|
"tokens_seen": 579993600 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0003998605069037261, |
|
"loss": 2.6625, |
|
"theoretical_loss": 3.851280142900463, |
|
"tokens_seen": 583270400 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0003992694344618877, |
|
"loss": 2.682, |
|
"theoretical_loss": 3.8490193463196127, |
|
"tokens_seen": 586547200 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0003986783620200492, |
|
"loss": 2.6965, |
|
"theoretical_loss": 3.8467746588009692, |
|
"tokens_seen": 589824000 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.00039808728957821073, |
|
"loss": 2.6945, |
|
"theoretical_loss": 3.8445458770181116, |
|
"tokens_seen": 593100800 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.00039749621713637225, |
|
"loss": 2.7248, |
|
"theoretical_loss": 3.842332801316254, |
|
"tokens_seen": 596377600 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0003969051446945338, |
|
"loss": 2.7115, |
|
"theoretical_loss": 3.840135235626204, |
|
"tokens_seen": 599654400 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0003963140722526953, |
|
"loss": 2.7297, |
|
"theoretical_loss": 3.8379529873807945, |
|
"tokens_seen": 602931200 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.00039572299981085687, |
|
"loss": 2.7183, |
|
"theoretical_loss": 3.835785867433705, |
|
"tokens_seen": 606208000 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0003951319273690184, |
|
"loss": 2.7276, |
|
"theoretical_loss": 3.8336336899805916, |
|
"tokens_seen": 609484800 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0003945408549271799, |
|
"loss": 2.7407, |
|
"theoretical_loss": 3.8314962724824495, |
|
"tokens_seen": 612761600 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.00039394978248534143, |
|
"loss": 2.6874, |
|
"theoretical_loss": 3.8293734355911297, |
|
"tokens_seen": 616038400 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.00039335871004350295, |
|
"loss": 2.7106, |
|
"theoretical_loss": 3.8272650030769433, |
|
"tokens_seen": 619315200 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0003927676376016645, |
|
"loss": 2.6863, |
|
"theoretical_loss": 3.8251708017582793, |
|
"tokens_seen": 622592000 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.000392176565159826, |
|
"loss": 2.6762, |
|
"theoretical_loss": 3.8230906614331737, |
|
"tokens_seen": 625868800 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.00039158549271798757, |
|
"loss": 2.6754, |
|
"theoretical_loss": 3.821024414812761, |
|
"tokens_seen": 629145600 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0003909944202761491, |
|
"loss": 2.6633, |
|
"theoretical_loss": 3.818971897456552, |
|
"tokens_seen": 632422400 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.00039040334783431056, |
|
"loss": 2.6559, |
|
"theoretical_loss": 3.8169329477094784, |
|
"tokens_seen": 635699200 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0003898122753924721, |
|
"loss": 2.6369, |
|
"theoretical_loss": 3.814907406640639, |
|
"tokens_seen": 638976000 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0003892212029506336, |
|
"loss": 2.6693, |
|
"theoretical_loss": 3.8128951179837056, |
|
"tokens_seen": 642252800 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0003886301305087951, |
|
"loss": 2.6337, |
|
"theoretical_loss": 3.8108959280789243, |
|
"tokens_seen": 645529600 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0003880390580669567, |
|
"loss": 2.673, |
|
"theoretical_loss": 3.8089096858166718, |
|
"tokens_seen": 648806400 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0003874479856251182, |
|
"loss": 2.6076, |
|
"theoretical_loss": 3.8069362425825037, |
|
"tokens_seen": 652083200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.6085921086021859, |
|
"debugging/distinct-1-grams": 0.759172491106508, |
|
"debugging/distinct-2-grams": 0.9540913153898007, |
|
"debugging/entropy-1-grams": 5.725213200833921, |
|
"debugging/entropy-2-grams": 6.5743955103968785, |
|
"debugging/length": 625.4545454545455, |
|
"debugging/num_segments": 11, |
|
"epoch": 0.23, |
|
"objective/train/docs_used": 351918, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.686396598815918, |
|
"objective/train/theoretical_loss": 3.8049754522036645, |
|
"objective/train/tokens_used": 675820000, |
|
"theoretical_loss": 3.8049754522036645, |
|
"tokens_seen": 655360000 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.00038685691318327974, |
|
"loss": 2.6693, |
|
"theoretical_loss": 3.8049754522036645, |
|
"tokens_seen": 655360000 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.00038626584074144126, |
|
"loss": 2.638, |
|
"theoretical_loss": 3.8030271708970003, |
|
"tokens_seen": 658636800 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0003856747682996028, |
|
"loss": 2.5839, |
|
"theoretical_loss": 3.801091257218237, |
|
"tokens_seen": 661913600 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0003850836958577643, |
|
"loss": 2.6656, |
|
"theoretical_loss": 3.79916757201258, |
|
"tokens_seen": 665190400 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0003844926234159259, |
|
"loss": 2.6729, |
|
"theoretical_loss": 3.7972559783665965, |
|
"tokens_seen": 668467200 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0003839015509740874, |
|
"loss": 2.6729, |
|
"theoretical_loss": 3.7953563415613325, |
|
"tokens_seen": 671744000 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0003833104785322489, |
|
"loss": 2.6863, |
|
"theoretical_loss": 3.7934685290266454, |
|
"tokens_seen": 675020800 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.00038271940609041044, |
|
"loss": 2.7095, |
|
"theoretical_loss": 3.7915924102966914, |
|
"tokens_seen": 678297600 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.00038212833364857196, |
|
"loss": 2.6793, |
|
"theoretical_loss": 3.789727856966552, |
|
"tokens_seen": 681574400 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0003815372612067335, |
|
"loss": 2.6638, |
|
"theoretical_loss": 3.787874742649958, |
|
"tokens_seen": 684851200 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.000380946188764895, |
|
"loss": 2.6621, |
|
"theoretical_loss": 3.786032942938073, |
|
"tokens_seen": 688128000 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0003803551163230566, |
|
"loss": 2.6211, |
|
"theoretical_loss": 3.784202335359316, |
|
"tokens_seen": 691404800 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0003797640438812181, |
|
"loss": 2.6663, |
|
"theoretical_loss": 3.7823827993401844, |
|
"tokens_seen": 694681600 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0003791729714393796, |
|
"loss": 2.6912, |
|
"theoretical_loss": 3.7805742161670466, |
|
"tokens_seen": 697958400 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.00037858189899754114, |
|
"loss": 2.6644, |
|
"theoretical_loss": 3.7787764689488847, |
|
"tokens_seen": 701235200 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0003780026480045394, |
|
"loss": 2.6506, |
|
"theoretical_loss": 3.77698944258095, |
|
"tokens_seen": 704512000 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.00037741157556270093, |
|
"loss": 2.6822, |
|
"theoretical_loss": 3.7752130237093064, |
|
"tokens_seen": 707788800 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0003768205031208625, |
|
"loss": 2.6831, |
|
"theoretical_loss": 3.773447100696245, |
|
"tokens_seen": 711065600 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.000376229430679024, |
|
"loss": 2.7173, |
|
"theoretical_loss": 3.771691563586529, |
|
"tokens_seen": 714342400 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.00037563835823718555, |
|
"loss": 2.6928, |
|
"theoretical_loss": 3.7699463040744616, |
|
"tokens_seen": 717619200 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.00037504728579534707, |
|
"loss": 2.6826, |
|
"theoretical_loss": 3.768211215471741, |
|
"tokens_seen": 720896000 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0003744562133535086, |
|
"loss": 2.7257, |
|
"theoretical_loss": 3.766486192676084, |
|
"tokens_seen": 724172800 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0003738651409116701, |
|
"loss": 2.6754, |
|
"theoretical_loss": 3.764771132140602, |
|
"tokens_seen": 727449600 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0003732740684698317, |
|
"loss": 2.6115, |
|
"theoretical_loss": 3.763065931843898, |
|
"tokens_seen": 730726400 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0003726829960279932, |
|
"loss": 2.643, |
|
"theoretical_loss": 3.7613704912608723, |
|
"tokens_seen": 734003200 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0003720919235861547, |
|
"loss": 2.6489, |
|
"theoretical_loss": 3.759684711334213, |
|
"tokens_seen": 737280000 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.00037150085114431625, |
|
"loss": 2.6539, |
|
"theoretical_loss": 3.7580084944465555, |
|
"tokens_seen": 740556800 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.00037090977870247777, |
|
"loss": 2.6001, |
|
"theoretical_loss": 3.7563417443932905, |
|
"tokens_seen": 743833600 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0003703187062606393, |
|
"loss": 2.6737, |
|
"theoretical_loss": 3.754684366355999, |
|
"tokens_seen": 747110400 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0003697276338188008, |
|
"loss": 2.6438, |
|
"theoretical_loss": 3.753036266876505, |
|
"tokens_seen": 750387200 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0003691365613769624, |
|
"loss": 2.6378, |
|
"theoretical_loss": 3.751397353831524, |
|
"tokens_seen": 753664000 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0003685454889351239, |
|
"loss": 2.6325, |
|
"theoretical_loss": 3.749767536407891, |
|
"tokens_seen": 756940800 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0003679544164932854, |
|
"loss": 2.6137, |
|
"theoretical_loss": 3.7481467250783504, |
|
"tokens_seen": 760217600 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.00036736334405144695, |
|
"loss": 2.5941, |
|
"theoretical_loss": 3.746534831577904, |
|
"tokens_seen": 763494400 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.00036677227160960847, |
|
"loss": 2.569, |
|
"theoretical_loss": 3.744931768880681, |
|
"tokens_seen": 766771200 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.00036618119916777, |
|
"loss": 2.5835, |
|
"theoretical_loss": 3.743337451177343, |
|
"tokens_seen": 770048000 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.00036559012672593156, |
|
"loss": 2.5841, |
|
"theoretical_loss": 3.74175179385298, |
|
"tokens_seen": 773324800 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0003649990542840931, |
|
"loss": 2.6426, |
|
"theoretical_loss": 3.740174713465512, |
|
"tokens_seen": 776601600 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0003644079818422546, |
|
"loss": 2.6589, |
|
"theoretical_loss": 3.7386061277245655, |
|
"tokens_seen": 779878400 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0003638169094004161, |
|
"loss": 2.6461, |
|
"theoretical_loss": 3.7370459554708146, |
|
"tokens_seen": 783155200 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.00036322583695857765, |
|
"loss": 2.6418, |
|
"theoretical_loss": 3.735494116655784, |
|
"tokens_seen": 786432000 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.00036263476451673917, |
|
"loss": 2.6318, |
|
"theoretical_loss": 3.733950532322087, |
|
"tokens_seen": 789708800 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.00036204369207490074, |
|
"loss": 2.5808, |
|
"theoretical_loss": 3.7324151245841044, |
|
"tokens_seen": 792985600 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.00036145261963306226, |
|
"loss": 2.633, |
|
"theoretical_loss": 3.730887816609077, |
|
"tokens_seen": 796262400 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0003608615471912238, |
|
"loss": 2.601, |
|
"theoretical_loss": 3.729368532598609, |
|
"tokens_seen": 799539200 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0003602704747493853, |
|
"loss": 2.5986, |
|
"theoretical_loss": 3.7278571977705734, |
|
"tokens_seen": 802816000 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0003596794023075468, |
|
"loss": 2.5859, |
|
"theoretical_loss": 3.7263537383414023, |
|
"tokens_seen": 806092800 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.00035908832986570834, |
|
"loss": 2.5601, |
|
"theoretical_loss": 3.7248580815087626, |
|
"tokens_seen": 809369600 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0003584972574238699, |
|
"loss": 2.5597, |
|
"theoretical_loss": 3.7233701554345924, |
|
"tokens_seen": 812646400 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.00035790618498203144, |
|
"loss": 2.6287, |
|
"theoretical_loss": 3.721889889228506, |
|
"tokens_seen": 815923200 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"objective/train/docs_used": 436442, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.479308605194092, |
|
"objective/train/theoretical_loss": 3.720417212931543, |
|
"objective/train/tokens_used": 839660000, |
|
"theoretical_loss": 3.720417212931543, |
|
"tokens_seen": 819200000 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.00035731511254019296, |
|
"loss": 2.6199, |
|
"theoretical_loss": 3.720417212931543, |
|
"tokens_seen": 819200000 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0003567240400983545, |
|
"loss": 2.6091, |
|
"theoretical_loss": 3.7189520575002666, |
|
"tokens_seen": 822476800 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.000356132967656516, |
|
"loss": 2.621, |
|
"theoretical_loss": 3.717494354791188, |
|
"tokens_seen": 825753600 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0003555418952146775, |
|
"loss": 2.6322, |
|
"theoretical_loss": 3.716044037545523, |
|
"tokens_seen": 829030400 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.00035495082277283904, |
|
"loss": 2.6668, |
|
"theoretical_loss": 3.714601039374263, |
|
"tokens_seen": 832307200 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0003543597503310006, |
|
"loss": 2.6629, |
|
"theoretical_loss": 3.7131652947435536, |
|
"tokens_seen": 835584000 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.00035376867788916214, |
|
"loss": 2.6306, |
|
"theoretical_loss": 3.7117367389603793, |
|
"tokens_seen": 838860800 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.00035317760544732366, |
|
"loss": 2.6598, |
|
"theoretical_loss": 3.710315308158541, |
|
"tokens_seen": 842137600 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0003525865330054852, |
|
"loss": 2.6335, |
|
"theoretical_loss": 3.7089009392849173, |
|
"tokens_seen": 845414400 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.00035199546056364665, |
|
"loss": 2.6136, |
|
"theoretical_loss": 3.7074935700860143, |
|
"tokens_seen": 848691200 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.00035140438812180817, |
|
"loss": 2.616, |
|
"theoretical_loss": 3.706093139094781, |
|
"tokens_seen": 851968000 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00035081331567996974, |
|
"loss": 2.6422, |
|
"theoretical_loss": 3.7046995856176954, |
|
"tokens_seen": 855244800 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00035022224323813127, |
|
"loss": 2.6281, |
|
"theoretical_loss": 3.703312849722111, |
|
"tokens_seen": 858521600 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0003496311707962928, |
|
"loss": 2.6291, |
|
"theoretical_loss": 3.701932872223858, |
|
"tokens_seen": 861798400 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0003490400983544543, |
|
"loss": 2.6291, |
|
"theoretical_loss": 3.7005595946750924, |
|
"tokens_seen": 865075200 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00034846084736145263, |
|
"loss": 2.6033, |
|
"theoretical_loss": 3.699192959352386, |
|
"tokens_seen": 868352000 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00034786977491961415, |
|
"loss": 2.6207, |
|
"theoretical_loss": 3.6978329092450557, |
|
"tokens_seen": 871628800 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00034727870247777573, |
|
"loss": 2.6345, |
|
"theoretical_loss": 3.6964793880437226, |
|
"tokens_seen": 874905600 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00034668763003593725, |
|
"loss": 2.6001, |
|
"theoretical_loss": 3.6951323401290974, |
|
"tokens_seen": 878182400 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00034609655759409877, |
|
"loss": 2.6119, |
|
"theoretical_loss": 3.6937917105609834, |
|
"tokens_seen": 881459200 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0003455054851522603, |
|
"loss": 2.5726, |
|
"theoretical_loss": 3.692457445067501, |
|
"tokens_seen": 884736000 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0003449144127104218, |
|
"loss": 2.5372, |
|
"theoretical_loss": 3.6911294900345166, |
|
"tokens_seen": 888012800 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.00034432334026858333, |
|
"loss": 2.562, |
|
"theoretical_loss": 3.6898077924952775, |
|
"tokens_seen": 891289600 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.00034373226782674485, |
|
"loss": 2.5867, |
|
"theoretical_loss": 3.6884923001202505, |
|
"tokens_seen": 894566400 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.00034314119538490643, |
|
"loss": 2.5896, |
|
"theoretical_loss": 3.6871829612071583, |
|
"tokens_seen": 897843200 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0003425501229430679, |
|
"loss": 2.5958, |
|
"theoretical_loss": 3.6858797246711976, |
|
"tokens_seen": 901120000 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0003419590505012294, |
|
"loss": 2.5516, |
|
"theoretical_loss": 3.684582540035456, |
|
"tokens_seen": 904396800 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.00034136797805939094, |
|
"loss": 2.5677, |
|
"theoretical_loss": 3.683291357421508, |
|
"tokens_seen": 907673600 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00034077690561755246, |
|
"loss": 2.5641, |
|
"theoretical_loss": 3.682006127540184, |
|
"tokens_seen": 910950400 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.000340185833175714, |
|
"loss": 2.5547, |
|
"theoretical_loss": 3.680726801682522, |
|
"tokens_seen": 914227200 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00033959476073387555, |
|
"loss": 2.5809, |
|
"theoretical_loss": 3.679453331710889, |
|
"tokens_seen": 917504000 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.0003390036882920371, |
|
"loss": 2.5972, |
|
"theoretical_loss": 3.6781856700502646, |
|
"tokens_seen": 920780800 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.0003384126158501986, |
|
"loss": 2.62, |
|
"theoretical_loss": 3.6769237696796933, |
|
"tokens_seen": 924057600 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.0003378215434083601, |
|
"loss": 2.6161, |
|
"theoretical_loss": 3.6756675841238913, |
|
"tokens_seen": 927334400 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00033723047096652164, |
|
"loss": 2.5708, |
|
"theoretical_loss": 3.6744170674450176, |
|
"tokens_seen": 930611200 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00033663939852468316, |
|
"loss": 2.5809, |
|
"theoretical_loss": 3.673172174234587, |
|
"tokens_seen": 933888000 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00033604832608284473, |
|
"loss": 2.5826, |
|
"theoretical_loss": 3.6719328596055423, |
|
"tokens_seen": 937164800 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.00033545725364100625, |
|
"loss": 2.5825, |
|
"theoretical_loss": 3.670699079184467, |
|
"tokens_seen": 940441600 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0003348661811991678, |
|
"loss": 2.5917, |
|
"theoretical_loss": 3.669470789103942, |
|
"tokens_seen": 943718400 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0003342751087573293, |
|
"loss": 2.5624, |
|
"theoretical_loss": 3.6682479459950446, |
|
"tokens_seen": 946995200 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0003336840363154908, |
|
"loss": 2.5358, |
|
"theoretical_loss": 3.6670305069799785, |
|
"tokens_seen": 950272000 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.00033309296387365234, |
|
"loss": 2.5804, |
|
"theoretical_loss": 3.6658184296648457, |
|
"tokens_seen": 953548800 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.00033250189143181386, |
|
"loss": 2.5341, |
|
"theoretical_loss": 3.6646116721325415, |
|
"tokens_seen": 956825600 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.00033191081898997543, |
|
"loss": 2.5757, |
|
"theoretical_loss": 3.6634101929357836, |
|
"tokens_seen": 960102400 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.00033131974654813695, |
|
"loss": 2.5753, |
|
"theoretical_loss": 3.6622139510902625, |
|
"tokens_seen": 963379200 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0003307286741062985, |
|
"loss": 2.5634, |
|
"theoretical_loss": 3.6610229060679167, |
|
"tokens_seen": 966656000 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00033013760166446, |
|
"loss": 2.5877, |
|
"theoretical_loss": 3.659837017790328, |
|
"tokens_seen": 969932800 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0003295465292226215, |
|
"loss": 2.5654, |
|
"theoretical_loss": 3.658656246622233, |
|
"tokens_seen": 973209600 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00032895545678078304, |
|
"loss": 2.5835, |
|
"theoretical_loss": 3.6574805533651515, |
|
"tokens_seen": 976486400 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0003283643843389446, |
|
"loss": 2.6081, |
|
"theoretical_loss": 3.6563098992511267, |
|
"tokens_seen": 979763200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.5126037496134678, |
|
"debugging/distinct-1-grams": 0.765098344066948, |
|
"debugging/distinct-2-grams": 0.9645141262208726, |
|
"debugging/entropy-1-grams": 5.153832208994379, |
|
"debugging/entropy-2-grams": 5.8804952928462, |
|
"debugging/length": 688.0, |
|
"debugging/num_segments": 5, |
|
"epoch": 0.35, |
|
"objective/train/docs_used": 522194, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.8662092685699463, |
|
"objective/train/theoretical_loss": 3.655144245936574, |
|
"objective/train/tokens_used": 1003500000, |
|
"theoretical_loss": 3.655144245936574, |
|
"tokens_seen": 983040000 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00032777331189710613, |
|
"loss": 2.5693, |
|
"theoretical_loss": 3.655144245936574, |
|
"tokens_seen": 983040000 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00032718223945526765, |
|
"loss": 2.5931, |
|
"theoretical_loss": 3.653983555496242, |
|
"tokens_seen": 986316800 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0003265911670134292, |
|
"loss": 2.6015, |
|
"theoretical_loss": 3.6528277904172755, |
|
"tokens_seen": 989593600 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0003260000945715907, |
|
"loss": 2.5755, |
|
"theoretical_loss": 3.6516769135933815, |
|
"tokens_seen": 992870400 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0003254090221297522, |
|
"loss": 2.5793, |
|
"theoretical_loss": 3.650530888319103, |
|
"tokens_seen": 996147200 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0003248179496879138, |
|
"loss": 2.5388, |
|
"theoretical_loss": 3.649389678284182, |
|
"tokens_seen": 999424000 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0003242268772460753, |
|
"loss": 2.5747, |
|
"theoretical_loss": 3.6482532475680287, |
|
"tokens_seen": 1002700800 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.00032363580480423683, |
|
"loss": 2.5263, |
|
"theoretical_loss": 3.6471215606342833, |
|
"tokens_seen": 1005977600 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.00032304473236239835, |
|
"loss": 2.5382, |
|
"theoretical_loss": 3.645994582325468, |
|
"tokens_seen": 1009254400 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.00032245365992055987, |
|
"loss": 2.5374, |
|
"theoretical_loss": 3.6448722778577327, |
|
"tokens_seen": 1012531200 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0003218625874787214, |
|
"loss": 2.5239, |
|
"theoretical_loss": 3.6437546128156946, |
|
"tokens_seen": 1015808000 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0003212715150368829, |
|
"loss": 2.5326, |
|
"theoretical_loss": 3.6426415531473566, |
|
"tokens_seen": 1019084800 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0003206804425950445, |
|
"loss": 2.5245, |
|
"theoretical_loss": 3.641533065159118, |
|
"tokens_seen": 1022361600 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.000320089370153206, |
|
"loss": 2.554, |
|
"theoretical_loss": 3.6404291155108712, |
|
"tokens_seen": 1025638400 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00031949829771136753, |
|
"loss": 2.5465, |
|
"theoretical_loss": 3.639329671211173, |
|
"tokens_seen": 1028915200 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00031890722526952905, |
|
"loss": 2.523, |
|
"theoretical_loss": 3.6382346996125055, |
|
"tokens_seen": 1032192000 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00031831615282769057, |
|
"loss": 2.5361, |
|
"theoretical_loss": 3.6371441684066097, |
|
"tokens_seen": 1035468800 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0003177250803858521, |
|
"loss": 2.5422, |
|
"theoretical_loss": 3.6360580456199036, |
|
"tokens_seen": 1038745600 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00031713400794401367, |
|
"loss": 2.562, |
|
"theoretical_loss": 3.6349762996089683, |
|
"tokens_seen": 1042022400 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0003165429355021752, |
|
"loss": 2.5864, |
|
"theoretical_loss": 3.633898899056115, |
|
"tokens_seen": 1045299200 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0003159518630603367, |
|
"loss": 2.6185, |
|
"theoretical_loss": 3.6328258129650246, |
|
"tokens_seen": 1048576000 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.00031536079061849823, |
|
"loss": 2.6483, |
|
"theoretical_loss": 3.6317570106564565, |
|
"tokens_seen": 1051852800 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.00031476971817665975, |
|
"loss": 2.655, |
|
"theoretical_loss": 3.6306924617640295, |
|
"tokens_seen": 1055129600 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0003141786457348212, |
|
"loss": 2.6392, |
|
"theoretical_loss": 3.6296321362300716, |
|
"tokens_seen": 1058406400 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0003135875732929828, |
|
"loss": 2.6395, |
|
"theoretical_loss": 3.6285760043015385, |
|
"tokens_seen": 1061683200 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0003129965008511443, |
|
"loss": 2.6157, |
|
"theoretical_loss": 3.6275240365259958, |
|
"tokens_seen": 1064960000 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.00031240542840930583, |
|
"loss": 2.6223, |
|
"theoretical_loss": 3.6264762037476683, |
|
"tokens_seen": 1068236800 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.00031181435596746736, |
|
"loss": 2.6601, |
|
"theoretical_loss": 3.625432477103554, |
|
"tokens_seen": 1071513600 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0003112232835256289, |
|
"loss": 2.6198, |
|
"theoretical_loss": 3.6243928280195976, |
|
"tokens_seen": 1074790400 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0003106322110837904, |
|
"loss": 2.6219, |
|
"theoretical_loss": 3.62335722820693, |
|
"tokens_seen": 1078067200 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0003100411386419519, |
|
"loss": 2.5738, |
|
"theoretical_loss": 3.6223256496581637, |
|
"tokens_seen": 1081344000 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0003094500662001135, |
|
"loss": 2.5973, |
|
"theoretical_loss": 3.6212980646437485, |
|
"tokens_seen": 1084620800 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.000308858993758275, |
|
"loss": 2.6107, |
|
"theoretical_loss": 3.6202744457083877, |
|
"tokens_seen": 1087897600 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.00030826792131643653, |
|
"loss": 2.6013, |
|
"theoretical_loss": 3.6192547656675083, |
|
"tokens_seen": 1091174400 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.00030767684887459806, |
|
"loss": 2.5822, |
|
"theoretical_loss": 3.618238997603788, |
|
"tokens_seen": 1094451200 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0003070857764327596, |
|
"loss": 2.6122, |
|
"theoretical_loss": 3.617227114863738, |
|
"tokens_seen": 1097728000 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0003064947039909211, |
|
"loss": 2.5904, |
|
"theoretical_loss": 3.6162190910543366, |
|
"tokens_seen": 1101004800 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.00030590363154908267, |
|
"loss": 2.63, |
|
"theoretical_loss": 3.615214900039721, |
|
"tokens_seen": 1104281600 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0003053125591072442, |
|
"loss": 2.6046, |
|
"theoretical_loss": 3.614214515937924, |
|
"tokens_seen": 1107558400 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0003047214866654057, |
|
"loss": 2.63, |
|
"theoretical_loss": 3.613217913117667, |
|
"tokens_seen": 1110835200 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00030413041422356723, |
|
"loss": 2.573, |
|
"theoretical_loss": 3.612225066195201, |
|
"tokens_seen": 1114112000 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00030353934178172875, |
|
"loss": 2.5856, |
|
"theoretical_loss": 3.611235950031194, |
|
"tokens_seen": 1117388800 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0003029482693398903, |
|
"loss": 2.5853, |
|
"theoretical_loss": 3.6102505397276743, |
|
"tokens_seen": 1120665600 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00030235719689805185, |
|
"loss": 2.5413, |
|
"theoretical_loss": 3.60926881062501, |
|
"tokens_seen": 1123942400 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00030176612445621337, |
|
"loss": 2.5508, |
|
"theoretical_loss": 3.608290738298942, |
|
"tokens_seen": 1127219200 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0003011750520143749, |
|
"loss": 2.5489, |
|
"theoretical_loss": 3.6073162985576643, |
|
"tokens_seen": 1130496000 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0003005839795725364, |
|
"loss": 2.557, |
|
"theoretical_loss": 3.606345467438941, |
|
"tokens_seen": 1133772800 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00029999290713069793, |
|
"loss": 2.5535, |
|
"theoretical_loss": 3.6053782212072747, |
|
"tokens_seen": 1137049600 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00029940183468885945, |
|
"loss": 2.5767, |
|
"theoretical_loss": 3.604414536351113, |
|
"tokens_seen": 1140326400 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00029881076224702103, |
|
"loss": 2.5432, |
|
"theoretical_loss": 3.6034543895801017, |
|
"tokens_seen": 1143603200 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"objective/train/docs_used": 604425, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.4892125129699707, |
|
"objective/train/theoretical_loss": 3.6024977578223742, |
|
"objective/train/tokens_used": 1167340000, |
|
"theoretical_loss": 3.6024977578223742, |
|
"tokens_seen": 1146880000 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00029821968980518255, |
|
"loss": 2.5827, |
|
"theoretical_loss": 3.6024977578223742, |
|
"tokens_seen": 1146880000 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00029762861736334407, |
|
"loss": 2.555, |
|
"theoretical_loss": 3.6015446182218875, |
|
"tokens_seen": 1150156800 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.0002970375449215056, |
|
"loss": 2.5682, |
|
"theoretical_loss": 3.600594948135793, |
|
"tokens_seen": 1153433600 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.0002964464724796671, |
|
"loss": 2.5187, |
|
"theoretical_loss": 3.59964872513185, |
|
"tokens_seen": 1156710400 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00029585540003782863, |
|
"loss": 2.5455, |
|
"theoretical_loss": 3.5987059269858763, |
|
"tokens_seen": 1159987200 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00029526432759599015, |
|
"loss": 2.5247, |
|
"theoretical_loss": 3.5977665316792375, |
|
"tokens_seen": 1163264000 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00029467325515415173, |
|
"loss": 2.5639, |
|
"theoretical_loss": 3.5968305173963744, |
|
"tokens_seen": 1166540800 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00029408218271231325, |
|
"loss": 2.5256, |
|
"theoretical_loss": 3.5958978625223628, |
|
"tokens_seen": 1169817600 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0002935147531681483, |
|
"loss": 2.5869, |
|
"theoretical_loss": 3.5949685456405165, |
|
"tokens_seen": 1173094400 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0002929236807263098, |
|
"loss": 2.5532, |
|
"theoretical_loss": 3.5940425455300176, |
|
"tokens_seen": 1176371200 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0002923326082844713, |
|
"loss": 2.5344, |
|
"theoretical_loss": 3.593119841163589, |
|
"tokens_seen": 1179648000 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00029174153584263284, |
|
"loss": 2.5555, |
|
"theoretical_loss": 3.5922004117051944, |
|
"tokens_seen": 1182924800 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0002911504634007944, |
|
"loss": 2.5522, |
|
"theoretical_loss": 3.5912842365077777, |
|
"tokens_seen": 1186201600 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00029055939095895593, |
|
"loss": 2.5228, |
|
"theoretical_loss": 3.5903712951110305, |
|
"tokens_seen": 1189478400 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.00028996831851711745, |
|
"loss": 2.5614, |
|
"theoretical_loss": 3.5894615672391947, |
|
"tokens_seen": 1192755200 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.000289377246075279, |
|
"loss": 2.548, |
|
"theoretical_loss": 3.5885550327988973, |
|
"tokens_seen": 1196032000 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.0002887861736334405, |
|
"loss": 2.5532, |
|
"theoretical_loss": 3.587651671877014, |
|
"tokens_seen": 1199308800 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.000288195101191602, |
|
"loss": 2.5361, |
|
"theoretical_loss": 3.5867514647385663, |
|
"tokens_seen": 1202585600 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.0002876040287497636, |
|
"loss": 2.5397, |
|
"theoretical_loss": 3.585854391824647, |
|
"tokens_seen": 1205862400 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.0002870129563079251, |
|
"loss": 2.5966, |
|
"theoretical_loss": 3.584960433750375, |
|
"tokens_seen": 1209139200 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.00028642188386608663, |
|
"loss": 2.6267, |
|
"theoretical_loss": 3.5840695713028827, |
|
"tokens_seen": 1212416000 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.00028583081142424815, |
|
"loss": 2.5783, |
|
"theoretical_loss": 3.5831817854393266, |
|
"tokens_seen": 1215692800 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00028523973898240967, |
|
"loss": 2.5645, |
|
"theoretical_loss": 3.582297057284933, |
|
"tokens_seen": 1218969600 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0002846486665405712, |
|
"loss": 2.5307, |
|
"theoretical_loss": 3.5814153681310623, |
|
"tokens_seen": 1222246400 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0002840575940987327, |
|
"loss": 2.5653, |
|
"theoretical_loss": 3.5805366994333125, |
|
"tokens_seen": 1225523200 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0002834665216568943, |
|
"loss": 2.5458, |
|
"theoretical_loss": 3.5796610328096365, |
|
"tokens_seen": 1228800000 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0002828754492150558, |
|
"loss": 2.5545, |
|
"theoretical_loss": 3.578788350038497, |
|
"tokens_seen": 1232076800 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00028228437677321733, |
|
"loss": 2.5606, |
|
"theoretical_loss": 3.5779186330570405, |
|
"tokens_seen": 1235353600 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00028169330433137885, |
|
"loss": 2.5751, |
|
"theoretical_loss": 3.5770518639592983, |
|
"tokens_seen": 1238630400 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00028110223188954037, |
|
"loss": 2.546, |
|
"theoretical_loss": 3.5761880249944147, |
|
"tokens_seen": 1241907200 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0002805111594477019, |
|
"loss": 2.5182, |
|
"theoretical_loss": 3.5753270985648973, |
|
"tokens_seen": 1245184000 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00027992008700586347, |
|
"loss": 2.556, |
|
"theoretical_loss": 3.574469067224892, |
|
"tokens_seen": 1248460800 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.000279329014564025, |
|
"loss": 2.537, |
|
"theoretical_loss": 3.573613913678484, |
|
"tokens_seen": 1251737600 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.0002787379421221865, |
|
"loss": 2.5756, |
|
"theoretical_loss": 3.57276162077802, |
|
"tokens_seen": 1255014400 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00027814686968034803, |
|
"loss": 2.5297, |
|
"theoretical_loss": 3.5719121715224524, |
|
"tokens_seen": 1258291200 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00027755579723850955, |
|
"loss": 2.5528, |
|
"theoretical_loss": 3.571065549055712, |
|
"tokens_seen": 1261568000 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00027696472479667107, |
|
"loss": 2.5302, |
|
"theoretical_loss": 3.5702217366650935, |
|
"tokens_seen": 1264844800 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00027637365235483265, |
|
"loss": 2.5298, |
|
"theoretical_loss": 3.5693807177796737, |
|
"tokens_seen": 1268121600 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00027578257991299417, |
|
"loss": 2.5087, |
|
"theoretical_loss": 3.5685424759687434, |
|
"tokens_seen": 1271398400 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0002751915074711557, |
|
"loss": 2.5285, |
|
"theoretical_loss": 3.567706994940263, |
|
"tokens_seen": 1274675200 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0002746004350293172, |
|
"loss": 2.5489, |
|
"theoretical_loss": 3.5668742585393405, |
|
"tokens_seen": 1277952000 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00027400936258747873, |
|
"loss": 2.5389, |
|
"theoretical_loss": 3.566044250746728, |
|
"tokens_seen": 1281228800 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00027341829014564025, |
|
"loss": 2.5822, |
|
"theoretical_loss": 3.5652169556773403, |
|
"tokens_seen": 1284505600 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0002728272177038018, |
|
"loss": 2.5496, |
|
"theoretical_loss": 3.5643923575787912, |
|
"tokens_seen": 1287782400 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00027223614526196335, |
|
"loss": 2.5455, |
|
"theoretical_loss": 3.563570440829951, |
|
"tokens_seen": 1291059200 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00027164507282012487, |
|
"loss": 2.511, |
|
"theoretical_loss": 3.562751189939524, |
|
"tokens_seen": 1294336000 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0002710540003782864, |
|
"loss": 2.5489, |
|
"theoretical_loss": 3.5619345895446424, |
|
"tokens_seen": 1297612800 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0002704629279364479, |
|
"loss": 2.5534, |
|
"theoretical_loss": 3.561120624409482, |
|
"tokens_seen": 1300889600 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00026987185549460943, |
|
"loss": 2.5526, |
|
"theoretical_loss": 3.560309279423894, |
|
"tokens_seen": 1304166400 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00026928078305277095, |
|
"loss": 2.5701, |
|
"theoretical_loss": 3.5595005396020554, |
|
"tokens_seen": 1307443200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.651313007003922, |
|
"debugging/distinct-1-grams": 0.7951009299528532, |
|
"debugging/distinct-2-grams": 0.9619369637440748, |
|
"debugging/entropy-1-grams": 6.171223849610252, |
|
"debugging/entropy-2-grams": 7.04386858885556, |
|
"debugging/length": 624.7647058823529, |
|
"debugging/num_segments": 17, |
|
"epoch": 0.47, |
|
"objective/train/docs_used": 687328, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.8087964057922363, |
|
"objective/train/theoretical_loss": 3.558694390081137, |
|
"objective/train/tokens_used": 1331180000, |
|
"theoretical_loss": 3.558694390081137, |
|
"tokens_seen": 1310720000 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.0002686897106109325, |
|
"loss": 2.5949, |
|
"theoretical_loss": 3.558694390081137, |
|
"tokens_seen": 1310720000 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00026809863816909405, |
|
"loss": 2.6024, |
|
"theoretical_loss": 3.5578908161199934, |
|
"tokens_seen": 1313996800 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00026750756572725557, |
|
"loss": 2.591, |
|
"theoretical_loss": 3.5570898030978584, |
|
"tokens_seen": 1317273600 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.0002669164932854171, |
|
"loss": 2.5953, |
|
"theoretical_loss": 3.556291336513074, |
|
"tokens_seen": 1320550400 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00026632542084357855, |
|
"loss": 2.5562, |
|
"theoretical_loss": 3.5554954019818235, |
|
"tokens_seen": 1323827200 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.0002657343484017401, |
|
"loss": 2.5448, |
|
"theoretical_loss": 3.554701985236883, |
|
"tokens_seen": 1327104000 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026514327595990165, |
|
"loss": 2.5412, |
|
"theoretical_loss": 3.553911072126394, |
|
"tokens_seen": 1330380800 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026455220351806317, |
|
"loss": 2.5461, |
|
"theoretical_loss": 3.5531226486126504, |
|
"tokens_seen": 1333657600 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.0002639611310762247, |
|
"loss": 2.5721, |
|
"theoretical_loss": 3.552336700770896, |
|
"tokens_seen": 1336934400 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.0002633700586343862, |
|
"loss": 2.5316, |
|
"theoretical_loss": 3.5515532147881443, |
|
"tokens_seen": 1340211200 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026277898619254773, |
|
"loss": 2.5278, |
|
"theoretical_loss": 3.5507721769620098, |
|
"tokens_seen": 1343488000 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026218791375070925, |
|
"loss": 2.5232, |
|
"theoretical_loss": 3.549993573699556, |
|
"tokens_seen": 1346764800 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026159684130887083, |
|
"loss": 2.5212, |
|
"theoretical_loss": 3.5492173915161565, |
|
"tokens_seen": 1350041600 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026100576886703235, |
|
"loss": 2.4865, |
|
"theoretical_loss": 3.548443617034371, |
|
"tokens_seen": 1353318400 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026041469642519387, |
|
"loss": 2.5037, |
|
"theoretical_loss": 3.547672236982839, |
|
"tokens_seen": 1356595200 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.0002598236239833554, |
|
"loss": 2.4909, |
|
"theoretical_loss": 3.5469032381951804, |
|
"tokens_seen": 1359872000 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.0002592325515415169, |
|
"loss": 2.5257, |
|
"theoretical_loss": 3.5461366076089202, |
|
"tokens_seen": 1363148800 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00025864147909967843, |
|
"loss": 2.5239, |
|
"theoretical_loss": 3.5453723322644146, |
|
"tokens_seen": 1366425600 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00025805040665783995, |
|
"loss": 2.511, |
|
"theoretical_loss": 3.544610399303803, |
|
"tokens_seen": 1369702400 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00025745933421600153, |
|
"loss": 2.485, |
|
"theoretical_loss": 3.5438507959699637, |
|
"tokens_seen": 1372979200 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00025686826177416305, |
|
"loss": 2.5092, |
|
"theoretical_loss": 3.5430935096054883, |
|
"tokens_seen": 1376256000 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00025627718933232457, |
|
"loss": 2.5144, |
|
"theoretical_loss": 3.5423385276516663, |
|
"tokens_seen": 1379532800 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.0002556861168904861, |
|
"loss": 2.4963, |
|
"theoretical_loss": 3.5415858376474825, |
|
"tokens_seen": 1382809600 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.0002550950444486476, |
|
"loss": 2.4899, |
|
"theoretical_loss": 3.5408354272286298, |
|
"tokens_seen": 1386086400 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.00025450397200680913, |
|
"loss": 2.4869, |
|
"theoretical_loss": 3.540087284126531, |
|
"tokens_seen": 1389363200 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.0002539128995649707, |
|
"loss": 2.4922, |
|
"theoretical_loss": 3.539341396167372, |
|
"tokens_seen": 1392640000 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.00025332182712313223, |
|
"loss": 2.518, |
|
"theoretical_loss": 3.538597751271153, |
|
"tokens_seen": 1395916800 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.00025273075468129375, |
|
"loss": 2.4965, |
|
"theoretical_loss": 3.5378563374507443, |
|
"tokens_seen": 1399193600 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.00025213968223945527, |
|
"loss": 2.5072, |
|
"theoretical_loss": 3.5371171428109576, |
|
"tokens_seen": 1402470400 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.0002515486097976168, |
|
"loss": 2.5016, |
|
"theoretical_loss": 3.5363801555476275, |
|
"tokens_seen": 1405747200 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.0002509575373557783, |
|
"loss": 2.5045, |
|
"theoretical_loss": 3.5356453639467063, |
|
"tokens_seen": 1409024000 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.0002503664649139399, |
|
"loss": 2.5249, |
|
"theoretical_loss": 3.534912756383366, |
|
"tokens_seen": 1412300800 |
|
}, |
|
{ |
|
"epoch": 0.51, |
|
"learning_rate": 0.00024977539247210135, |
|
"loss": 2.5094, |
|
"theoretical_loss": 3.534182321321116, |
|
"tokens_seen": 1415577600 |
|
}, |
|
{ |
|
"epoch": 0.51, |
|
"learning_rate": 0.00024918432003026293, |
|
"loss": 2.5422, |
|
"theoretical_loss": 3.5334540473109275, |
|
"tokens_seen": 1418854400 |
|
}, |
|
{ |
|
"epoch": 0.51, |
|
"learning_rate": 0.00024859324758842445, |
|
"loss": 2.5008, |
|
"theoretical_loss": 3.532727922990372, |
|
"tokens_seen": 1422131200 |
|
}, |
|
{ |
|
"epoch": 0.51, |
|
"learning_rate": 0.00024800217514658597, |
|
"loss": 2.5131, |
|
"theoretical_loss": 3.532003937082766, |
|
"tokens_seen": 1425408000 |
|
}, |
|
{ |
|
"epoch": 0.51, |
|
"learning_rate": 0.0002474111027047475, |
|
"loss": 2.5424, |
|
"theoretical_loss": 3.5312820783963303, |
|
"tokens_seen": 1428684800 |
|
}, |
|
{ |
|
"epoch": 0.51, |
|
"learning_rate": 0.000246820030262909, |
|
"loss": 2.493, |
|
"theoretical_loss": 3.5305623358233578, |
|
"tokens_seen": 1431961600 |
|
}, |
|
{ |
|
"epoch": 0.51, |
|
"learning_rate": 0.00024622895782107053, |
|
"loss": 2.5305, |
|
"theoretical_loss": 3.529844698339391, |
|
"tokens_seen": 1435238400 |
|
}, |
|
{ |
|
"epoch": 0.51, |
|
"learning_rate": 0.0002456378853792321, |
|
"loss": 2.5218, |
|
"theoretical_loss": 3.5291291550024093, |
|
"tokens_seen": 1438515200 |
|
}, |
|
{ |
|
"epoch": 0.51, |
|
"learning_rate": 0.00024504681293739363, |
|
"loss": 2.5075, |
|
"theoretical_loss": 3.5284156949520264, |
|
"tokens_seen": 1441792000 |
|
}, |
|
{ |
|
"epoch": 0.52, |
|
"learning_rate": 0.00024445574049555515, |
|
"loss": 2.5002, |
|
"theoretical_loss": 3.5277043074086993, |
|
"tokens_seen": 1445068800 |
|
}, |
|
{ |
|
"epoch": 0.52, |
|
"learning_rate": 0.00024386466805371667, |
|
"loss": 2.4871, |
|
"theoretical_loss": 3.5269949816729422, |
|
"tokens_seen": 1448345600 |
|
}, |
|
{ |
|
"epoch": 0.52, |
|
"learning_rate": 0.00024328541706071497, |
|
"loss": 2.5129, |
|
"theoretical_loss": 3.5262877071245526, |
|
"tokens_seen": 1451622400 |
|
}, |
|
{ |
|
"epoch": 0.52, |
|
"learning_rate": 0.0002426943446188765, |
|
"loss": 2.482, |
|
"theoretical_loss": 3.525582473221849, |
|
"tokens_seen": 1454899200 |
|
}, |
|
{ |
|
"epoch": 0.52, |
|
"learning_rate": 0.00024210327217703804, |
|
"loss": 2.4745, |
|
"theoretical_loss": 3.5248792695009126, |
|
"tokens_seen": 1458176000 |
|
}, |
|
{ |
|
"epoch": 0.52, |
|
"learning_rate": 0.00024151219973519956, |
|
"loss": 2.505, |
|
"theoretical_loss": 3.5241780855748415, |
|
"tokens_seen": 1461452800 |
|
}, |
|
{ |
|
"epoch": 0.52, |
|
"learning_rate": 0.00024092112729336108, |
|
"loss": 2.5109, |
|
"theoretical_loss": 3.523478911133012, |
|
"tokens_seen": 1464729600 |
|
}, |
|
{ |
|
"epoch": 0.52, |
|
"learning_rate": 0.0002403300548515226, |
|
"loss": 2.542, |
|
"theoretical_loss": 3.5227817359403506, |
|
"tokens_seen": 1468006400 |
|
}, |
|
{ |
|
"epoch": 0.53, |
|
"learning_rate": 0.00023973898240968415, |
|
"loss": 2.5396, |
|
"theoretical_loss": 3.5220865498366107, |
|
"tokens_seen": 1471283200 |
|
}, |
|
{ |
|
"epoch": 0.53, |
|
"objective/train/docs_used": 774394, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.5788989067077637, |
|
"objective/train/theoretical_loss": 3.5213933427356636, |
|
"objective/train/tokens_used": 1495020000, |
|
"theoretical_loss": 3.5213933427356636, |
|
"tokens_seen": 1474560000 |
|
}, |
|
{ |
|
"epoch": 0.53, |
|
"learning_rate": 0.00023914790996784567, |
|
"loss": 2.5359, |
|
"theoretical_loss": 3.5213933427356636, |
|
"tokens_seen": 1474560000 |
|
}, |
|
{ |
|
"epoch": 0.53, |
|
"learning_rate": 0.0002385568375260072, |
|
"loss": 2.5404, |
|
"theoretical_loss": 3.5207021046247924, |
|
"tokens_seen": 1477836800 |
|
}, |
|
{ |
|
"epoch": 0.53, |
|
"learning_rate": 0.00023796576508416874, |
|
"loss": 2.5319, |
|
"theoretical_loss": 3.520012825563997, |
|
"tokens_seen": 1481113600 |
|
}, |
|
{ |
|
"epoch": 0.53, |
|
"learning_rate": 0.00023737469264233026, |
|
"loss": 2.5595, |
|
"theoretical_loss": 3.519325495685306, |
|
"tokens_seen": 1484390400 |
|
}, |
|
{ |
|
"epoch": 0.53, |
|
"learning_rate": 0.00023678362020049178, |
|
"loss": 2.5018, |
|
"theoretical_loss": 3.5186401051920977, |
|
"tokens_seen": 1487667200 |
|
}, |
|
{ |
|
"epoch": 0.53, |
|
"learning_rate": 0.00023619254775865333, |
|
"loss": 2.5427, |
|
"theoretical_loss": 3.517956644358428, |
|
"tokens_seen": 1490944000 |
|
}, |
|
{ |
|
"epoch": 0.53, |
|
"learning_rate": 0.00023560147531681482, |
|
"loss": 2.5077, |
|
"theoretical_loss": 3.517275103528368, |
|
"tokens_seen": 1494220800 |
|
}, |
|
{ |
|
"epoch": 0.53, |
|
"learning_rate": 0.00023501040287497634, |
|
"loss": 2.5064, |
|
"theoretical_loss": 3.516595473115344, |
|
"tokens_seen": 1497497600 |
|
}, |
|
{ |
|
"epoch": 0.54, |
|
"learning_rate": 0.0002344193304331379, |
|
"loss": 2.4872, |
|
"theoretical_loss": 3.5159177436014932, |
|
"tokens_seen": 1500774400 |
|
}, |
|
{ |
|
"epoch": 0.54, |
|
"learning_rate": 0.0002338400794401362, |
|
"loss": 2.5219, |
|
"theoretical_loss": 3.5152419055370205, |
|
"tokens_seen": 1504051200 |
|
}, |
|
{ |
|
"epoch": 0.54, |
|
"learning_rate": 0.0002332490069982977, |
|
"loss": 2.5326, |
|
"theoretical_loss": 3.5145679495395643, |
|
"tokens_seen": 1507328000 |
|
}, |
|
{ |
|
"epoch": 0.54, |
|
"learning_rate": 0.00023265793455645926, |
|
"loss": 2.5093, |
|
"theoretical_loss": 3.513895866293571, |
|
"tokens_seen": 1510604800 |
|
}, |
|
{ |
|
"epoch": 0.54, |
|
"learning_rate": 0.00023206686211462078, |
|
"loss": 2.4849, |
|
"theoretical_loss": 3.5132256465496736, |
|
"tokens_seen": 1513881600 |
|
}, |
|
{ |
|
"epoch": 0.54, |
|
"learning_rate": 0.0002314757896727823, |
|
"loss": 2.5022, |
|
"theoretical_loss": 3.5125572811240833, |
|
"tokens_seen": 1517158400 |
|
}, |
|
{ |
|
"epoch": 0.54, |
|
"learning_rate": 0.00023088471723094385, |
|
"loss": 2.4699, |
|
"theoretical_loss": 3.5118907608979786, |
|
"tokens_seen": 1520435200 |
|
}, |
|
{ |
|
"epoch": 0.54, |
|
"learning_rate": 0.00023029364478910537, |
|
"loss": 2.4548, |
|
"theoretical_loss": 3.5112260768169135, |
|
"tokens_seen": 1523712000 |
|
}, |
|
{ |
|
"epoch": 0.55, |
|
"learning_rate": 0.0002297025723472669, |
|
"loss": 2.456, |
|
"theoretical_loss": 3.510563219890219, |
|
"tokens_seen": 1526988800 |
|
}, |
|
{ |
|
"epoch": 0.55, |
|
"learning_rate": 0.00022911149990542844, |
|
"loss": 2.4904, |
|
"theoretical_loss": 3.509902181190423, |
|
"tokens_seen": 1530265600 |
|
}, |
|
{ |
|
"epoch": 0.55, |
|
"learning_rate": 0.00022852042746358996, |
|
"loss": 2.4782, |
|
"theoretical_loss": 3.50924295185267, |
|
"tokens_seen": 1533542400 |
|
}, |
|
{ |
|
"epoch": 0.55, |
|
"learning_rate": 0.00022792935502175148, |
|
"loss": 2.4734, |
|
"theoretical_loss": 3.508585523074149, |
|
"tokens_seen": 1536819200 |
|
}, |
|
{ |
|
"epoch": 0.55, |
|
"learning_rate": 0.000227338282579913, |
|
"loss": 2.4759, |
|
"theoretical_loss": 3.5079298861135295, |
|
"tokens_seen": 1540096000 |
|
}, |
|
{ |
|
"epoch": 0.55, |
|
"learning_rate": 0.00022674721013807455, |
|
"loss": 2.5035, |
|
"theoretical_loss": 3.5072760322904006, |
|
"tokens_seen": 1543372800 |
|
}, |
|
{ |
|
"epoch": 0.55, |
|
"learning_rate": 0.00022615613769623604, |
|
"loss": 2.4635, |
|
"theoretical_loss": 3.50662395298472, |
|
"tokens_seen": 1546649600 |
|
}, |
|
{ |
|
"epoch": 0.55, |
|
"learning_rate": 0.00022556506525439756, |
|
"loss": 2.4466, |
|
"theoretical_loss": 3.505973639636264, |
|
"tokens_seen": 1549926400 |
|
}, |
|
{ |
|
"epoch": 0.55, |
|
"learning_rate": 0.0002249739928125591, |
|
"loss": 2.3926, |
|
"theoretical_loss": 3.5053250837440926, |
|
"tokens_seen": 1553203200 |
|
}, |
|
{ |
|
"epoch": 0.56, |
|
"learning_rate": 0.00022438292037072063, |
|
"loss": 2.4428, |
|
"theoretical_loss": 3.5046782768660085, |
|
"tokens_seen": 1556480000 |
|
}, |
|
{ |
|
"epoch": 0.56, |
|
"learning_rate": 0.00022379184792888215, |
|
"loss": 2.4584, |
|
"theoretical_loss": 3.5040332106180347, |
|
"tokens_seen": 1559756800 |
|
}, |
|
{ |
|
"epoch": 0.56, |
|
"learning_rate": 0.0002232007754870437, |
|
"loss": 2.4734, |
|
"theoretical_loss": 3.503389876673886, |
|
"tokens_seen": 1563033600 |
|
}, |
|
{ |
|
"epoch": 0.56, |
|
"learning_rate": 0.00022260970304520522, |
|
"loss": 2.4559, |
|
"theoretical_loss": 3.5027482667644554, |
|
"tokens_seen": 1566310400 |
|
}, |
|
{ |
|
"epoch": 0.56, |
|
"learning_rate": 0.00022201863060336674, |
|
"loss": 2.4612, |
|
"theoretical_loss": 3.502108372677303, |
|
"tokens_seen": 1569587200 |
|
}, |
|
{ |
|
"epoch": 0.56, |
|
"learning_rate": 0.0002214275581615283, |
|
"loss": 2.4611, |
|
"theoretical_loss": 3.5014701862561486, |
|
"tokens_seen": 1572864000 |
|
}, |
|
{ |
|
"epoch": 0.56, |
|
"learning_rate": 0.0002208364857196898, |
|
"loss": 2.4715, |
|
"theoretical_loss": 3.5008336994003724, |
|
"tokens_seen": 1576140800 |
|
}, |
|
{ |
|
"epoch": 0.56, |
|
"learning_rate": 0.00022024541327785133, |
|
"loss": 2.4858, |
|
"theoretical_loss": 3.500198904064521, |
|
"tokens_seen": 1579417600 |
|
}, |
|
{ |
|
"epoch": 0.57, |
|
"learning_rate": 0.00021965434083601288, |
|
"loss": 2.5049, |
|
"theoretical_loss": 3.499565792257817, |
|
"tokens_seen": 1582694400 |
|
}, |
|
{ |
|
"epoch": 0.57, |
|
"learning_rate": 0.0002190632683941744, |
|
"loss": 2.438, |
|
"theoretical_loss": 3.4989343560436756, |
|
"tokens_seen": 1585971200 |
|
}, |
|
{ |
|
"epoch": 0.57, |
|
"learning_rate": 0.00021847219595233592, |
|
"loss": 2.4662, |
|
"theoretical_loss": 3.498304587539227, |
|
"tokens_seen": 1589248000 |
|
}, |
|
{ |
|
"epoch": 0.57, |
|
"learning_rate": 0.00021788112351049747, |
|
"loss": 2.4803, |
|
"theoretical_loss": 3.4976764789148422, |
|
"tokens_seen": 1592524800 |
|
}, |
|
{ |
|
"epoch": 0.57, |
|
"learning_rate": 0.000217290051068659, |
|
"loss": 2.4692, |
|
"theoretical_loss": 3.4970500223936627, |
|
"tokens_seen": 1595801600 |
|
}, |
|
{ |
|
"epoch": 0.57, |
|
"learning_rate": 0.0002166989786268205, |
|
"loss": 2.5094, |
|
"theoretical_loss": 3.496425210251142, |
|
"tokens_seen": 1599078400 |
|
}, |
|
{ |
|
"epoch": 0.57, |
|
"learning_rate": 0.00021610790618498203, |
|
"loss": 2.5145, |
|
"theoretical_loss": 3.4958020348145835, |
|
"tokens_seen": 1602355200 |
|
}, |
|
{ |
|
"epoch": 0.57, |
|
"learning_rate": 0.00021551683374314358, |
|
"loss": 2.5384, |
|
"theoretical_loss": 3.4951804884626885, |
|
"tokens_seen": 1605632000 |
|
}, |
|
{ |
|
"epoch": 0.57, |
|
"learning_rate": 0.0002149257613013051, |
|
"loss": 2.5115, |
|
"theoretical_loss": 3.494560563625107, |
|
"tokens_seen": 1608908800 |
|
}, |
|
{ |
|
"epoch": 0.58, |
|
"learning_rate": 0.00021433468885946662, |
|
"loss": 2.517, |
|
"theoretical_loss": 3.4939422527819968, |
|
"tokens_seen": 1612185600 |
|
}, |
|
{ |
|
"epoch": 0.58, |
|
"learning_rate": 0.00021374361641762817, |
|
"loss": 2.5163, |
|
"theoretical_loss": 3.4933255484635826, |
|
"tokens_seen": 1615462400 |
|
}, |
|
{ |
|
"epoch": 0.58, |
|
"learning_rate": 0.0002131525439757897, |
|
"loss": 2.5088, |
|
"theoretical_loss": 3.4927104432497202, |
|
"tokens_seen": 1618739200 |
|
}, |
|
{ |
|
"epoch": 0.58, |
|
"learning_rate": 0.0002125614715339512, |
|
"loss": 2.51, |
|
"theoretical_loss": 3.4920969297694713, |
|
"tokens_seen": 1622016000 |
|
}, |
|
{ |
|
"epoch": 0.58, |
|
"learning_rate": 0.00021197039909211273, |
|
"loss": 2.4924, |
|
"theoretical_loss": 3.491485000700675, |
|
"tokens_seen": 1625292800 |
|
}, |
|
{ |
|
"epoch": 0.58, |
|
"learning_rate": 0.00021137932665027425, |
|
"loss": 2.5034, |
|
"theoretical_loss": 3.4908746487695304, |
|
"tokens_seen": 1628569600 |
|
}, |
|
{ |
|
"epoch": 0.58, |
|
"learning_rate": 0.00021078825420843577, |
|
"loss": 2.4943, |
|
"theoretical_loss": 3.4902658667501774, |
|
"tokens_seen": 1631846400 |
|
}, |
|
{ |
|
"epoch": 0.58, |
|
"learning_rate": 0.00021019718176659732, |
|
"loss": 2.5155, |
|
"theoretical_loss": 3.489658647464288, |
|
"tokens_seen": 1635123200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.6771635043759617, |
|
"debugging/distinct-1-grams": 0.7447450230244734, |
|
"debugging/distinct-2-grams": 0.9552741346496469, |
|
"debugging/entropy-1-grams": 5.940930202608803, |
|
"debugging/entropy-2-grams": 6.9613187686578115, |
|
"debugging/length": 670.5333333333333, |
|
"debugging/num_segments": 15, |
|
"epoch": 0.59, |
|
"objective/train/docs_used": 859276, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.7492685317993164, |
|
"objective/train/theoretical_loss": 3.489052983780656, |
|
"objective/train/tokens_used": 1658860000, |
|
"theoretical_loss": 3.489052983780656, |
|
"tokens_seen": 1638400000 |
|
}, |
|
{ |
|
"epoch": 0.59, |
|
"learning_rate": 0.00020960610932475884, |
|
"loss": 2.4645, |
|
"theoretical_loss": 3.489052983780656, |
|
"tokens_seen": 1638400000 |
|
}, |
|
{ |
|
"epoch": 0.59, |
|
"learning_rate": 0.00020901503688292036, |
|
"loss": 2.4995, |
|
"theoretical_loss": 3.488448868614799, |
|
"tokens_seen": 1641676800 |
|
}, |
|
{ |
|
"epoch": 0.59, |
|
"learning_rate": 0.0002084239644410819, |
|
"loss": 2.4711, |
|
"theoretical_loss": 3.4878462949285556, |
|
"tokens_seen": 1644953600 |
|
}, |
|
{ |
|
"epoch": 0.59, |
|
"learning_rate": 0.00020783289199924343, |
|
"loss": 2.4798, |
|
"theoretical_loss": 3.4872452557296922, |
|
"tokens_seen": 1648230400 |
|
}, |
|
{ |
|
"epoch": 0.59, |
|
"learning_rate": 0.00020724181955740495, |
|
"loss": 2.474, |
|
"theoretical_loss": 3.486645744071512, |
|
"tokens_seen": 1651507200 |
|
}, |
|
{ |
|
"epoch": 0.59, |
|
"learning_rate": 0.0002066507471155665, |
|
"loss": 2.4483, |
|
"theoretical_loss": 3.486047753052471, |
|
"tokens_seen": 1654784000 |
|
}, |
|
{ |
|
"epoch": 0.59, |
|
"learning_rate": 0.00020605967467372802, |
|
"loss": 2.4571, |
|
"theoretical_loss": 3.4854512758157945, |
|
"tokens_seen": 1658060800 |
|
}, |
|
{ |
|
"epoch": 0.59, |
|
"learning_rate": 0.00020546860223188954, |
|
"loss": 2.4862, |
|
"theoretical_loss": 3.484856305549097, |
|
"tokens_seen": 1661337600 |
|
}, |
|
{ |
|
"epoch": 0.59, |
|
"learning_rate": 0.00020487752979005106, |
|
"loss": 2.4548, |
|
"theoretical_loss": 3.484262835484011, |
|
"tokens_seen": 1664614400 |
|
}, |
|
{ |
|
"epoch": 0.6, |
|
"learning_rate": 0.0002042864573482126, |
|
"loss": 2.5052, |
|
"theoretical_loss": 3.483670858895815, |
|
"tokens_seen": 1667891200 |
|
}, |
|
{ |
|
"epoch": 0.6, |
|
"learning_rate": 0.00020369538490637413, |
|
"loss": 2.4935, |
|
"theoretical_loss": 3.4830803691030683, |
|
"tokens_seen": 1671168000 |
|
}, |
|
{ |
|
"epoch": 0.6, |
|
"learning_rate": 0.00020310431246453565, |
|
"loss": 2.4934, |
|
"theoretical_loss": 3.4824913594672458, |
|
"tokens_seen": 1674444800 |
|
}, |
|
{ |
|
"epoch": 0.6, |
|
"learning_rate": 0.0002025132400226972, |
|
"loss": 2.4525, |
|
"theoretical_loss": 3.481903823392381, |
|
"tokens_seen": 1677721600 |
|
}, |
|
{ |
|
"epoch": 0.6, |
|
"learning_rate": 0.00020192216758085872, |
|
"loss": 2.4616, |
|
"theoretical_loss": 3.4813177543247096, |
|
"tokens_seen": 1680998400 |
|
}, |
|
{ |
|
"epoch": 0.6, |
|
"learning_rate": 0.00020133109513902024, |
|
"loss": 2.455, |
|
"theoretical_loss": 3.480733145752319, |
|
"tokens_seen": 1684275200 |
|
}, |
|
{ |
|
"epoch": 0.6, |
|
"learning_rate": 0.00020074002269718179, |
|
"loss": 2.4402, |
|
"theoretical_loss": 3.4801499912048, |
|
"tokens_seen": 1687552000 |
|
}, |
|
{ |
|
"epoch": 0.6, |
|
"learning_rate": 0.0002001489502553433, |
|
"loss": 2.4687, |
|
"theoretical_loss": 3.4795682842529, |
|
"tokens_seen": 1690828800 |
|
}, |
|
{ |
|
"epoch": 0.61, |
|
"learning_rate": 0.00019955787781350483, |
|
"loss": 2.4772, |
|
"theoretical_loss": 3.4789880185081863, |
|
"tokens_seen": 1694105600 |
|
}, |
|
{ |
|
"epoch": 0.61, |
|
"learning_rate": 0.00019896680537166638, |
|
"loss": 2.5039, |
|
"theoretical_loss": 3.4784091876227046, |
|
"tokens_seen": 1697382400 |
|
}, |
|
{ |
|
"epoch": 0.61, |
|
"learning_rate": 0.0001983757329298279, |
|
"loss": 2.4945, |
|
"theoretical_loss": 3.4778317852886476, |
|
"tokens_seen": 1700659200 |
|
}, |
|
{ |
|
"epoch": 0.61, |
|
"learning_rate": 0.00019778466048798942, |
|
"loss": 2.5114, |
|
"theoretical_loss": 3.4772558052380216, |
|
"tokens_seen": 1703936000 |
|
}, |
|
{ |
|
"epoch": 0.61, |
|
"learning_rate": 0.00019719358804615094, |
|
"loss": 2.4591, |
|
"theoretical_loss": 3.476681241242323, |
|
"tokens_seen": 1707212800 |
|
}, |
|
{ |
|
"epoch": 0.61, |
|
"learning_rate": 0.00019660251560431246, |
|
"loss": 2.457, |
|
"theoretical_loss": 3.4761080871122116, |
|
"tokens_seen": 1710489600 |
|
}, |
|
{ |
|
"epoch": 0.61, |
|
"learning_rate": 0.00019601144316247398, |
|
"loss": 2.4321, |
|
"theoretical_loss": 3.4755363366971896, |
|
"tokens_seen": 1713766400 |
|
}, |
|
{ |
|
"epoch": 0.61, |
|
"learning_rate": 0.00019542037072063553, |
|
"loss": 2.4711, |
|
"theoretical_loss": 3.4749659838852858, |
|
"tokens_seen": 1717043200 |
|
}, |
|
{ |
|
"epoch": 0.61, |
|
"learning_rate": 0.00019482929827879705, |
|
"loss": 2.4388, |
|
"theoretical_loss": 3.4743970226027403, |
|
"tokens_seen": 1720320000 |
|
}, |
|
{ |
|
"epoch": 0.62, |
|
"learning_rate": 0.00019423822583695857, |
|
"loss": 2.474, |
|
"theoretical_loss": 3.4738294468136934, |
|
"tokens_seen": 1723596800 |
|
}, |
|
{ |
|
"epoch": 0.62, |
|
"learning_rate": 0.00019364715339512012, |
|
"loss": 2.4821, |
|
"theoretical_loss": 3.473263250519878, |
|
"tokens_seen": 1726873600 |
|
}, |
|
{ |
|
"epoch": 0.62, |
|
"learning_rate": 0.00019305608095328164, |
|
"loss": 2.4539, |
|
"theoretical_loss": 3.4726984277603146, |
|
"tokens_seen": 1730150400 |
|
}, |
|
{ |
|
"epoch": 0.62, |
|
"learning_rate": 0.00019246500851144316, |
|
"loss": 2.4642, |
|
"theoretical_loss": 3.472134972611009, |
|
"tokens_seen": 1733427200 |
|
}, |
|
{ |
|
"epoch": 0.62, |
|
"learning_rate": 0.00019187393606960468, |
|
"loss": 2.4656, |
|
"theoretical_loss": 3.471572879184653, |
|
"tokens_seen": 1736704000 |
|
}, |
|
{ |
|
"epoch": 0.62, |
|
"learning_rate": 0.00019128286362776623, |
|
"loss": 2.48, |
|
"theoretical_loss": 3.4710121416303306, |
|
"tokens_seen": 1739980800 |
|
}, |
|
{ |
|
"epoch": 0.62, |
|
"learning_rate": 0.00019069179118592775, |
|
"loss": 2.4936, |
|
"theoretical_loss": 3.4704527541332224, |
|
"tokens_seen": 1743257600 |
|
}, |
|
{ |
|
"epoch": 0.62, |
|
"learning_rate": 0.00019010071874408927, |
|
"loss": 2.4814, |
|
"theoretical_loss": 3.469894710914317, |
|
"tokens_seen": 1746534400 |
|
}, |
|
{ |
|
"epoch": 0.62, |
|
"learning_rate": 0.00018950964630225082, |
|
"loss": 2.4557, |
|
"theoretical_loss": 3.469338006230123, |
|
"tokens_seen": 1749811200 |
|
}, |
|
{ |
|
"epoch": 0.63, |
|
"learning_rate": 0.00018891857386041234, |
|
"loss": 2.4764, |
|
"theoretical_loss": 3.468782634372387, |
|
"tokens_seen": 1753088000 |
|
}, |
|
{ |
|
"epoch": 0.63, |
|
"learning_rate": 0.00018832750141857386, |
|
"loss": 2.4245, |
|
"theoretical_loss": 3.4682285896678087, |
|
"tokens_seen": 1756364800 |
|
}, |
|
{ |
|
"epoch": 0.63, |
|
"learning_rate": 0.0001877364289767354, |
|
"loss": 2.424, |
|
"theoretical_loss": 3.467675866477763, |
|
"tokens_seen": 1759641600 |
|
}, |
|
{ |
|
"epoch": 0.63, |
|
"learning_rate": 0.00018714535653489693, |
|
"loss": 2.4322, |
|
"theoretical_loss": 3.4671244591980264, |
|
"tokens_seen": 1762918400 |
|
}, |
|
{ |
|
"epoch": 0.63, |
|
"learning_rate": 0.00018655428409305845, |
|
"loss": 2.4741, |
|
"theoretical_loss": 3.466574362258501, |
|
"tokens_seen": 1766195200 |
|
}, |
|
{ |
|
"epoch": 0.63, |
|
"learning_rate": 0.00018596321165122, |
|
"loss": 2.444, |
|
"theoretical_loss": 3.4660255701229445, |
|
"tokens_seen": 1769472000 |
|
}, |
|
{ |
|
"epoch": 0.63, |
|
"learning_rate": 0.00018538396065821827, |
|
"loss": 2.4318, |
|
"theoretical_loss": 3.4654780772887026, |
|
"tokens_seen": 1772748800 |
|
}, |
|
{ |
|
"epoch": 0.63, |
|
"learning_rate": 0.0001847928882163798, |
|
"loss": 2.4666, |
|
"theoretical_loss": 3.464931878286443, |
|
"tokens_seen": 1776025600 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"learning_rate": 0.00018420181577454134, |
|
"loss": 2.4665, |
|
"theoretical_loss": 3.4643869676798937, |
|
"tokens_seen": 1779302400 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"learning_rate": 0.00018361074333270286, |
|
"loss": 2.4588, |
|
"theoretical_loss": 3.463843340065581, |
|
"tokens_seen": 1782579200 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"learning_rate": 0.00018301967089086438, |
|
"loss": 2.4388, |
|
"theoretical_loss": 3.4633009900725735, |
|
"tokens_seen": 1785856000 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"learning_rate": 0.00018242859844902593, |
|
"loss": 2.417, |
|
"theoretical_loss": 3.4627599123622255, |
|
"tokens_seen": 1789132800 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"learning_rate": 0.00018183752600718745, |
|
"loss": 2.4062, |
|
"theoretical_loss": 3.462220101627925, |
|
"tokens_seen": 1792409600 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"learning_rate": 0.00018124645356534897, |
|
"loss": 2.4199, |
|
"theoretical_loss": 3.4616815525948432, |
|
"tokens_seen": 1795686400 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"learning_rate": 0.00018065538112351052, |
|
"loss": 2.4419, |
|
"theoretical_loss": 3.4611442600196867, |
|
"tokens_seen": 1798963200 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"objective/train/docs_used": 945295, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.365281105041504, |
|
"objective/train/theoretical_loss": 3.460608218690451, |
|
"objective/train/tokens_used": 1822700000, |
|
"theoretical_loss": 3.460608218690451, |
|
"tokens_seen": 1802240000 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"learning_rate": 0.00018006430868167204, |
|
"loss": 2.4277, |
|
"theoretical_loss": 3.460608218690451, |
|
"tokens_seen": 1802240000 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"learning_rate": 0.00017947323623983356, |
|
"loss": 2.4436, |
|
"theoretical_loss": 3.460073423426179, |
|
"tokens_seen": 1805516800 |
|
}, |
|
{ |
|
"epoch": 0.65, |
|
"learning_rate": 0.00017888216379799508, |
|
"loss": 2.4346, |
|
"theoretical_loss": 3.4595398690767176, |
|
"tokens_seen": 1808793600 |
|
}, |
|
{ |
|
"epoch": 0.65, |
|
"learning_rate": 0.00017829109135615663, |
|
"loss": 2.4369, |
|
"theoretical_loss": 3.459007550522482, |
|
"tokens_seen": 1812070400 |
|
}, |
|
{ |
|
"epoch": 0.65, |
|
"learning_rate": 0.00017770001891431815, |
|
"loss": 2.4609, |
|
"theoretical_loss": 3.4584764626742173, |
|
"tokens_seen": 1815347200 |
|
}, |
|
{ |
|
"epoch": 0.65, |
|
"learning_rate": 0.00017710894647247967, |
|
"loss": 2.4618, |
|
"theoretical_loss": 3.4579466004727664, |
|
"tokens_seen": 1818624000 |
|
}, |
|
{ |
|
"epoch": 0.65, |
|
"learning_rate": 0.00017651787403064122, |
|
"loss": 2.494, |
|
"theoretical_loss": 3.4574179588888345, |
|
"tokens_seen": 1821900800 |
|
}, |
|
{ |
|
"epoch": 0.65, |
|
"learning_rate": 0.00017592680158880274, |
|
"loss": 2.5059, |
|
"theoretical_loss": 3.456890532922765, |
|
"tokens_seen": 1825177600 |
|
}, |
|
{ |
|
"epoch": 0.65, |
|
"learning_rate": 0.00017533572914696426, |
|
"loss": 2.4566, |
|
"theoretical_loss": 3.456364317604306, |
|
"tokens_seen": 1828454400 |
|
}, |
|
{ |
|
"epoch": 0.65, |
|
"learning_rate": 0.0001747446567051258, |
|
"loss": 2.4419, |
|
"theoretical_loss": 3.455839307992391, |
|
"tokens_seen": 1831731200 |
|
}, |
|
{ |
|
"epoch": 0.66, |
|
"learning_rate": 0.00017415358426328733, |
|
"loss": 2.4701, |
|
"theoretical_loss": 3.45531549917491, |
|
"tokens_seen": 1835008000 |
|
}, |
|
{ |
|
"epoch": 0.66, |
|
"learning_rate": 0.00017356251182144882, |
|
"loss": 2.4844, |
|
"theoretical_loss": 3.454792886268492, |
|
"tokens_seen": 1838284800 |
|
}, |
|
{ |
|
"epoch": 0.66, |
|
"learning_rate": 0.00017297143937961037, |
|
"loss": 2.4593, |
|
"theoretical_loss": 3.454271464418285, |
|
"tokens_seen": 1841561600 |
|
}, |
|
{ |
|
"epoch": 0.66, |
|
"learning_rate": 0.0001723803669377719, |
|
"loss": 2.4483, |
|
"theoretical_loss": 3.4537512287977403, |
|
"tokens_seen": 1844838400 |
|
}, |
|
{ |
|
"epoch": 0.66, |
|
"learning_rate": 0.0001717892944959334, |
|
"loss": 2.4506, |
|
"theoretical_loss": 3.4532321746083956, |
|
"tokens_seen": 1848115200 |
|
}, |
|
{ |
|
"epoch": 0.66, |
|
"learning_rate": 0.00017119822205409496, |
|
"loss": 2.4313, |
|
"theoretical_loss": 3.4527142970796625, |
|
"tokens_seen": 1851392000 |
|
}, |
|
{ |
|
"epoch": 0.66, |
|
"learning_rate": 0.00017060714961225648, |
|
"loss": 2.49, |
|
"theoretical_loss": 3.4521975914686163, |
|
"tokens_seen": 1854668800 |
|
}, |
|
{ |
|
"epoch": 0.66, |
|
"learning_rate": 0.000170016077170418, |
|
"loss": 2.4758, |
|
"theoretical_loss": 3.4516820530597867, |
|
"tokens_seen": 1857945600 |
|
}, |
|
{ |
|
"epoch": 0.66, |
|
"learning_rate": 0.00016942500472857955, |
|
"loss": 2.4362, |
|
"theoretical_loss": 3.4511676771649524, |
|
"tokens_seen": 1861222400 |
|
}, |
|
{ |
|
"epoch": 0.67, |
|
"learning_rate": 0.00016883393228674107, |
|
"loss": 2.4684, |
|
"theoretical_loss": 3.450654459122932, |
|
"tokens_seen": 1864499200 |
|
}, |
|
{ |
|
"epoch": 0.67, |
|
"learning_rate": 0.0001682428598449026, |
|
"loss": 2.4891, |
|
"theoretical_loss": 3.4501423942993843, |
|
"tokens_seen": 1867776000 |
|
}, |
|
{ |
|
"epoch": 0.67, |
|
"learning_rate": 0.0001676517874030641, |
|
"loss": 2.4823, |
|
"theoretical_loss": 3.449631478086607, |
|
"tokens_seen": 1871052800 |
|
}, |
|
{ |
|
"epoch": 0.67, |
|
"learning_rate": 0.00016706071496122566, |
|
"loss": 2.4885, |
|
"theoretical_loss": 3.449121705903335, |
|
"tokens_seen": 1874329600 |
|
}, |
|
{ |
|
"epoch": 0.67, |
|
"learning_rate": 0.00016646964251938718, |
|
"loss": 2.4589, |
|
"theoretical_loss": 3.448613073194544, |
|
"tokens_seen": 1877606400 |
|
}, |
|
{ |
|
"epoch": 0.67, |
|
"learning_rate": 0.0001658785700775487, |
|
"loss": 2.4636, |
|
"theoretical_loss": 3.4481055754312564, |
|
"tokens_seen": 1880883200 |
|
}, |
|
{ |
|
"epoch": 0.67, |
|
"learning_rate": 0.00016528749763571025, |
|
"loss": 2.4867, |
|
"theoretical_loss": 3.4475992081103435, |
|
"tokens_seen": 1884160000 |
|
}, |
|
{ |
|
"epoch": 0.67, |
|
"learning_rate": 0.00016469642519387177, |
|
"loss": 2.519, |
|
"theoretical_loss": 3.4470939667543377, |
|
"tokens_seen": 1887436800 |
|
}, |
|
{ |
|
"epoch": 0.68, |
|
"learning_rate": 0.0001641053527520333, |
|
"loss": 2.4916, |
|
"theoretical_loss": 3.4465898469112366, |
|
"tokens_seen": 1890713600 |
|
}, |
|
{ |
|
"epoch": 0.68, |
|
"learning_rate": 0.00016351428031019484, |
|
"loss": 2.4705, |
|
"theoretical_loss": 3.4460868441543218, |
|
"tokens_seen": 1893990400 |
|
}, |
|
{ |
|
"epoch": 0.68, |
|
"learning_rate": 0.00016292320786835636, |
|
"loss": 2.4589, |
|
"theoretical_loss": 3.445584954081964, |
|
"tokens_seen": 1897267200 |
|
}, |
|
{ |
|
"epoch": 0.68, |
|
"learning_rate": 0.00016233213542651788, |
|
"loss": 2.4901, |
|
"theoretical_loss": 3.4450841723174426, |
|
"tokens_seen": 1900544000 |
|
}, |
|
{ |
|
"epoch": 0.68, |
|
"learning_rate": 0.00016174106298467942, |
|
"loss": 2.4879, |
|
"theoretical_loss": 3.444584494508761, |
|
"tokens_seen": 1903820800 |
|
}, |
|
{ |
|
"epoch": 0.68, |
|
"learning_rate": 0.00016114999054284095, |
|
"loss": 2.4446, |
|
"theoretical_loss": 3.444085916328465, |
|
"tokens_seen": 1907097600 |
|
}, |
|
{ |
|
"epoch": 0.68, |
|
"learning_rate": 0.00016055891810100247, |
|
"loss": 2.4553, |
|
"theoretical_loss": 3.443588433473461, |
|
"tokens_seen": 1910374400 |
|
}, |
|
{ |
|
"epoch": 0.68, |
|
"learning_rate": 0.00015996784565916401, |
|
"loss": 2.4802, |
|
"theoretical_loss": 3.4430920416648396, |
|
"tokens_seen": 1913651200 |
|
}, |
|
{ |
|
"epoch": 0.68, |
|
"learning_rate": 0.0001593767732173255, |
|
"loss": 2.4771, |
|
"theoretical_loss": 3.4425967366476957, |
|
"tokens_seen": 1916928000 |
|
}, |
|
{ |
|
"epoch": 0.69, |
|
"learning_rate": 0.00015878570077548703, |
|
"loss": 2.4596, |
|
"theoretical_loss": 3.442102514190956, |
|
"tokens_seen": 1920204800 |
|
}, |
|
{ |
|
"epoch": 0.69, |
|
"learning_rate": 0.00015819462833364858, |
|
"loss": 2.44, |
|
"theoretical_loss": 3.4416093700872032, |
|
"tokens_seen": 1923481600 |
|
}, |
|
{ |
|
"epoch": 0.69, |
|
"learning_rate": 0.0001576035558918101, |
|
"loss": 2.435, |
|
"theoretical_loss": 3.441117300152505, |
|
"tokens_seen": 1926758400 |
|
}, |
|
{ |
|
"epoch": 0.69, |
|
"learning_rate": 0.00015701248344997162, |
|
"loss": 2.4455, |
|
"theoretical_loss": 3.440626300226242, |
|
"tokens_seen": 1930035200 |
|
}, |
|
{ |
|
"epoch": 0.69, |
|
"learning_rate": 0.00015642141100813314, |
|
"loss": 2.4604, |
|
"theoretical_loss": 3.4401363661709397, |
|
"tokens_seen": 1933312000 |
|
}, |
|
{ |
|
"epoch": 0.69, |
|
"learning_rate": 0.0001558303385662947, |
|
"loss": 2.4326, |
|
"theoretical_loss": 3.4396474938720996, |
|
"tokens_seen": 1936588800 |
|
}, |
|
{ |
|
"epoch": 0.69, |
|
"learning_rate": 0.0001552392661244562, |
|
"loss": 2.4513, |
|
"theoretical_loss": 3.439159679238035, |
|
"tokens_seen": 1939865600 |
|
}, |
|
{ |
|
"epoch": 0.69, |
|
"learning_rate": 0.0001546600151314545, |
|
"loss": 2.4745, |
|
"theoretical_loss": 3.4386729181997024, |
|
"tokens_seen": 1943142400 |
|
}, |
|
{ |
|
"epoch": 0.7, |
|
"learning_rate": 0.0001540807641384528, |
|
"loss": 2.4564, |
|
"theoretical_loss": 3.438187206710543, |
|
"tokens_seen": 1946419200 |
|
}, |
|
{ |
|
"epoch": 0.7, |
|
"learning_rate": 0.00015348969169661433, |
|
"loss": 2.404, |
|
"theoretical_loss": 3.437702540746318, |
|
"tokens_seen": 1949696000 |
|
}, |
|
{ |
|
"epoch": 0.7, |
|
"learning_rate": 0.00015289861925477585, |
|
"loss": 2.4208, |
|
"theoretical_loss": 3.437218916304949, |
|
"tokens_seen": 1952972800 |
|
}, |
|
{ |
|
"epoch": 0.7, |
|
"learning_rate": 0.0001523075468129374, |
|
"loss": 2.4347, |
|
"theoretical_loss": 3.4367363294063575, |
|
"tokens_seen": 1956249600 |
|
}, |
|
{ |
|
"epoch": 0.7, |
|
"learning_rate": 0.00015171647437109892, |
|
"loss": 2.3931, |
|
"theoretical_loss": 3.4362547760923103, |
|
"tokens_seen": 1959526400 |
|
}, |
|
{ |
|
"epoch": 0.7, |
|
"learning_rate": 0.00015112540192926044, |
|
"loss": 2.4392, |
|
"theoretical_loss": 3.4357742524262616, |
|
"tokens_seen": 1962803200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.6836628981108827, |
|
"debugging/distinct-1-grams": 0.7616626811767836, |
|
"debugging/distinct-2-grams": 0.9476970777635669, |
|
"debugging/entropy-1-grams": 6.3691534254577125, |
|
"debugging/entropy-2-grams": 7.393988761324881, |
|
"debugging/length": 633.6296296296297, |
|
"debugging/num_segments": 27, |
|
"epoch": 0.7, |
|
"objective/train/docs_used": 1029668, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.7217116355895996, |
|
"objective/train/theoretical_loss": 3.4352947544931967, |
|
"objective/train/tokens_used": 1986540000, |
|
"theoretical_loss": 3.4352947544931967, |
|
"tokens_seen": 1966080000 |
|
}, |
|
{ |
|
"epoch": 0.7, |
|
"learning_rate": 0.00015053432948742198, |
|
"loss": 2.4542, |
|
"theoretical_loss": 3.4352947544931967, |
|
"tokens_seen": 1966080000 |
|
}, |
|
{ |
|
"epoch": 0.7, |
|
"learning_rate": 0.0001499432570455835, |
|
"loss": 2.4273, |
|
"theoretical_loss": 3.4348162783994827, |
|
"tokens_seen": 1969356800 |
|
}, |
|
{ |
|
"epoch": 0.7, |
|
"learning_rate": 0.00014935218460374503, |
|
"loss": 2.4366, |
|
"theoretical_loss": 3.4343388202727114, |
|
"tokens_seen": 1972633600 |
|
}, |
|
{ |
|
"epoch": 0.71, |
|
"learning_rate": 0.00014876111216190657, |
|
"loss": 2.4107, |
|
"theoretical_loss": 3.4338623762615526, |
|
"tokens_seen": 1975910400 |
|
}, |
|
{ |
|
"epoch": 0.71, |
|
"learning_rate": 0.0001481700397200681, |
|
"loss": 2.4322, |
|
"theoretical_loss": 3.4333869425356025, |
|
"tokens_seen": 1979187200 |
|
}, |
|
{ |
|
"epoch": 0.71, |
|
"learning_rate": 0.00014757896727822962, |
|
"loss": 2.4118, |
|
"theoretical_loss": 3.4329125152852367, |
|
"tokens_seen": 1982464000 |
|
}, |
|
{ |
|
"epoch": 0.71, |
|
"learning_rate": 0.00014698789483639116, |
|
"loss": 2.4514, |
|
"theoretical_loss": 3.4324390907214624, |
|
"tokens_seen": 1985740800 |
|
}, |
|
{ |
|
"epoch": 0.71, |
|
"learning_rate": 0.00014639682239455268, |
|
"loss": 2.4395, |
|
"theoretical_loss": 3.431966665075773, |
|
"tokens_seen": 1989017600 |
|
}, |
|
{ |
|
"epoch": 0.71, |
|
"learning_rate": 0.0001458057499527142, |
|
"loss": 2.4433, |
|
"theoretical_loss": 3.4314952346000047, |
|
"tokens_seen": 1992294400 |
|
}, |
|
{ |
|
"epoch": 0.71, |
|
"learning_rate": 0.00014521467751087575, |
|
"loss": 2.4814, |
|
"theoretical_loss": 3.4310247955661923, |
|
"tokens_seen": 1995571200 |
|
}, |
|
{ |
|
"epoch": 0.71, |
|
"learning_rate": 0.00014462360506903727, |
|
"loss": 2.5063, |
|
"theoretical_loss": 3.430555344266428, |
|
"tokens_seen": 1998848000 |
|
}, |
|
{ |
|
"epoch": 0.72, |
|
"learning_rate": 0.0001440325326271988, |
|
"loss": 2.5024, |
|
"theoretical_loss": 3.4300868770127213, |
|
"tokens_seen": 2002124800 |
|
}, |
|
{ |
|
"epoch": 0.72, |
|
"learning_rate": 0.00014344146018536034, |
|
"loss": 2.4791, |
|
"theoretical_loss": 3.4296193901368577, |
|
"tokens_seen": 2005401600 |
|
}, |
|
{ |
|
"epoch": 0.72, |
|
"learning_rate": 0.00014285038774352186, |
|
"loss": 2.5186, |
|
"theoretical_loss": 3.429152879990263, |
|
"tokens_seen": 2008678400 |
|
}, |
|
{ |
|
"epoch": 0.72, |
|
"learning_rate": 0.00014225931530168338, |
|
"loss": 2.4783, |
|
"theoretical_loss": 3.428687342943866, |
|
"tokens_seen": 2011955200 |
|
}, |
|
{ |
|
"epoch": 0.72, |
|
"learning_rate": 0.0001416682428598449, |
|
"loss": 2.472, |
|
"theoretical_loss": 3.428222775387959, |
|
"tokens_seen": 2015232000 |
|
}, |
|
{ |
|
"epoch": 0.72, |
|
"learning_rate": 0.00014107717041800645, |
|
"loss": 2.4656, |
|
"theoretical_loss": 3.4277591737320696, |
|
"tokens_seen": 2018508800 |
|
}, |
|
{ |
|
"epoch": 0.72, |
|
"learning_rate": 0.00014048609797616795, |
|
"loss": 2.444, |
|
"theoretical_loss": 3.4272965344048214, |
|
"tokens_seen": 2021785600 |
|
}, |
|
{ |
|
"epoch": 0.72, |
|
"learning_rate": 0.00013989502553432947, |
|
"loss": 2.4369, |
|
"theoretical_loss": 3.426834853853804, |
|
"tokens_seen": 2025062400 |
|
}, |
|
{ |
|
"epoch": 0.72, |
|
"learning_rate": 0.00013930395309249102, |
|
"loss": 2.3802, |
|
"theoretical_loss": 3.4263741285454428, |
|
"tokens_seen": 2028339200 |
|
}, |
|
{ |
|
"epoch": 0.73, |
|
"learning_rate": 0.00013871288065065254, |
|
"loss": 2.3818, |
|
"theoretical_loss": 3.4259143549648674, |
|
"tokens_seen": 2031616000 |
|
}, |
|
{ |
|
"epoch": 0.73, |
|
"learning_rate": 0.00013812180820881406, |
|
"loss": 2.4045, |
|
"theoretical_loss": 3.4254555296157827, |
|
"tokens_seen": 2034892800 |
|
}, |
|
{ |
|
"epoch": 0.73, |
|
"learning_rate": 0.0001375307357669756, |
|
"loss": 2.4114, |
|
"theoretical_loss": 3.424997649020341, |
|
"tokens_seen": 2038169600 |
|
}, |
|
{ |
|
"epoch": 0.73, |
|
"learning_rate": 0.00013693966332513713, |
|
"loss": 2.4215, |
|
"theoretical_loss": 3.4245407097190173, |
|
"tokens_seen": 2041446400 |
|
}, |
|
{ |
|
"epoch": 0.73, |
|
"learning_rate": 0.00013634859088329865, |
|
"loss": 2.4328, |
|
"theoretical_loss": 3.4240847082704793, |
|
"tokens_seen": 2044723200 |
|
}, |
|
{ |
|
"epoch": 0.73, |
|
"learning_rate": 0.0001357575184414602, |
|
"loss": 2.4254, |
|
"theoretical_loss": 3.423629641251467, |
|
"tokens_seen": 2048000000 |
|
}, |
|
{ |
|
"epoch": 0.73, |
|
"learning_rate": 0.00013516644599962171, |
|
"loss": 2.4395, |
|
"theoretical_loss": 3.4231755052566673, |
|
"tokens_seen": 2051276800 |
|
}, |
|
{ |
|
"epoch": 0.73, |
|
"learning_rate": 0.00013457537355778324, |
|
"loss": 2.3617, |
|
"theoretical_loss": 3.4227222968985904, |
|
"tokens_seen": 2054553600 |
|
}, |
|
{ |
|
"epoch": 0.73, |
|
"learning_rate": 0.00013398430111594478, |
|
"loss": 2.3928, |
|
"theoretical_loss": 3.4222700128074495, |
|
"tokens_seen": 2057830400 |
|
}, |
|
{ |
|
"epoch": 0.74, |
|
"learning_rate": 0.0001333932286741063, |
|
"loss": 2.391, |
|
"theoretical_loss": 3.4218186496310414, |
|
"tokens_seen": 2061107200 |
|
}, |
|
{ |
|
"epoch": 0.74, |
|
"learning_rate": 0.00013280215623226783, |
|
"loss": 2.379, |
|
"theoretical_loss": 3.421368204034624, |
|
"tokens_seen": 2064384000 |
|
}, |
|
{ |
|
"epoch": 0.74, |
|
"learning_rate": 0.00013221108379042937, |
|
"loss": 2.404, |
|
"theoretical_loss": 3.4209186727008, |
|
"tokens_seen": 2067660800 |
|
}, |
|
{ |
|
"epoch": 0.74, |
|
"learning_rate": 0.0001316200113485909, |
|
"loss": 2.4566, |
|
"theoretical_loss": 3.4204700523293985, |
|
"tokens_seen": 2070937600 |
|
}, |
|
{ |
|
"epoch": 0.74, |
|
"learning_rate": 0.00013102893890675241, |
|
"loss": 2.4358, |
|
"theoretical_loss": 3.42002233963736, |
|
"tokens_seen": 2074214400 |
|
}, |
|
{ |
|
"epoch": 0.74, |
|
"learning_rate": 0.00013043786646491394, |
|
"loss": 2.4708, |
|
"theoretical_loss": 3.4195755313586176, |
|
"tokens_seen": 2077491200 |
|
}, |
|
{ |
|
"epoch": 0.74, |
|
"learning_rate": 0.00012984679402307548, |
|
"loss": 2.4677, |
|
"theoretical_loss": 3.419129624243986, |
|
"tokens_seen": 2080768000 |
|
}, |
|
{ |
|
"epoch": 0.74, |
|
"learning_rate": 0.000129255721581237, |
|
"loss": 2.4889, |
|
"theoretical_loss": 3.4186846150610464, |
|
"tokens_seen": 2084044800 |
|
}, |
|
{ |
|
"epoch": 0.75, |
|
"learning_rate": 0.00012866464913939852, |
|
"loss": 2.4554, |
|
"theoretical_loss": 3.418240500594031, |
|
"tokens_seen": 2087321600 |
|
}, |
|
{ |
|
"epoch": 0.75, |
|
"learning_rate": 0.00012807357669756007, |
|
"loss": 2.4883, |
|
"theoretical_loss": 3.417797277643717, |
|
"tokens_seen": 2090598400 |
|
}, |
|
{ |
|
"epoch": 0.75, |
|
"learning_rate": 0.0001274825042557216, |
|
"loss": 2.4779, |
|
"theoretical_loss": 3.417354943027311, |
|
"tokens_seen": 2093875200 |
|
}, |
|
{ |
|
"epoch": 0.75, |
|
"learning_rate": 0.00012689143181388311, |
|
"loss": 2.4636, |
|
"theoretical_loss": 3.41691349357834, |
|
"tokens_seen": 2097152000 |
|
}, |
|
{ |
|
"epoch": 0.75, |
|
"learning_rate": 0.00012630035937204464, |
|
"loss": 2.4565, |
|
"theoretical_loss": 3.416472926146545, |
|
"tokens_seen": 2100428800 |
|
}, |
|
{ |
|
"epoch": 0.75, |
|
"learning_rate": 0.00012570928693020616, |
|
"loss": 2.4487, |
|
"theoretical_loss": 3.4160332375977687, |
|
"tokens_seen": 2103705600 |
|
}, |
|
{ |
|
"epoch": 0.75, |
|
"learning_rate": 0.00012511821448836768, |
|
"loss": 2.4188, |
|
"theoretical_loss": 3.4155944248138512, |
|
"tokens_seen": 2106982400 |
|
}, |
|
{ |
|
"epoch": 0.75, |
|
"learning_rate": 0.00012452714204652922, |
|
"loss": 2.4229, |
|
"theoretical_loss": 3.415156484692524, |
|
"tokens_seen": 2110259200 |
|
}, |
|
{ |
|
"epoch": 0.75, |
|
"learning_rate": 0.00012393606960469075, |
|
"loss": 2.4877, |
|
"theoretical_loss": 3.4147194141473003, |
|
"tokens_seen": 2113536000 |
|
}, |
|
{ |
|
"epoch": 0.76, |
|
"learning_rate": 0.0001233449971628523, |
|
"loss": 2.4388, |
|
"theoretical_loss": 3.414283210107377, |
|
"tokens_seen": 2116812800 |
|
}, |
|
{ |
|
"epoch": 0.76, |
|
"learning_rate": 0.00012275392472101381, |
|
"loss": 2.4381, |
|
"theoretical_loss": 3.4138478695175243, |
|
"tokens_seen": 2120089600 |
|
}, |
|
{ |
|
"epoch": 0.76, |
|
"learning_rate": 0.00012216285227917533, |
|
"loss": 2.4608, |
|
"theoretical_loss": 3.413413389337988, |
|
"tokens_seen": 2123366400 |
|
}, |
|
{ |
|
"epoch": 0.76, |
|
"learning_rate": 0.00012157177983733686, |
|
"loss": 2.4231, |
|
"theoretical_loss": 3.4129797665443826, |
|
"tokens_seen": 2126643200 |
|
}, |
|
{ |
|
"epoch": 0.76, |
|
"objective/train/docs_used": 1116325, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.264505386352539, |
|
"objective/train/theoretical_loss": 3.4125469981275955, |
|
"objective/train/tokens_used": 2150380000, |
|
"theoretical_loss": 3.4125469981275955, |
|
"tokens_seen": 2129920000 |
|
}, |
|
{ |
|
"epoch": 0.76, |
|
"learning_rate": 0.00012098070739549839, |
|
"loss": 2.4111, |
|
"theoretical_loss": 3.4125469981275955, |
|
"tokens_seen": 2129920000 |
|
}, |
|
{ |
|
"epoch": 0.76, |
|
"learning_rate": 0.00012038963495365992, |
|
"loss": 2.3977, |
|
"theoretical_loss": 3.4121150810936816, |
|
"tokens_seen": 2133196800 |
|
}, |
|
{ |
|
"epoch": 0.76, |
|
"learning_rate": 0.00011979856251182145, |
|
"loss": 2.4035, |
|
"theoretical_loss": 3.411684012463766, |
|
"tokens_seen": 2136473600 |
|
}, |
|
{ |
|
"epoch": 0.76, |
|
"learning_rate": 0.00011920749006998298, |
|
"loss": 2.4057, |
|
"theoretical_loss": 3.411253789273946, |
|
"tokens_seen": 2139750400 |
|
}, |
|
{ |
|
"epoch": 0.77, |
|
"learning_rate": 0.00011861641762814451, |
|
"loss": 2.4384, |
|
"theoretical_loss": 3.4108244085751904, |
|
"tokens_seen": 2143027200 |
|
}, |
|
{ |
|
"epoch": 0.77, |
|
"learning_rate": 0.00011802534518630603, |
|
"loss": 2.4642, |
|
"theoretical_loss": 3.410395867433245, |
|
"tokens_seen": 2146304000 |
|
}, |
|
{ |
|
"epoch": 0.77, |
|
"learning_rate": 0.00011743427274446757, |
|
"loss": 2.4547, |
|
"theoretical_loss": 3.409968162928534, |
|
"tokens_seen": 2149580800 |
|
}, |
|
{ |
|
"epoch": 0.77, |
|
"learning_rate": 0.0001168432003026291, |
|
"loss": 2.4689, |
|
"theoretical_loss": 3.4095412921560655, |
|
"tokens_seen": 2152857600 |
|
}, |
|
{ |
|
"epoch": 0.77, |
|
"learning_rate": 0.00011625212786079062, |
|
"loss": 2.4921, |
|
"theoretical_loss": 3.4091152522253374, |
|
"tokens_seen": 2156134400 |
|
}, |
|
{ |
|
"epoch": 0.77, |
|
"learning_rate": 0.00011566105541895216, |
|
"loss": 2.4892, |
|
"theoretical_loss": 3.40869004026024, |
|
"tokens_seen": 2159411200 |
|
}, |
|
{ |
|
"epoch": 0.77, |
|
"learning_rate": 0.00011506998297711367, |
|
"loss": 2.4749, |
|
"theoretical_loss": 3.408265653398967, |
|
"tokens_seen": 2162688000 |
|
}, |
|
{ |
|
"epoch": 0.77, |
|
"learning_rate": 0.0001144789105352752, |
|
"loss": 2.451, |
|
"theoretical_loss": 3.40784208879392, |
|
"tokens_seen": 2165964800 |
|
}, |
|
{ |
|
"epoch": 0.77, |
|
"learning_rate": 0.00011388783809343673, |
|
"loss": 2.4631, |
|
"theoretical_loss": 3.407419343611616, |
|
"tokens_seen": 2169241600 |
|
}, |
|
{ |
|
"epoch": 0.78, |
|
"learning_rate": 0.00011329676565159826, |
|
"loss": 2.4487, |
|
"theoretical_loss": 3.406997415032599, |
|
"tokens_seen": 2172518400 |
|
}, |
|
{ |
|
"epoch": 0.78, |
|
"learning_rate": 0.00011270569320975979, |
|
"loss": 2.4736, |
|
"theoretical_loss": 3.406576300251348, |
|
"tokens_seen": 2175795200 |
|
}, |
|
{ |
|
"epoch": 0.78, |
|
"learning_rate": 0.00011211462076792132, |
|
"loss": 2.4502, |
|
"theoretical_loss": 3.4061559964761856, |
|
"tokens_seen": 2179072000 |
|
}, |
|
{ |
|
"epoch": 0.78, |
|
"learning_rate": 0.00011152354832608284, |
|
"loss": 2.4325, |
|
"theoretical_loss": 3.4057365009291924, |
|
"tokens_seen": 2182348800 |
|
}, |
|
{ |
|
"epoch": 0.78, |
|
"learning_rate": 0.00011093247588424438, |
|
"loss": 2.429, |
|
"theoretical_loss": 3.405317810846116, |
|
"tokens_seen": 2185625600 |
|
}, |
|
{ |
|
"epoch": 0.78, |
|
"learning_rate": 0.00011034140344240591, |
|
"loss": 2.4029, |
|
"theoretical_loss": 3.404899923476285, |
|
"tokens_seen": 2188902400 |
|
}, |
|
{ |
|
"epoch": 0.78, |
|
"learning_rate": 0.00010975033100056743, |
|
"loss": 2.4063, |
|
"theoretical_loss": 3.404482836082521, |
|
"tokens_seen": 2192179200 |
|
}, |
|
{ |
|
"epoch": 0.78, |
|
"learning_rate": 0.00010915925855872897, |
|
"loss": 2.3936, |
|
"theoretical_loss": 3.4040665459410517, |
|
"tokens_seen": 2195456000 |
|
}, |
|
{ |
|
"epoch": 0.79, |
|
"learning_rate": 0.00010856818611689049, |
|
"loss": 2.3936, |
|
"theoretical_loss": 3.4036510503414283, |
|
"tokens_seen": 2198732800 |
|
}, |
|
{ |
|
"epoch": 0.79, |
|
"learning_rate": 0.00010797711367505201, |
|
"loss": 2.3916, |
|
"theoretical_loss": 3.4032363465864375, |
|
"tokens_seen": 2202009600 |
|
}, |
|
{ |
|
"epoch": 0.79, |
|
"learning_rate": 0.00010738604123321354, |
|
"loss": 2.3999, |
|
"theoretical_loss": 3.402822431992018, |
|
"tokens_seen": 2205286400 |
|
}, |
|
{ |
|
"epoch": 0.79, |
|
"learning_rate": 0.00010679496879137507, |
|
"loss": 2.3894, |
|
"theoretical_loss": 3.402409303887179, |
|
"tokens_seen": 2208563200 |
|
}, |
|
{ |
|
"epoch": 0.79, |
|
"learning_rate": 0.0001062038963495366, |
|
"loss": 2.3971, |
|
"theoretical_loss": 3.401996959613914, |
|
"tokens_seen": 2211840000 |
|
}, |
|
{ |
|
"epoch": 0.79, |
|
"learning_rate": 0.00010561282390769813, |
|
"loss": 2.38, |
|
"theoretical_loss": 3.4015853965271217, |
|
"tokens_seen": 2215116800 |
|
}, |
|
{ |
|
"epoch": 0.79, |
|
"learning_rate": 0.00010502175146585965, |
|
"loss": 2.3588, |
|
"theoretical_loss": 3.4011746119945214, |
|
"tokens_seen": 2218393600 |
|
}, |
|
{ |
|
"epoch": 0.79, |
|
"learning_rate": 0.00010443067902402119, |
|
"loss": 2.3617, |
|
"theoretical_loss": 3.4007646033965746, |
|
"tokens_seen": 2221670400 |
|
}, |
|
{ |
|
"epoch": 0.79, |
|
"learning_rate": 0.00010383960658218271, |
|
"loss": 2.3746, |
|
"theoretical_loss": 3.4003553681264025, |
|
"tokens_seen": 2224947200 |
|
}, |
|
{ |
|
"epoch": 0.8, |
|
"learning_rate": 0.00010324853414034424, |
|
"loss": 2.4094, |
|
"theoretical_loss": 3.399946903589709, |
|
"tokens_seen": 2228224000 |
|
}, |
|
{ |
|
"epoch": 0.8, |
|
"learning_rate": 0.00010265746169850578, |
|
"loss": 2.4003, |
|
"theoretical_loss": 3.3995392072046986, |
|
"tokens_seen": 2231500800 |
|
}, |
|
{ |
|
"epoch": 0.8, |
|
"learning_rate": 0.0001020663892566673, |
|
"loss": 2.3784, |
|
"theoretical_loss": 3.3991322764019993, |
|
"tokens_seen": 2234777600 |
|
}, |
|
{ |
|
"epoch": 0.8, |
|
"learning_rate": 0.00010147531681482883, |
|
"loss": 2.3991, |
|
"theoretical_loss": 3.398726108624585, |
|
"tokens_seen": 2238054400 |
|
}, |
|
{ |
|
"epoch": 0.8, |
|
"learning_rate": 0.00010088424437299035, |
|
"loss": 2.4118, |
|
"theoretical_loss": 3.3983207013276977, |
|
"tokens_seen": 2241331200 |
|
}, |
|
{ |
|
"epoch": 0.8, |
|
"learning_rate": 0.00010029317193115188, |
|
"loss": 2.4643, |
|
"theoretical_loss": 3.3979160519787706, |
|
"tokens_seen": 2244608000 |
|
}, |
|
{ |
|
"epoch": 0.8, |
|
"learning_rate": 9.970209948931341e-05, |
|
"loss": 2.4424, |
|
"theoretical_loss": 3.397512158057353, |
|
"tokens_seen": 2247884800 |
|
}, |
|
{ |
|
"epoch": 0.8, |
|
"learning_rate": 9.911102704747494e-05, |
|
"loss": 2.3987, |
|
"theoretical_loss": 3.3971090170550333, |
|
"tokens_seen": 2251161600 |
|
}, |
|
{ |
|
"epoch": 0.81, |
|
"learning_rate": 9.851995460563646e-05, |
|
"loss": 2.3874, |
|
"theoretical_loss": 3.396706626475366, |
|
"tokens_seen": 2254438400 |
|
}, |
|
{ |
|
"epoch": 0.81, |
|
"learning_rate": 9.7928882163798e-05, |
|
"loss": 2.4175, |
|
"theoretical_loss": 3.396304983833794, |
|
"tokens_seen": 2257715200 |
|
}, |
|
{ |
|
"epoch": 0.81, |
|
"learning_rate": 9.733780972195952e-05, |
|
"loss": 2.3753, |
|
"theoretical_loss": 3.3959040866575805, |
|
"tokens_seen": 2260992000 |
|
}, |
|
{ |
|
"epoch": 0.81, |
|
"learning_rate": 9.674673728012105e-05, |
|
"loss": 2.4353, |
|
"theoretical_loss": 3.3955039324857292, |
|
"tokens_seen": 2264268800 |
|
}, |
|
{ |
|
"epoch": 0.81, |
|
"learning_rate": 9.615566483828259e-05, |
|
"loss": 2.4694, |
|
"theoretical_loss": 3.395104518868916, |
|
"tokens_seen": 2267545600 |
|
}, |
|
{ |
|
"epoch": 0.81, |
|
"learning_rate": 9.556459239644411e-05, |
|
"loss": 2.4502, |
|
"theoretical_loss": 3.3947058433694157, |
|
"tokens_seen": 2270822400 |
|
}, |
|
{ |
|
"epoch": 0.81, |
|
"learning_rate": 9.497351995460564e-05, |
|
"loss": 2.4503, |
|
"theoretical_loss": 3.39430790356103, |
|
"tokens_seen": 2274099200 |
|
}, |
|
{ |
|
"epoch": 0.81, |
|
"learning_rate": 9.438244751276718e-05, |
|
"loss": 2.4401, |
|
"theoretical_loss": 3.3939106970290163, |
|
"tokens_seen": 2277376000 |
|
}, |
|
{ |
|
"epoch": 0.81, |
|
"learning_rate": 9.379137507092869e-05, |
|
"loss": 2.427, |
|
"theoretical_loss": 3.3935142213700185, |
|
"tokens_seen": 2280652800 |
|
}, |
|
{ |
|
"epoch": 0.82, |
|
"learning_rate": 9.320030262909022e-05, |
|
"loss": 2.431, |
|
"theoretical_loss": 3.393118474191996, |
|
"tokens_seen": 2283929600 |
|
}, |
|
{ |
|
"epoch": 0.82, |
|
"learning_rate": 9.260923018725175e-05, |
|
"loss": 2.4092, |
|
"theoretical_loss": 3.392723453114155, |
|
"tokens_seen": 2287206400 |
|
}, |
|
{ |
|
"epoch": 0.82, |
|
"learning_rate": 9.201815774541327e-05, |
|
"loss": 2.4166, |
|
"theoretical_loss": 3.3923291557668778, |
|
"tokens_seen": 2290483200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.6045646500593793, |
|
"debugging/distinct-1-grams": 0.8200208525648512, |
|
"debugging/distinct-2-grams": 0.9770135273511986, |
|
"debugging/entropy-1-grams": 5.916762726693028, |
|
"debugging/entropy-2-grams": 6.785406901896722, |
|
"debugging/length": 547.1333333333333, |
|
"debugging/num_segments": 15, |
|
"epoch": 0.82, |
|
"objective/train/docs_used": 1203700, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.3816142082214355, |
|
"objective/train/theoretical_loss": 3.391935579791658, |
|
"objective/train/tokens_used": 2314220000, |
|
"theoretical_loss": 3.391935579791658, |
|
"tokens_seen": 2293760000 |
|
}, |
|
{ |
|
"epoch": 0.82, |
|
"learning_rate": 9.142708530357481e-05, |
|
"loss": 2.403, |
|
"theoretical_loss": 3.391935579791658, |
|
"tokens_seen": 2293760000 |
|
}, |
|
{ |
|
"epoch": 0.82, |
|
"learning_rate": 9.083601286173633e-05, |
|
"loss": 2.3557, |
|
"theoretical_loss": 3.3915427228410278, |
|
"tokens_seen": 2297036800 |
|
}, |
|
{ |
|
"epoch": 0.82, |
|
"learning_rate": 9.024494041989786e-05, |
|
"loss": 2.4006, |
|
"theoretical_loss": 3.391150582578495, |
|
"tokens_seen": 2300313600 |
|
}, |
|
{ |
|
"epoch": 0.82, |
|
"learning_rate": 8.96538679780594e-05, |
|
"loss": 2.4151, |
|
"theoretical_loss": 3.390759156678474, |
|
"tokens_seen": 2303590400 |
|
}, |
|
{ |
|
"epoch": 0.82, |
|
"learning_rate": 8.906279553622092e-05, |
|
"loss": 2.421, |
|
"theoretical_loss": 3.3903684428262197, |
|
"tokens_seen": 2306867200 |
|
}, |
|
{ |
|
"epoch": 0.83, |
|
"learning_rate": 8.847172309438245e-05, |
|
"loss": 2.4127, |
|
"theoretical_loss": 3.3899784387177623, |
|
"tokens_seen": 2310144000 |
|
}, |
|
{ |
|
"epoch": 0.83, |
|
"learning_rate": 8.788065065254399e-05, |
|
"loss": 2.4125, |
|
"theoretical_loss": 3.3895891420598403, |
|
"tokens_seen": 2313420800 |
|
}, |
|
{ |
|
"epoch": 0.83, |
|
"learning_rate": 8.728957821070551e-05, |
|
"loss": 2.4025, |
|
"theoretical_loss": 3.3892005505698384, |
|
"tokens_seen": 2316697600 |
|
}, |
|
{ |
|
"epoch": 0.83, |
|
"learning_rate": 8.669850576886704e-05, |
|
"loss": 2.4533, |
|
"theoretical_loss": 3.3888126619757197, |
|
"tokens_seen": 2319974400 |
|
}, |
|
{ |
|
"epoch": 0.83, |
|
"learning_rate": 8.610743332702855e-05, |
|
"loss": 2.413, |
|
"theoretical_loss": 3.388425474015966, |
|
"tokens_seen": 2323251200 |
|
}, |
|
{ |
|
"epoch": 0.83, |
|
"learning_rate": 8.551636088519008e-05, |
|
"loss": 2.3765, |
|
"theoretical_loss": 3.3880389844395093, |
|
"tokens_seen": 2326528000 |
|
}, |
|
{ |
|
"epoch": 0.83, |
|
"learning_rate": 8.492528844335162e-05, |
|
"loss": 2.3757, |
|
"theoretical_loss": 3.3876531910056733, |
|
"tokens_seen": 2329804800 |
|
}, |
|
{ |
|
"epoch": 0.83, |
|
"learning_rate": 8.433421600151314e-05, |
|
"loss": 2.4111, |
|
"theoretical_loss": 3.387268091484109, |
|
"tokens_seen": 2333081600 |
|
}, |
|
{ |
|
"epoch": 0.83, |
|
"learning_rate": 8.374314355967467e-05, |
|
"loss": 2.4184, |
|
"theoretical_loss": 3.386883683654732, |
|
"tokens_seen": 2336358400 |
|
}, |
|
{ |
|
"epoch": 0.84, |
|
"learning_rate": 8.315207111783621e-05, |
|
"loss": 2.4241, |
|
"theoretical_loss": 3.386499965307662, |
|
"tokens_seen": 2339635200 |
|
}, |
|
{ |
|
"epoch": 0.84, |
|
"learning_rate": 8.256099867599773e-05, |
|
"loss": 2.4644, |
|
"theoretical_loss": 3.3861169342431627, |
|
"tokens_seen": 2342912000 |
|
}, |
|
{ |
|
"epoch": 0.84, |
|
"learning_rate": 8.198174768299603e-05, |
|
"loss": 2.4192, |
|
"theoretical_loss": 3.3857345882715784, |
|
"tokens_seen": 2346188800 |
|
}, |
|
{ |
|
"epoch": 0.84, |
|
"learning_rate": 8.139067524115756e-05, |
|
"loss": 2.4344, |
|
"theoretical_loss": 3.3853529252132755, |
|
"tokens_seen": 2349465600 |
|
}, |
|
{ |
|
"epoch": 0.84, |
|
"learning_rate": 8.079960279931908e-05, |
|
"loss": 2.3963, |
|
"theoretical_loss": 3.384971942898583, |
|
"tokens_seen": 2352742400 |
|
}, |
|
{ |
|
"epoch": 0.84, |
|
"learning_rate": 8.020853035748062e-05, |
|
"loss": 2.3802, |
|
"theoretical_loss": 3.384591639167732, |
|
"tokens_seen": 2356019200 |
|
}, |
|
{ |
|
"epoch": 0.84, |
|
"learning_rate": 7.961745791564215e-05, |
|
"loss": 2.3893, |
|
"theoretical_loss": 3.3842120118707983, |
|
"tokens_seen": 2359296000 |
|
}, |
|
{ |
|
"epoch": 0.84, |
|
"learning_rate": 7.902638547380367e-05, |
|
"loss": 2.4057, |
|
"theoretical_loss": 3.383833058867643, |
|
"tokens_seen": 2362572800 |
|
}, |
|
{ |
|
"epoch": 0.84, |
|
"learning_rate": 7.843531303196521e-05, |
|
"loss": 2.4165, |
|
"theoretical_loss": 3.3834547780278537, |
|
"tokens_seen": 2365849600 |
|
}, |
|
{ |
|
"epoch": 0.85, |
|
"learning_rate": 7.784424059012673e-05, |
|
"loss": 2.4218, |
|
"theoretical_loss": 3.3830771672306885, |
|
"tokens_seen": 2369126400 |
|
}, |
|
{ |
|
"epoch": 0.85, |
|
"learning_rate": 7.725316814828825e-05, |
|
"loss": 2.4073, |
|
"theoretical_loss": 3.382700224365018, |
|
"tokens_seen": 2372403200 |
|
}, |
|
{ |
|
"epoch": 0.85, |
|
"learning_rate": 7.666209570644978e-05, |
|
"loss": 2.4458, |
|
"theoretical_loss": 3.382323947329268, |
|
"tokens_seen": 2375680000 |
|
}, |
|
{ |
|
"epoch": 0.85, |
|
"learning_rate": 7.60710232646113e-05, |
|
"loss": 2.44, |
|
"theoretical_loss": 3.3819483340313647, |
|
"tokens_seen": 2378956800 |
|
}, |
|
{ |
|
"epoch": 0.85, |
|
"learning_rate": 7.547995082277284e-05, |
|
"loss": 2.5253, |
|
"theoretical_loss": 3.3815733823886776, |
|
"tokens_seen": 2382233600 |
|
}, |
|
{ |
|
"epoch": 0.85, |
|
"learning_rate": 7.488887838093437e-05, |
|
"loss": 2.6199, |
|
"theoretical_loss": 3.381199090327963, |
|
"tokens_seen": 2385510400 |
|
}, |
|
{ |
|
"epoch": 0.85, |
|
"learning_rate": 7.42978059390959e-05, |
|
"loss": 2.6376, |
|
"theoretical_loss": 3.3808254557853115, |
|
"tokens_seen": 2388787200 |
|
}, |
|
{ |
|
"epoch": 0.85, |
|
"learning_rate": 7.370673349725743e-05, |
|
"loss": 2.6499, |
|
"theoretical_loss": 3.3804524767060906, |
|
"tokens_seen": 2392064000 |
|
}, |
|
{ |
|
"epoch": 0.86, |
|
"learning_rate": 7.311566105541895e-05, |
|
"loss": 2.5915, |
|
"theoretical_loss": 3.3800801510448917, |
|
"tokens_seen": 2395340800 |
|
}, |
|
{ |
|
"epoch": 0.86, |
|
"learning_rate": 7.252458861358048e-05, |
|
"loss": 2.5823, |
|
"theoretical_loss": 3.379708476765477, |
|
"tokens_seen": 2398617600 |
|
}, |
|
{ |
|
"epoch": 0.86, |
|
"learning_rate": 7.193351617174202e-05, |
|
"loss": 2.567, |
|
"theoretical_loss": 3.3793374518407235, |
|
"tokens_seen": 2401894400 |
|
}, |
|
{ |
|
"epoch": 0.86, |
|
"learning_rate": 7.134244372990354e-05, |
|
"loss": 2.544, |
|
"theoretical_loss": 3.3789670742525715, |
|
"tokens_seen": 2405171200 |
|
}, |
|
{ |
|
"epoch": 0.86, |
|
"learning_rate": 7.075137128806507e-05, |
|
"loss": 2.4954, |
|
"theoretical_loss": 3.3785973419919717, |
|
"tokens_seen": 2408448000 |
|
}, |
|
{ |
|
"epoch": 0.86, |
|
"learning_rate": 7.01602988462266e-05, |
|
"loss": 2.4908, |
|
"theoretical_loss": 3.3782282530588335, |
|
"tokens_seen": 2411724800 |
|
}, |
|
{ |
|
"epoch": 0.86, |
|
"learning_rate": 6.956922640438811e-05, |
|
"loss": 2.4519, |
|
"theoretical_loss": 3.3778598054619695, |
|
"tokens_seen": 2415001600 |
|
}, |
|
{ |
|
"epoch": 0.86, |
|
"learning_rate": 6.897815396254965e-05, |
|
"loss": 2.4621, |
|
"theoretical_loss": 3.377491997219049, |
|
"tokens_seen": 2418278400 |
|
}, |
|
{ |
|
"epoch": 0.86, |
|
"learning_rate": 6.838708152071118e-05, |
|
"loss": 2.4315, |
|
"theoretical_loss": 3.3771248263565425, |
|
"tokens_seen": 2421555200 |
|
}, |
|
{ |
|
"epoch": 0.87, |
|
"learning_rate": 6.77960090788727e-05, |
|
"loss": 2.4171, |
|
"theoretical_loss": 3.376758290909673, |
|
"tokens_seen": 2424832000 |
|
}, |
|
{ |
|
"epoch": 0.87, |
|
"learning_rate": 6.720493663703424e-05, |
|
"loss": 2.4037, |
|
"theoretical_loss": 3.3763923889223646, |
|
"tokens_seen": 2428108800 |
|
}, |
|
{ |
|
"epoch": 0.87, |
|
"learning_rate": 6.661386419519576e-05, |
|
"loss": 2.4047, |
|
"theoretical_loss": 3.3760271184471917, |
|
"tokens_seen": 2431385600 |
|
}, |
|
{ |
|
"epoch": 0.87, |
|
"learning_rate": 6.602279175335729e-05, |
|
"loss": 2.3996, |
|
"theoretical_loss": 3.3756624775453306, |
|
"tokens_seen": 2434662400 |
|
}, |
|
{ |
|
"epoch": 0.87, |
|
"learning_rate": 6.543171931151883e-05, |
|
"loss": 2.4013, |
|
"theoretical_loss": 3.3752984642865087, |
|
"tokens_seen": 2437939200 |
|
}, |
|
{ |
|
"epoch": 0.87, |
|
"learning_rate": 6.484064686968035e-05, |
|
"loss": 2.4035, |
|
"theoretical_loss": 3.374935076748957, |
|
"tokens_seen": 2441216000 |
|
}, |
|
{ |
|
"epoch": 0.87, |
|
"learning_rate": 6.424957442784188e-05, |
|
"loss": 2.4074, |
|
"theoretical_loss": 3.374572313019359, |
|
"tokens_seen": 2444492800 |
|
}, |
|
{ |
|
"epoch": 0.87, |
|
"learning_rate": 6.365850198600342e-05, |
|
"loss": 2.3941, |
|
"theoretical_loss": 3.3742101711928036, |
|
"tokens_seen": 2447769600 |
|
}, |
|
{ |
|
"epoch": 0.88, |
|
"learning_rate": 6.306742954416494e-05, |
|
"loss": 2.396, |
|
"theoretical_loss": 3.3738486493727375, |
|
"tokens_seen": 2451046400 |
|
}, |
|
{ |
|
"epoch": 0.88, |
|
"learning_rate": 6.247635710232646e-05, |
|
"loss": 2.4039, |
|
"theoretical_loss": 3.3734877456709165, |
|
"tokens_seen": 2454323200 |
|
}, |
|
{ |
|
"epoch": 0.88, |
|
"objective/train/docs_used": 1284127, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.513523578643799, |
|
"objective/train/theoretical_loss": 3.373127458207357, |
|
"objective/train/tokens_used": 2478060000, |
|
"theoretical_loss": 3.373127458207357, |
|
"tokens_seen": 2457600000 |
|
}, |
|
{ |
|
"epoch": 0.88, |
|
"learning_rate": 6.188528466048799e-05, |
|
"loss": 2.4194, |
|
"theoretical_loss": 3.373127458207357, |
|
"tokens_seen": 2457600000 |
|
}, |
|
{ |
|
"epoch": 0.88, |
|
"learning_rate": 6.129421221864951e-05, |
|
"loss": 2.387, |
|
"theoretical_loss": 3.372767785110293, |
|
"tokens_seen": 2460876800 |
|
}, |
|
{ |
|
"epoch": 0.88, |
|
"learning_rate": 6.070313977681105e-05, |
|
"loss": 2.3984, |
|
"theoretical_loss": 3.3724087245161227, |
|
"tokens_seen": 2464153600 |
|
}, |
|
{ |
|
"epoch": 0.88, |
|
"learning_rate": 6.0112067334972575e-05, |
|
"loss": 2.4228, |
|
"theoretical_loss": 3.3720502745693697, |
|
"tokens_seen": 2467430400 |
|
}, |
|
{ |
|
"epoch": 0.88, |
|
"learning_rate": 5.95209948931341e-05, |
|
"loss": 2.4133, |
|
"theoretical_loss": 3.37169243342263, |
|
"tokens_seen": 2470707200 |
|
}, |
|
{ |
|
"epoch": 0.88, |
|
"learning_rate": 5.892992245129563e-05, |
|
"loss": 2.4173, |
|
"theoretical_loss": 3.3713351992365315, |
|
"tokens_seen": 2473984000 |
|
}, |
|
{ |
|
"epoch": 0.88, |
|
"learning_rate": 5.835067145829393e-05, |
|
"loss": 2.4167, |
|
"theoretical_loss": 3.3709785701796844, |
|
"tokens_seen": 2477260800 |
|
}, |
|
{ |
|
"epoch": 0.89, |
|
"learning_rate": 5.775959901645546e-05, |
|
"loss": 2.4542, |
|
"theoretical_loss": 3.3706225444286395, |
|
"tokens_seen": 2480537600 |
|
}, |
|
{ |
|
"epoch": 0.89, |
|
"learning_rate": 5.7168526574616985e-05, |
|
"loss": 2.5331, |
|
"theoretical_loss": 3.3702671201678407, |
|
"tokens_seen": 2483814400 |
|
}, |
|
{ |
|
"epoch": 0.89, |
|
"learning_rate": 5.657745413277852e-05, |
|
"loss": 2.6007, |
|
"theoretical_loss": 3.3699122955895815, |
|
"tokens_seen": 2487091200 |
|
}, |
|
{ |
|
"epoch": 0.89, |
|
"learning_rate": 5.5986381690940047e-05, |
|
"loss": 2.5565, |
|
"theoretical_loss": 3.3695580688939617, |
|
"tokens_seen": 2490368000 |
|
}, |
|
{ |
|
"epoch": 0.89, |
|
"learning_rate": 5.539530924910157e-05, |
|
"loss": 2.5357, |
|
"theoretical_loss": 3.3692044382888415, |
|
"tokens_seen": 2493644800 |
|
}, |
|
{ |
|
"epoch": 0.89, |
|
"learning_rate": 5.4804236807263095e-05, |
|
"loss": 2.4901, |
|
"theoretical_loss": 3.368851401989799, |
|
"tokens_seen": 2496921600 |
|
}, |
|
{ |
|
"epoch": 0.89, |
|
"learning_rate": 5.421316436542463e-05, |
|
"loss": 2.4725, |
|
"theoretical_loss": 3.3684989582200884, |
|
"tokens_seen": 2500198400 |
|
}, |
|
{ |
|
"epoch": 0.89, |
|
"learning_rate": 5.362209192358616e-05, |
|
"loss": 2.4387, |
|
"theoretical_loss": 3.368147105210593, |
|
"tokens_seen": 2503475200 |
|
}, |
|
{ |
|
"epoch": 0.9, |
|
"learning_rate": 5.3031019481747684e-05, |
|
"loss": 2.4684, |
|
"theoretical_loss": 3.3677958411997873, |
|
"tokens_seen": 2506752000 |
|
}, |
|
{ |
|
"epoch": 0.9, |
|
"learning_rate": 5.243994703990921e-05, |
|
"loss": 2.4419, |
|
"theoretical_loss": 3.36744516443369, |
|
"tokens_seen": 2510028800 |
|
}, |
|
{ |
|
"epoch": 0.9, |
|
"learning_rate": 5.1848874598070746e-05, |
|
"loss": 2.4515, |
|
"theoretical_loss": 3.3670950731658253, |
|
"tokens_seen": 2513305600 |
|
}, |
|
{ |
|
"epoch": 0.9, |
|
"learning_rate": 5.125780215623227e-05, |
|
"loss": 2.4286, |
|
"theoretical_loss": 3.36674556565718, |
|
"tokens_seen": 2516582400 |
|
}, |
|
{ |
|
"epoch": 0.9, |
|
"learning_rate": 5.0666729714393795e-05, |
|
"loss": 2.3916, |
|
"theoretical_loss": 3.3663966401761614, |
|
"tokens_seen": 2519859200 |
|
}, |
|
{ |
|
"epoch": 0.9, |
|
"learning_rate": 5.007565727255532e-05, |
|
"loss": 2.4221, |
|
"theoretical_loss": 3.3660482949985555, |
|
"tokens_seen": 2523136000 |
|
}, |
|
{ |
|
"epoch": 0.9, |
|
"learning_rate": 4.9484584830716857e-05, |
|
"loss": 2.432, |
|
"theoretical_loss": 3.3657005284074883, |
|
"tokens_seen": 2526412800 |
|
}, |
|
{ |
|
"epoch": 0.9, |
|
"learning_rate": 4.8893512388878384e-05, |
|
"loss": 2.4407, |
|
"theoretical_loss": 3.3653533386933834, |
|
"tokens_seen": 2529689600 |
|
}, |
|
{ |
|
"epoch": 0.9, |
|
"learning_rate": 4.830243994703991e-05, |
|
"loss": 2.4079, |
|
"theoretical_loss": 3.3650067241539205, |
|
"tokens_seen": 2532966400 |
|
}, |
|
{ |
|
"epoch": 0.91, |
|
"learning_rate": 4.771136750520144e-05, |
|
"loss": 2.3975, |
|
"theoretical_loss": 3.364660683093999, |
|
"tokens_seen": 2536243200 |
|
}, |
|
{ |
|
"epoch": 0.91, |
|
"learning_rate": 4.712029506336297e-05, |
|
"loss": 2.4039, |
|
"theoretical_loss": 3.364315213825694, |
|
"tokens_seen": 2539520000 |
|
}, |
|
{ |
|
"epoch": 0.91, |
|
"learning_rate": 4.6529222621524494e-05, |
|
"loss": 2.4214, |
|
"theoretical_loss": 3.3639703146682187, |
|
"tokens_seen": 2542796800 |
|
}, |
|
{ |
|
"epoch": 0.91, |
|
"learning_rate": 4.593815017968602e-05, |
|
"loss": 2.5107, |
|
"theoretical_loss": 3.3636259839478853, |
|
"tokens_seen": 2546073600 |
|
}, |
|
{ |
|
"epoch": 0.91, |
|
"learning_rate": 4.5347077737847556e-05, |
|
"loss": 2.4944, |
|
"theoretical_loss": 3.363282219998065, |
|
"tokens_seen": 2549350400 |
|
}, |
|
{ |
|
"epoch": 0.91, |
|
"learning_rate": 4.4756005296009084e-05, |
|
"loss": 2.4843, |
|
"theoretical_loss": 3.3629390211591494, |
|
"tokens_seen": 2552627200 |
|
}, |
|
{ |
|
"epoch": 0.91, |
|
"learning_rate": 4.4164932854170605e-05, |
|
"loss": 2.4308, |
|
"theoretical_loss": 3.3625963857785135, |
|
"tokens_seen": 2555904000 |
|
}, |
|
{ |
|
"epoch": 0.91, |
|
"learning_rate": 4.357386041233213e-05, |
|
"loss": 2.4214, |
|
"theoretical_loss": 3.3622543122104744, |
|
"tokens_seen": 2559180800 |
|
}, |
|
{ |
|
"epoch": 0.92, |
|
"learning_rate": 4.2982787970493667e-05, |
|
"loss": 2.4213, |
|
"theoretical_loss": 3.361912798816256, |
|
"tokens_seen": 2562457600 |
|
}, |
|
{ |
|
"epoch": 0.92, |
|
"learning_rate": 4.2391715528655194e-05, |
|
"loss": 2.4573, |
|
"theoretical_loss": 3.36157184396395, |
|
"tokens_seen": 2565734400 |
|
}, |
|
{ |
|
"epoch": 0.92, |
|
"learning_rate": 4.180064308681672e-05, |
|
"loss": 2.4148, |
|
"theoretical_loss": 3.3612314460284782, |
|
"tokens_seen": 2569011200 |
|
}, |
|
{ |
|
"epoch": 0.92, |
|
"learning_rate": 4.120957064497825e-05, |
|
"loss": 2.4398, |
|
"theoretical_loss": 3.360891603391557, |
|
"tokens_seen": 2572288000 |
|
}, |
|
{ |
|
"epoch": 0.92, |
|
"learning_rate": 4.061849820313978e-05, |
|
"loss": 2.4694, |
|
"theoretical_loss": 3.3605523144416574, |
|
"tokens_seen": 2575564800 |
|
}, |
|
{ |
|
"epoch": 0.92, |
|
"learning_rate": 4.0027425761301304e-05, |
|
"loss": 2.4478, |
|
"theoretical_loss": 3.3602135775739708, |
|
"tokens_seen": 2578841600 |
|
}, |
|
{ |
|
"epoch": 0.92, |
|
"learning_rate": 3.943635331946283e-05, |
|
"loss": 2.4369, |
|
"theoretical_loss": 3.3598753911903705, |
|
"tokens_seen": 2582118400 |
|
}, |
|
{ |
|
"epoch": 0.92, |
|
"learning_rate": 3.8845280877624366e-05, |
|
"loss": 2.4568, |
|
"theoretical_loss": 3.359537753699379, |
|
"tokens_seen": 2585395200 |
|
}, |
|
{ |
|
"epoch": 0.92, |
|
"learning_rate": 3.8254208435785894e-05, |
|
"loss": 2.4285, |
|
"theoretical_loss": 3.3592006635161265, |
|
"tokens_seen": 2588672000 |
|
}, |
|
{ |
|
"epoch": 0.93, |
|
"learning_rate": 3.766313599394742e-05, |
|
"loss": 2.4122, |
|
"theoretical_loss": 3.35886411906232, |
|
"tokens_seen": 2591948800 |
|
}, |
|
{ |
|
"epoch": 0.93, |
|
"learning_rate": 3.707206355210894e-05, |
|
"loss": 2.4245, |
|
"theoretical_loss": 3.358528118766205, |
|
"tokens_seen": 2595225600 |
|
}, |
|
{ |
|
"epoch": 0.93, |
|
"learning_rate": 3.6480991110270477e-05, |
|
"loss": 2.432, |
|
"theoretical_loss": 3.3581926610625317, |
|
"tokens_seen": 2598502400 |
|
}, |
|
{ |
|
"epoch": 0.93, |
|
"learning_rate": 3.5889918668432004e-05, |
|
"loss": 2.3827, |
|
"theoretical_loss": 3.357857744392518, |
|
"tokens_seen": 2601779200 |
|
}, |
|
{ |
|
"epoch": 0.93, |
|
"learning_rate": 3.529884622659353e-05, |
|
"loss": 2.3907, |
|
"theoretical_loss": 3.3575233672038176, |
|
"tokens_seen": 2605056000 |
|
}, |
|
{ |
|
"epoch": 0.93, |
|
"learning_rate": 3.470777378475506e-05, |
|
"loss": 2.4026, |
|
"theoretical_loss": 3.3571895279504824, |
|
"tokens_seen": 2608332800 |
|
}, |
|
{ |
|
"epoch": 0.93, |
|
"learning_rate": 3.4116701342916594e-05, |
|
"loss": 2.4039, |
|
"theoretical_loss": 3.3568562250929306, |
|
"tokens_seen": 2611609600 |
|
}, |
|
{ |
|
"epoch": 0.93, |
|
"learning_rate": 3.3525628901078114e-05, |
|
"loss": 2.4276, |
|
"theoretical_loss": 3.3565234570979094, |
|
"tokens_seen": 2614886400 |
|
}, |
|
{ |
|
"epoch": 0.94, |
|
"learning_rate": 3.293455645923964e-05, |
|
"loss": 2.4217, |
|
"theoretical_loss": 3.356191222438466, |
|
"tokens_seen": 2618163200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.6462337884534494, |
|
"debugging/distinct-1-grams": 0.793787628277825, |
|
"debugging/distinct-2-grams": 0.9589607128973304, |
|
"debugging/entropy-1-grams": 6.137356991455709, |
|
"debugging/entropy-2-grams": 7.027986693751148, |
|
"debugging/length": 572.8, |
|
"debugging/num_segments": 20, |
|
"epoch": 0.94, |
|
"objective/train/docs_used": 1365290, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.245260715484619, |
|
"objective/train/theoretical_loss": 3.355859519593907, |
|
"objective/train/tokens_used": 2641900000, |
|
"theoretical_loss": 3.355859519593907, |
|
"tokens_seen": 2621440000 |
|
}, |
|
{ |
|
"epoch": 0.94, |
|
"learning_rate": 3.234348401740117e-05, |
|
"loss": 2.4023, |
|
"theoretical_loss": 3.355859519593907, |
|
"tokens_seen": 2621440000 |
|
}, |
|
{ |
|
"epoch": 0.94, |
|
"learning_rate": 3.1752411575562704e-05, |
|
"loss": 2.3756, |
|
"theoretical_loss": 3.355528347049773, |
|
"tokens_seen": 2624716800 |
|
}, |
|
{ |
|
"epoch": 0.94, |
|
"learning_rate": 3.116133913372423e-05, |
|
"loss": 2.3919, |
|
"theoretical_loss": 3.355197703297798, |
|
"tokens_seen": 2627993600 |
|
}, |
|
{ |
|
"epoch": 0.94, |
|
"learning_rate": 3.057026669188576e-05, |
|
"loss": 2.3867, |
|
"theoretical_loss": 3.354867586835882, |
|
"tokens_seen": 2631270400 |
|
}, |
|
{ |
|
"epoch": 0.94, |
|
"learning_rate": 2.9979194250047287e-05, |
|
"loss": 2.3714, |
|
"theoretical_loss": 3.354537996168054, |
|
"tokens_seen": 2634547200 |
|
}, |
|
{ |
|
"epoch": 0.94, |
|
"learning_rate": 2.9388121808208818e-05, |
|
"loss": 2.3994, |
|
"theoretical_loss": 3.3542089298044413, |
|
"tokens_seen": 2637824000 |
|
}, |
|
{ |
|
"epoch": 0.94, |
|
"learning_rate": 2.8797049366370342e-05, |
|
"loss": 2.4045, |
|
"theoretical_loss": 3.3538803862612374, |
|
"tokens_seen": 2641100800 |
|
}, |
|
{ |
|
"epoch": 0.94, |
|
"learning_rate": 2.8205976924531873e-05, |
|
"loss": 2.4014, |
|
"theoretical_loss": 3.3535523640606697, |
|
"tokens_seen": 2644377600 |
|
}, |
|
{ |
|
"epoch": 0.95, |
|
"learning_rate": 2.76149044826934e-05, |
|
"loss": 2.4135, |
|
"theoretical_loss": 3.353224861730966, |
|
"tokens_seen": 2647654400 |
|
}, |
|
{ |
|
"epoch": 0.95, |
|
"learning_rate": 2.7023832040854928e-05, |
|
"loss": 2.3962, |
|
"theoretical_loss": 3.3528978778063254, |
|
"tokens_seen": 2650931200 |
|
}, |
|
{ |
|
"epoch": 0.95, |
|
"learning_rate": 2.6432759599016455e-05, |
|
"loss": 2.3932, |
|
"theoretical_loss": 3.3525714108268834, |
|
"tokens_seen": 2654208000 |
|
}, |
|
{ |
|
"epoch": 0.95, |
|
"learning_rate": 2.5841687157177986e-05, |
|
"loss": 2.3867, |
|
"theoretical_loss": 3.352245459338684, |
|
"tokens_seen": 2657484800 |
|
}, |
|
{ |
|
"epoch": 0.95, |
|
"learning_rate": 2.525061471533951e-05, |
|
"loss": 2.3879, |
|
"theoretical_loss": 3.351920021893646, |
|
"tokens_seen": 2660761600 |
|
}, |
|
{ |
|
"epoch": 0.95, |
|
"learning_rate": 2.465954227350104e-05, |
|
"loss": 2.4317, |
|
"theoretical_loss": 3.3515950970495334, |
|
"tokens_seen": 2664038400 |
|
}, |
|
{ |
|
"epoch": 0.95, |
|
"learning_rate": 2.406846983166257e-05, |
|
"loss": 2.3925, |
|
"theoretical_loss": 3.3512706833699237, |
|
"tokens_seen": 2667315200 |
|
}, |
|
{ |
|
"epoch": 0.95, |
|
"learning_rate": 2.3477397389824097e-05, |
|
"loss": 2.4097, |
|
"theoretical_loss": 3.3509467794241794, |
|
"tokens_seen": 2670592000 |
|
}, |
|
{ |
|
"epoch": 0.95, |
|
"learning_rate": 2.2886324947985624e-05, |
|
"loss": 2.382, |
|
"theoretical_loss": 3.350623383787414, |
|
"tokens_seen": 2673868800 |
|
}, |
|
{ |
|
"epoch": 0.96, |
|
"learning_rate": 2.2295252506147155e-05, |
|
"loss": 2.3853, |
|
"theoretical_loss": 3.350300495040466, |
|
"tokens_seen": 2677145600 |
|
}, |
|
{ |
|
"epoch": 0.96, |
|
"learning_rate": 2.1704180064308683e-05, |
|
"loss": 2.3617, |
|
"theoretical_loss": 3.3499781117698646, |
|
"tokens_seen": 2680422400 |
|
}, |
|
{ |
|
"epoch": 0.96, |
|
"learning_rate": 2.111310762247021e-05, |
|
"loss": 2.376, |
|
"theoretical_loss": 3.349656232567805, |
|
"tokens_seen": 2683699200 |
|
}, |
|
{ |
|
"epoch": 0.96, |
|
"learning_rate": 2.052203518063174e-05, |
|
"loss": 2.3458, |
|
"theoretical_loss": 3.3493348560321143, |
|
"tokens_seen": 2686976000 |
|
}, |
|
{ |
|
"epoch": 0.96, |
|
"learning_rate": 1.9930962738793265e-05, |
|
"loss": 2.3209, |
|
"theoretical_loss": 3.349013980766224, |
|
"tokens_seen": 2690252800 |
|
}, |
|
{ |
|
"epoch": 0.96, |
|
"learning_rate": 1.9339890296954796e-05, |
|
"loss": 2.3782, |
|
"theoretical_loss": 3.3486936053791423, |
|
"tokens_seen": 2693529600 |
|
}, |
|
{ |
|
"epoch": 0.96, |
|
"learning_rate": 1.8748817855116324e-05, |
|
"loss": 2.3657, |
|
"theoretical_loss": 3.348373728485421, |
|
"tokens_seen": 2696806400 |
|
}, |
|
{ |
|
"epoch": 0.96, |
|
"learning_rate": 1.815774541327785e-05, |
|
"loss": 2.3551, |
|
"theoretical_loss": 3.348054348705131, |
|
"tokens_seen": 2700083200 |
|
}, |
|
{ |
|
"epoch": 0.97, |
|
"learning_rate": 1.756667297143938e-05, |
|
"loss": 2.355, |
|
"theoretical_loss": 3.3477354646638315, |
|
"tokens_seen": 2703360000 |
|
}, |
|
{ |
|
"epoch": 0.97, |
|
"learning_rate": 1.697560052960091e-05, |
|
"loss": 2.3792, |
|
"theoretical_loss": 3.3474170749925416, |
|
"tokens_seen": 2706636800 |
|
}, |
|
{ |
|
"epoch": 0.97, |
|
"learning_rate": 1.6384528087762434e-05, |
|
"loss": 2.4055, |
|
"theoretical_loss": 3.347099178327713, |
|
"tokens_seen": 2709913600 |
|
}, |
|
{ |
|
"epoch": 0.97, |
|
"learning_rate": 1.5793455645923965e-05, |
|
"loss": 2.3966, |
|
"theoretical_loss": 3.3467817733112, |
|
"tokens_seen": 2713190400 |
|
}, |
|
{ |
|
"epoch": 0.97, |
|
"learning_rate": 1.5202383204085493e-05, |
|
"loss": 2.4374, |
|
"theoretical_loss": 3.3464648585902346, |
|
"tokens_seen": 2716467200 |
|
}, |
|
{ |
|
"epoch": 0.97, |
|
"learning_rate": 1.4611310762247022e-05, |
|
"loss": 2.3863, |
|
"theoretical_loss": 3.3461484328173956, |
|
"tokens_seen": 2719744000 |
|
}, |
|
{ |
|
"epoch": 0.97, |
|
"learning_rate": 1.402023832040855e-05, |
|
"loss": 2.3656, |
|
"theoretical_loss": 3.3458324946505824, |
|
"tokens_seen": 2723020800 |
|
}, |
|
{ |
|
"epoch": 0.97, |
|
"learning_rate": 1.3429165878570077e-05, |
|
"loss": 2.2521, |
|
"theoretical_loss": 3.3455170427529883, |
|
"tokens_seen": 2726297600 |
|
}, |
|
{ |
|
"epoch": 0.97, |
|
"learning_rate": 1.2838093436731606e-05, |
|
"loss": 2.3189, |
|
"theoretical_loss": 3.345202075793073, |
|
"tokens_seen": 2729574400 |
|
}, |
|
{ |
|
"epoch": 0.98, |
|
"learning_rate": 1.2247020994893134e-05, |
|
"loss": 2.3231, |
|
"theoretical_loss": 3.344887592444535, |
|
"tokens_seen": 2732851200 |
|
}, |
|
{ |
|
"epoch": 0.98, |
|
"learning_rate": 1.1655948553054663e-05, |
|
"loss": 2.3481, |
|
"theoretical_loss": 3.3445735913862835, |
|
"tokens_seen": 2736128000 |
|
}, |
|
{ |
|
"epoch": 0.98, |
|
"learning_rate": 1.106487611121619e-05, |
|
"loss": 2.3413, |
|
"theoretical_loss": 3.344260071302415, |
|
"tokens_seen": 2739404800 |
|
}, |
|
{ |
|
"epoch": 0.98, |
|
"learning_rate": 1.0473803669377718e-05, |
|
"loss": 2.3303, |
|
"theoretical_loss": 3.3439470308821844, |
|
"tokens_seen": 2742681600 |
|
}, |
|
{ |
|
"epoch": 0.98, |
|
"learning_rate": 9.882731227539248e-06, |
|
"loss": 2.344, |
|
"theoretical_loss": 3.3436344688199786, |
|
"tokens_seen": 2745958400 |
|
}, |
|
{ |
|
"epoch": 0.98, |
|
"learning_rate": 9.303480234537545e-06, |
|
"loss": 2.3619, |
|
"theoretical_loss": 3.3433223838152917, |
|
"tokens_seen": 2749235200 |
|
}, |
|
{ |
|
"epoch": 0.98, |
|
"learning_rate": 8.712407792699074e-06, |
|
"loss": 2.3408, |
|
"theoretical_loss": 3.3430107745726976, |
|
"tokens_seen": 2752512000 |
|
}, |
|
{ |
|
"epoch": 0.98, |
|
"learning_rate": 8.121335350860602e-06, |
|
"loss": 2.3466, |
|
"theoretical_loss": 3.3426996398018245, |
|
"tokens_seen": 2755788800 |
|
}, |
|
{ |
|
"epoch": 0.99, |
|
"learning_rate": 7.530262909022129e-06, |
|
"loss": 2.3517, |
|
"theoretical_loss": 3.3423889782173304, |
|
"tokens_seen": 2759065600 |
|
}, |
|
{ |
|
"epoch": 0.99, |
|
"learning_rate": 6.939190467183658e-06, |
|
"loss": 2.3374, |
|
"theoretical_loss": 3.342078788538875, |
|
"tokens_seen": 2762342400 |
|
}, |
|
{ |
|
"epoch": 0.99, |
|
"learning_rate": 6.348118025345187e-06, |
|
"loss": 2.3818, |
|
"theoretical_loss": 3.3417690694910975, |
|
"tokens_seen": 2765619200 |
|
}, |
|
{ |
|
"epoch": 0.99, |
|
"learning_rate": 5.757045583506715e-06, |
|
"loss": 2.3785, |
|
"theoretical_loss": 3.3414598198035885, |
|
"tokens_seen": 2768896000 |
|
}, |
|
{ |
|
"epoch": 0.99, |
|
"learning_rate": 5.165973141668243e-06, |
|
"loss": 2.3242, |
|
"theoretical_loss": 3.3411510382108665, |
|
"tokens_seen": 2772172800 |
|
}, |
|
{ |
|
"epoch": 0.99, |
|
"learning_rate": 4.574900699829771e-06, |
|
"loss": 2.3614, |
|
"theoretical_loss": 3.340842723452353, |
|
"tokens_seen": 2775449600 |
|
}, |
|
{ |
|
"epoch": 0.99, |
|
"learning_rate": 3.9838282579913e-06, |
|
"loss": 2.3681, |
|
"theoretical_loss": 3.340534874272347, |
|
"tokens_seen": 2778726400 |
|
}, |
|
{ |
|
"epoch": 0.99, |
|
"learning_rate": 3.392755816152828e-06, |
|
"loss": 2.3397, |
|
"theoretical_loss": 3.340227489420001, |
|
"tokens_seen": 2782003200 |
|
}, |
|
{ |
|
"epoch": 0.99, |
|
"objective/train/docs_used": 1452310, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.2117185592651367, |
|
"objective/train/theoretical_loss": 3.339920567649296, |
|
"objective/train/tokens_used": 2805740000, |
|
"theoretical_loss": 3.339920567649296, |
|
"tokens_seen": 2785280000 |
|
}, |
|
{ |
|
"epoch": 0.99, |
|
"learning_rate": 2.801683374314356e-06, |
|
"loss": 2.371, |
|
"theoretical_loss": 3.339920567649296, |
|
"tokens_seen": 2785280000 |
|
}, |
|
{ |
|
"epoch": 1.0, |
|
"learning_rate": 2.210610932475884e-06, |
|
"loss": 2.3568, |
|
"theoretical_loss": 3.339614107719018, |
|
"tokens_seen": 2788556800 |
|
}, |
|
{ |
|
"epoch": 1.0, |
|
"learning_rate": 1.6195384906374124e-06, |
|
"loss": 2.344, |
|
"theoretical_loss": 3.339308108392733, |
|
"tokens_seen": 2791833600 |
|
}, |
|
{ |
|
"epoch": 1.0, |
|
"learning_rate": 1.0284660487989408e-06, |
|
"loss": 2.3622, |
|
"theoretical_loss": 3.3390025684387643, |
|
"tokens_seen": 2795110400 |
|
}, |
|
{ |
|
"epoch": 1.0, |
|
"learning_rate": 4.373936069604691e-07, |
|
"loss": 2.37, |
|
"theoretical_loss": 3.3386974866301666, |
|
"tokens_seen": 2798387200 |
|
} |
|
], |
|
"max_steps": 42724, |
|
"num_train_epochs": 9223372036854775807, |
|
"total_flos": 1.428920419221504e+18, |
|
"trial_name": null, |
|
"trial_params": null |
|
} |
|
|