| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.5, | |
| "global_step": 21362, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 1.1682242990654204e-06, | |
| "loss": 10.8584, | |
| "theoretical_loss": 20.812814784551147, | |
| "tokens_seen": 65536 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 5.841121495327103e-05, | |
| "loss": 9.0184, | |
| "theoretical_loss": 8.563479647615063, | |
| "tokens_seen": 3276800 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00011682242990654206, | |
| "loss": 6.9386, | |
| "theoretical_loss": 7.4777557010520255, | |
| "tokens_seen": 6553600 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00017523364485981307, | |
| "loss": 6.096, | |
| "theoretical_loss": 6.933751471898896, | |
| "tokens_seen": 9830400 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00023364485981308412, | |
| "loss": 5.7275, | |
| "theoretical_loss": 6.583563211430409, | |
| "tokens_seen": 13107200 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00029205607476635517, | |
| "loss": 5.4399, | |
| "theoretical_loss": 6.330710548120079, | |
| "tokens_seen": 16384000 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00035046728971962614, | |
| "loss": 5.2491, | |
| "theoretical_loss": 6.135526214944321, | |
| "tokens_seen": 19660800 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004088785046728972, | |
| "loss": 5.1116, | |
| "theoretical_loss": 5.978098566873603, | |
| "tokens_seen": 22937600 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00046728971962616824, | |
| "loss": 4.9855, | |
| "theoretical_loss": 5.847114309269919, | |
| "tokens_seen": 26214400 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000499739928125591, | |
| "loss": 4.8409, | |
| "theoretical_loss": 5.73557379888612, | |
| "tokens_seen": 29491200 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004991488556837526, | |
| "loss": 4.7327, | |
| "theoretical_loss": 5.638867127075349, | |
| "tokens_seen": 32768000 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004985577832419141, | |
| "loss": 4.6398, | |
| "theoretical_loss": 5.553809364848902, | |
| "tokens_seen": 36044800 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004979667108000757, | |
| "loss": 4.5391, | |
| "theoretical_loss": 5.478115063560433, | |
| "tokens_seen": 39321600 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004973756383582371, | |
| "loss": 4.4539, | |
| "theoretical_loss": 5.410092942583357, | |
| "tokens_seen": 42598400 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004967845659163987, | |
| "loss": 4.4011, | |
| "theoretical_loss": 5.34845906673983, | |
| "tokens_seen": 45875200 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004961934934745603, | |
| "loss": 4.2868, | |
| "theoretical_loss": 5.292217549941562, | |
| "tokens_seen": 49152000 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004956024210327218, | |
| "loss": 4.2131, | |
| "theoretical_loss": 5.240581608773973, | |
| "tokens_seen": 52428800 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004950113485908833, | |
| "loss": 4.1737, | |
| "theoretical_loss": 5.192919707529784, | |
| "tokens_seen": 55705600 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004944202761490448, | |
| "loss": 4.0132, | |
| "theoretical_loss": 5.1487178463604355, | |
| "tokens_seen": 58982400 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004938292037072064, | |
| "loss": 3.9839, | |
| "theoretical_loss": 5.107552545409097, | |
| "tokens_seen": 62259200 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004932381312653678, | |
| "loss": 3.8464, | |
| "theoretical_loss": 5.069071100147241, | |
| "tokens_seen": 65536000 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004926470588235294, | |
| "loss": 3.8002, | |
| "theoretical_loss": 5.032976892842002, | |
| "tokens_seen": 68812800 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.000492055986381691, | |
| "loss": 3.716, | |
| "theoretical_loss": 4.999018291228659, | |
| "tokens_seen": 72089600 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004914649139398525, | |
| "loss": 3.7048, | |
| "theoretical_loss": 4.966980138355957, | |
| "tokens_seen": 75366400 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.000490873841498014, | |
| "loss": 3.6445, | |
| "theoretical_loss": 4.936677144629131, | |
| "tokens_seen": 78643200 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004902827690561755, | |
| "loss": 3.6049, | |
| "theoretical_loss": 4.907948696834077, | |
| "tokens_seen": 81920000 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004896916966143371, | |
| "loss": 3.5803, | |
| "theoretical_loss": 4.880654736816922, | |
| "tokens_seen": 85196800 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004891006241724985, | |
| "loss": 3.5116, | |
| "theoretical_loss": 4.8546724574857745, | |
| "tokens_seen": 88473600 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004885095517306601, | |
| "loss": 3.4969, | |
| "theoretical_loss": 4.829893630312808, | |
| "tokens_seen": 91750400 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004879184792888217, | |
| "loss": 3.4364, | |
| "theoretical_loss": 4.806222425781916, | |
| "tokens_seen": 95027200 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004873274068469832, | |
| "loss": 3.444, | |
| "theoretical_loss": 4.783573622280253, | |
| "tokens_seen": 98304000 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.00048673633440514467, | |
| "loss": 3.4411, | |
| "theoretical_loss": 4.7618711237764035, | |
| "tokens_seen": 101580800 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004861452619633062, | |
| "loss": 3.448, | |
| "theoretical_loss": 4.741046724966468, | |
| "tokens_seen": 104857600 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004855541895214677, | |
| "loss": 3.3972, | |
| "theoretical_loss": 4.721039076253046, | |
| "tokens_seen": 108134400 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004849631170796293, | |
| "loss": 3.4331, | |
| "theoretical_loss": 4.701792811235861, | |
| "tokens_seen": 111411200 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004843720446377908, | |
| "loss": 3.4396, | |
| "theoretical_loss": 4.683257807239925, | |
| "tokens_seen": 114688000 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.00048378097219595233, | |
| "loss": 3.4205, | |
| "theoretical_loss": 4.665388555430277, | |
| "tokens_seen": 117964800 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.00048318989975411385, | |
| "loss": 3.3658, | |
| "theoretical_loss": 4.648143621723734, | |
| "tokens_seen": 121241600 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.00048259882731227537, | |
| "loss": 3.3609, | |
| "theoretical_loss": 4.631485183343639, | |
| "tokens_seen": 124518400 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0004820077548704369, | |
| "loss": 3.3609, | |
| "theoretical_loss": 4.6153786287197125, | |
| "tokens_seen": 127795200 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00048141668242859847, | |
| "loss": 3.3415, | |
| "theoretical_loss": 4.5997922106945, | |
| "tokens_seen": 131072000 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00048082560998676, | |
| "loss": 3.3173, | |
| "theoretical_loss": 4.58469674479667, | |
| "tokens_seen": 134348800 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0004802345375449215, | |
| "loss": 3.3429, | |
| "theoretical_loss": 4.570065345782512, | |
| "tokens_seen": 137625600 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00047964346510308303, | |
| "loss": 3.2678, | |
| "theoretical_loss": 4.555873196808033, | |
| "tokens_seen": 140902400 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00047905239266124455, | |
| "loss": 3.2872, | |
| "theoretical_loss": 4.542097346534794, | |
| "tokens_seen": 144179200 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00047846132021940607, | |
| "loss": 3.2486, | |
| "theoretical_loss": 4.528716530238812, | |
| "tokens_seen": 147456000 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0004778702477775676, | |
| "loss": 3.2852, | |
| "theoretical_loss": 4.515711011618992, | |
| "tokens_seen": 150732800 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047727917533572917, | |
| "loss": 3.2308, | |
| "theoretical_loss": 4.503062442517334, | |
| "tokens_seen": 154009600 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0004766881028938907, | |
| "loss": 3.2673, | |
| "theoretical_loss": 4.4907537381892615, | |
| "tokens_seen": 157286400 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0004760970304520522, | |
| "loss": 3.2672, | |
| "theoretical_loss": 4.478768966115963, | |
| "tokens_seen": 160563200 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "objective/train/docs_used": 101104, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 3.6775944232940674, | |
| "objective/train/theoretical_loss": 4.467093246645215, | |
| "objective/train/tokens_used": 184300000, | |
| "theoretical_loss": 4.467093246645215, | |
| "tokens_seen": 163840000 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047550595801021373, | |
| "loss": 3.2026, | |
| "theoretical_loss": 4.467093246645215, | |
| "tokens_seen": 163840000 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047491488556837525, | |
| "loss": 3.2085, | |
| "theoretical_loss": 4.455712663993541, | |
| "tokens_seen": 167116800 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047432381312653677, | |
| "loss": 3.2025, | |
| "theoretical_loss": 4.444614186349425, | |
| "tokens_seen": 170393600 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047373274068469835, | |
| "loss": 3.183, | |
| "theoretical_loss": 4.433785593991642, | |
| "tokens_seen": 173670400 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047314166824285987, | |
| "loss": 3.1657, | |
| "theoretical_loss": 4.423215414484155, | |
| "tokens_seen": 176947200 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0004725505958010214, | |
| "loss": 3.1526, | |
| "theoretical_loss": 4.412892864134137, | |
| "tokens_seen": 180224000 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0004719595233591829, | |
| "loss": 3.1314, | |
| "theoretical_loss": 4.402807795006074, | |
| "tokens_seen": 183500800 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00047136845091734443, | |
| "loss": 3.1306, | |
| "theoretical_loss": 4.392950646875857, | |
| "tokens_seen": 186777600 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00047077737847550595, | |
| "loss": 3.1734, | |
| "theoretical_loss": 4.383312403586528, | |
| "tokens_seen": 190054400 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0004701863060336675, | |
| "loss": 3.1533, | |
| "theoretical_loss": 4.373884553334271, | |
| "tokens_seen": 193331200 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00046959523359182905, | |
| "loss": 3.1653, | |
| "theoretical_loss": 4.364659052470699, | |
| "tokens_seen": 196608000 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00046900416114999057, | |
| "loss": 3.1275, | |
| "theoretical_loss": 4.355628292457278, | |
| "tokens_seen": 199884800 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0004684130887081521, | |
| "loss": 3.1234, | |
| "theoretical_loss": 4.346785069650667, | |
| "tokens_seen": 203161600 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0004678220162663136, | |
| "loss": 3.1054, | |
| "theoretical_loss": 4.338122557635156, | |
| "tokens_seen": 206438400 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00046723094382447513, | |
| "loss": 3.1138, | |
| "theoretical_loss": 4.329634281850807, | |
| "tokens_seen": 209715200 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.00046663987138263665, | |
| "loss": 3.1681, | |
| "theoretical_loss": 4.321314096294248, | |
| "tokens_seen": 212992000 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0004660487989407982, | |
| "loss": 3.0757, | |
| "theoretical_loss": 4.3131561620937875, | |
| "tokens_seen": 216268800 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.00046545772649895975, | |
| "loss": 3.0927, | |
| "theoretical_loss": 4.305154927782223, | |
| "tokens_seen": 219545600 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.00046486665405712127, | |
| "loss": 3.0431, | |
| "theoretical_loss": 4.297305111109683, | |
| "tokens_seen": 222822400 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0004642755816152828, | |
| "loss": 3.106, | |
| "theoretical_loss": 4.2896016822555945, | |
| "tokens_seen": 226099200 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0004636845091734443, | |
| "loss": 3.1308, | |
| "theoretical_loss": 4.282039848313611, | |
| "tokens_seen": 229376000 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.00046309343673160583, | |
| "loss": 3.122, | |
| "theoretical_loss": 4.274615038936293, | |
| "tokens_seen": 232652800 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0004625023642897674, | |
| "loss": 3.1477, | |
| "theoretical_loss": 4.267322893037893, | |
| "tokens_seen": 235929600 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004619112918479289, | |
| "loss": 3.1332, | |
| "theoretical_loss": 4.26015924646374, | |
| "tokens_seen": 239206400 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.00046132021940609044, | |
| "loss": 3.1128, | |
| "theoretical_loss": 4.253120120543809, | |
| "tokens_seen": 242483200 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.00046072914696425197, | |
| "loss": 3.0438, | |
| "theoretical_loss": 4.24620171145605, | |
| "tokens_seen": 245760000 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004601380745224135, | |
| "loss": 3.0475, | |
| "theoretical_loss": 4.239400380332256, | |
| "tokens_seen": 249036800 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.000459547002080575, | |
| "loss": 3.0507, | |
| "theoretical_loss": 4.232712644045627, | |
| "tokens_seen": 252313600 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004589559296387366, | |
| "loss": 3.0612, | |
| "theoretical_loss": 4.226135166624862, | |
| "tokens_seen": 255590400 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004583648571968981, | |
| "loss": 3.0783, | |
| "theoretical_loss": 4.21966475124477, | |
| "tokens_seen": 258867200 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004577737847550596, | |
| "loss": 3.0221, | |
| "theoretical_loss": 4.21329833274792, | |
| "tokens_seen": 262144000 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004571827123132211, | |
| "loss": 3.0286, | |
| "theoretical_loss": 4.207032970655965, | |
| "tokens_seen": 265420800 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0004565916398713826, | |
| "loss": 2.974, | |
| "theoretical_loss": 4.2008658426329974, | |
| "tokens_seen": 268697600 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.00045600056742954413, | |
| "loss": 2.995, | |
| "theoretical_loss": 4.194794238366544, | |
| "tokens_seen": 271974400 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0004554094949877057, | |
| "loss": 2.9599, | |
| "theoretical_loss": 4.188815553834879, | |
| "tokens_seen": 275251200 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.00045481842254586723, | |
| "loss": 2.9159, | |
| "theoretical_loss": 4.182927285931959, | |
| "tokens_seen": 278528000 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.00045422735010402875, | |
| "loss": 2.9718, | |
| "theoretical_loss": 4.177127027423776, | |
| "tokens_seen": 281804800 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.00045363627766219027, | |
| "loss": 3.0047, | |
| "theoretical_loss": 4.171412462212087, | |
| "tokens_seen": 285081600 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0004530452052203518, | |
| "loss": 2.9699, | |
| "theoretical_loss": 4.165781360883513, | |
| "tokens_seen": 288358400 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0004524541327785133, | |
| "loss": 2.9841, | |
| "theoretical_loss": 4.160231576523763, | |
| "tokens_seen": 291635200 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.00045186306033667483, | |
| "loss": 3.0051, | |
| "theoretical_loss": 4.1547610407784346, | |
| "tokens_seen": 294912000 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0004512719878948364, | |
| "loss": 3.0277, | |
| "theoretical_loss": 4.1493677601432815, | |
| "tokens_seen": 298188800 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.00045068091545299793, | |
| "loss": 3.0073, | |
| "theoretical_loss": 4.1440498124682446, | |
| "tokens_seen": 301465600 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.00045008984301115945, | |
| "loss": 3.0092, | |
| "theoretical_loss": 4.138805343660737, | |
| "tokens_seen": 304742400 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.00044949877056932097, | |
| "loss": 2.9717, | |
| "theoretical_loss": 4.133632564574831, | |
| "tokens_seen": 308019200 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0004489076981274825, | |
| "loss": 2.9231, | |
| "theoretical_loss": 4.128529748073999, | |
| "tokens_seen": 311296000 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.000448316625685644, | |
| "loss": 2.9461, | |
| "theoretical_loss": 4.123495226256027, | |
| "tokens_seen": 314572800 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0004477255532438056, | |
| "loss": 2.9478, | |
| "theoretical_loss": 4.118527387829552, | |
| "tokens_seen": 317849600 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0004471344808019671, | |
| "loss": 2.9396, | |
| "theoretical_loss": 4.11362467563246, | |
| "tokens_seen": 321126400 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.00044654340836012863, | |
| "loss": 2.9153, | |
| "theoretical_loss": 4.108785584283144, | |
| "tokens_seen": 324403200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.5369745433205646, | |
| "debugging/distinct-1-grams": 0.7852136036914666, | |
| "debugging/distinct-2-grams": 0.975898791228072, | |
| "debugging/entropy-1-grams": 5.9202049103095575, | |
| "debugging/entropy-2-grams": 6.8314825534787165, | |
| "debugging/length": 575.3333333333334, | |
| "debugging/num_segments": 12, | |
| "epoch": 0.12, | |
| "objective/train/docs_used": 191544, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 3.262237310409546, | |
| "objective/train/theoretical_loss": 4.104008657956216, | |
| "objective/train/tokens_used": 348140000, | |
| "theoretical_loss": 4.104008657956216, | |
| "tokens_seen": 327680000 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.00044595233591829015, | |
| "loss": 2.92, | |
| "theoretical_loss": 4.104008657956216, | |
| "tokens_seen": 327680000 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.00044536126347645167, | |
| "loss": 2.9517, | |
| "theoretical_loss": 4.099292488274917, | |
| "tokens_seen": 330956800 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0004447701910346132, | |
| "loss": 2.9843, | |
| "theoretical_loss": 4.094635712313026, | |
| "tokens_seen": 334233600 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.00044417911859277476, | |
| "loss": 2.9337, | |
| "theoretical_loss": 4.090037010699552, | |
| "tokens_seen": 337510400 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0004435880461509363, | |
| "loss": 2.9564, | |
| "theoretical_loss": 4.085495105819987, | |
| "tokens_seen": 340787200 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0004429969737090978, | |
| "loss": 2.9046, | |
| "theoretical_loss": 4.081008760108329, | |
| "tokens_seen": 344064000 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0004424059012672593, | |
| "loss": 2.922, | |
| "theoretical_loss": 4.076576774424465, | |
| "tokens_seen": 347340800 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00044181482882542085, | |
| "loss": 2.9275, | |
| "theoretical_loss": 4.072197986511911, | |
| "tokens_seen": 350617600 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00044122375638358237, | |
| "loss": 2.9031, | |
| "theoretical_loss": 4.067871269531193, | |
| "tokens_seen": 353894400 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0004406326839417439, | |
| "loss": 2.9345, | |
| "theoretical_loss": 4.063595530664515, | |
| "tokens_seen": 357171200 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00044004161149990546, | |
| "loss": 2.9301, | |
| "theoretical_loss": 4.059369709787625, | |
| "tokens_seen": 360448000 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.000439450539058067, | |
| "loss": 2.9386, | |
| "theoretical_loss": 4.055192778205064, | |
| "tokens_seen": 363724800 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0004388594666162285, | |
| "loss": 2.9477, | |
| "theoretical_loss": 4.051063737445231, | |
| "tokens_seen": 367001600 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00043826839417439, | |
| "loss": 2.8887, | |
| "theoretical_loss": 4.046981618111936, | |
| "tokens_seen": 370278400 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00043767732173255155, | |
| "loss": 2.9278, | |
| "theoretical_loss": 4.042945478789308, | |
| "tokens_seen": 373555200 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00043708624929071307, | |
| "loss": 2.9088, | |
| "theoretical_loss": 4.038954404997148, | |
| "tokens_seen": 376832000 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.00043649517684887464, | |
| "loss": 2.882, | |
| "theoretical_loss": 4.035007508193978, | |
| "tokens_seen": 380108800 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.00043590410440703616, | |
| "loss": 2.9125, | |
| "theoretical_loss": 4.031103924825214, | |
| "tokens_seen": 383385600 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0004353130319651977, | |
| "loss": 2.8289, | |
| "theoretical_loss": 4.027242815414074, | |
| "tokens_seen": 386662400 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0004347219595233592, | |
| "loss": 2.9065, | |
| "theoretical_loss": 4.023423363692939, | |
| "tokens_seen": 389939200 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0004341308870815207, | |
| "loss": 2.9067, | |
| "theoretical_loss": 4.019644775773043, | |
| "tokens_seen": 393216000 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.00043353981463968225, | |
| "loss": 2.8752, | |
| "theoretical_loss": 4.015906279350517, | |
| "tokens_seen": 396492800 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0004329487421978438, | |
| "loss": 2.8745, | |
| "theoretical_loss": 4.01220712294689, | |
| "tokens_seen": 399769600 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.00043235766975600534, | |
| "loss": 2.903, | |
| "theoretical_loss": 4.008546575182286, | |
| "tokens_seen": 403046400 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.00043176659731416686, | |
| "loss": 2.9016, | |
| "theoretical_loss": 4.0049239240796695, | |
| "tokens_seen": 406323200 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004311755248723284, | |
| "loss": 2.911, | |
| "theoretical_loss": 4.001338476398553, | |
| "tokens_seen": 409600000 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004305844524304899, | |
| "loss": 2.8983, | |
| "theoretical_loss": 3.9977895569967217, | |
| "tokens_seen": 412876800 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004299933799886514, | |
| "loss": 2.9451, | |
| "theoretical_loss": 3.99427650821855, | |
| "tokens_seen": 416153600 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.00042940230754681295, | |
| "loss": 2.9276, | |
| "theoretical_loss": 3.990798689308642, | |
| "tokens_seen": 419430400 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004288112351049745, | |
| "loss": 2.9441, | |
| "theoretical_loss": 3.987355475849527, | |
| "tokens_seen": 422707200 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.000428220162663136, | |
| "loss": 2.8943, | |
| "theoretical_loss": 3.98394625922226, | |
| "tokens_seen": 425984000 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004276290902212975, | |
| "loss": 2.9401, | |
| "theoretical_loss": 3.9805704460888256, | |
| "tokens_seen": 429260800 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.00042703801777945903, | |
| "loss": 2.9437, | |
| "theoretical_loss": 3.9772274578953013, | |
| "tokens_seen": 432537600 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042644694533762055, | |
| "loss": 2.9373, | |
| "theoretical_loss": 3.973916730394796, | |
| "tokens_seen": 435814400 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042585587289578207, | |
| "loss": 2.9168, | |
| "theoretical_loss": 3.970637713189244, | |
| "tokens_seen": 439091200 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042526480045394365, | |
| "loss": 2.8928, | |
| "theoretical_loss": 3.967389869289161, | |
| "tokens_seen": 442368000 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042467372801210517, | |
| "loss": 2.9101, | |
| "theoretical_loss": 3.964172674690542, | |
| "tokens_seen": 445644800 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0004240826555702667, | |
| "loss": 2.8902, | |
| "theoretical_loss": 3.9609856179681078, | |
| "tokens_seen": 448921600 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0004234915831284282, | |
| "loss": 2.8893, | |
| "theoretical_loss": 3.957828199884155, | |
| "tokens_seen": 452198400 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042290051068658973, | |
| "loss": 2.8546, | |
| "theoretical_loss": 3.9546999330123036, | |
| "tokens_seen": 455475200 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042230943824475125, | |
| "loss": 2.917, | |
| "theoretical_loss": 3.951600341375469, | |
| "tokens_seen": 458752000 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0004217183658029128, | |
| "loss": 2.866, | |
| "theoretical_loss": 3.9485289600974305, | |
| "tokens_seen": 462028800 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.00042112729336107435, | |
| "loss": 2.8189, | |
| "theoretical_loss": 3.945485335067386, | |
| "tokens_seen": 465305600 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.00042053622091923587, | |
| "loss": 2.8606, | |
| "theoretical_loss": 3.9424690226169212, | |
| "tokens_seen": 468582400 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0004199451484773974, | |
| "loss": 2.8887, | |
| "theoretical_loss": 3.9394795892088585, | |
| "tokens_seen": 471859200 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0004193540760355589, | |
| "loss": 2.8486, | |
| "theoretical_loss": 3.9365166111374608, | |
| "tokens_seen": 475136000 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.00041876300359372043, | |
| "loss": 2.7856, | |
| "theoretical_loss": 3.933579674239507, | |
| "tokens_seen": 478412800 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.00041817193115188195, | |
| "loss": 2.8535, | |
| "theoretical_loss": 3.930668373615765, | |
| "tokens_seen": 481689600 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0004175808587100435, | |
| "loss": 2.7963, | |
| "theoretical_loss": 3.9277823133624366, | |
| "tokens_seen": 484966400 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.00041698978626820505, | |
| "loss": 2.8532, | |
| "theoretical_loss": 3.9249211063121283, | |
| "tokens_seen": 488243200 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "objective/train/docs_used": 280311, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.946796417236328, | |
| "objective/train/theoretical_loss": 3.9220843737839752, | |
| "objective/train/tokens_used": 511980000, | |
| "theoretical_loss": 3.9220843737839752, | |
| "tokens_seen": 491520000 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.00041639871382636657, | |
| "loss": 2.8478, | |
| "theoretical_loss": 3.9220843737839752, | |
| "tokens_seen": 491520000 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0004158076413845281, | |
| "loss": 2.8832, | |
| "theoretical_loss": 3.919271745342514, | |
| "tokens_seen": 494796800 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0004152165689426896, | |
| "loss": 2.8583, | |
| "theoretical_loss": 3.916482858564957, | |
| "tokens_seen": 498073600 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.00041462549650085113, | |
| "loss": 2.7761, | |
| "theoretical_loss": 3.9137173588165135, | |
| "tokens_seen": 501350400 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0004140344240590127, | |
| "loss": 2.8483, | |
| "theoretical_loss": 3.9109748990334348, | |
| "tokens_seen": 504627200 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0004134433516171742, | |
| "loss": 2.8321, | |
| "theoretical_loss": 3.908255139513467, | |
| "tokens_seen": 507904000 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.00041285227917533575, | |
| "loss": 2.8297, | |
| "theoretical_loss": 3.905557747713412, | |
| "tokens_seen": 511180800 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.00041226120673349727, | |
| "loss": 2.7956, | |
| "theoretical_loss": 3.90288239805351, | |
| "tokens_seen": 514457600 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0004116701342916588, | |
| "loss": 2.7937, | |
| "theoretical_loss": 3.9002287717283783, | |
| "tokens_seen": 517734400 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0004110790618498203, | |
| "loss": 2.7995, | |
| "theoretical_loss": 3.8975965565242423, | |
| "tokens_seen": 521011200 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0004104879894079819, | |
| "loss": 2.7679, | |
| "theoretical_loss": 3.894985446642206, | |
| "tokens_seen": 524288000 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0004098969169661434, | |
| "loss": 2.7851, | |
| "theoretical_loss": 3.8923951425273398, | |
| "tokens_seen": 527564800 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0004093058445243049, | |
| "loss": 2.7783, | |
| "theoretical_loss": 3.889825350703344, | |
| "tokens_seen": 530841600 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.00040871477208246645, | |
| "loss": 2.8113, | |
| "theoretical_loss": 3.8872757836125844, | |
| "tokens_seen": 534118400 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.00040812369964062797, | |
| "loss": 2.7985, | |
| "theoretical_loss": 3.8847461594612884, | |
| "tokens_seen": 537395200 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0004075326271987895, | |
| "loss": 2.8391, | |
| "theoretical_loss": 3.882236202069703, | |
| "tokens_seen": 540672000 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.000406941554756951, | |
| "loss": 2.8309, | |
| "theoretical_loss": 3.8797456407270343, | |
| "tokens_seen": 543948800 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004063504823151126, | |
| "loss": 2.8188, | |
| "theoretical_loss": 3.8772742100509774, | |
| "tokens_seen": 547225600 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004057594098732741, | |
| "loss": 2.8115, | |
| "theoretical_loss": 3.874821649851678, | |
| "tokens_seen": 550502400 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004051683374314356, | |
| "loss": 2.8505, | |
| "theoretical_loss": 3.8723877049999444, | |
| "tokens_seen": 553779200 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.00040457726498959715, | |
| "loss": 2.7899, | |
| "theoretical_loss": 3.869972125299568, | |
| "tokens_seen": 557056000 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.00040398619254775867, | |
| "loss": 2.8405, | |
| "theoretical_loss": 3.867574665363595, | |
| "tokens_seen": 560332800 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004033951201059202, | |
| "loss": 2.8184, | |
| "theoretical_loss": 3.865195084494398, | |
| "tokens_seen": 563609600 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.00040280404766408176, | |
| "loss": 2.8252, | |
| "theoretical_loss": 3.8628331465674224, | |
| "tokens_seen": 566886400 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004022129752222433, | |
| "loss": 2.8865, | |
| "theoretical_loss": 3.860488619918462, | |
| "tokens_seen": 570163200 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004016219027804048, | |
| "loss": 2.8926, | |
| "theoretical_loss": 3.858161277234349, | |
| "tokens_seen": 573440000 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0004010308303385663, | |
| "loss": 2.8499, | |
| "theoretical_loss": 3.855850895446925, | |
| "tokens_seen": 576716800 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.00040043975789672785, | |
| "loss": 2.8563, | |
| "theoretical_loss": 3.8535572556301823, | |
| "tokens_seen": 579993600 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0003998486854548893, | |
| "loss": 2.8469, | |
| "theoretical_loss": 3.851280142900463, | |
| "tokens_seen": 583270400 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0003992576130130509, | |
| "loss": 2.8482, | |
| "theoretical_loss": 3.8490193463196127, | |
| "tokens_seen": 586547200 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0003986665405712124, | |
| "loss": 2.8413, | |
| "theoretical_loss": 3.8467746588009692, | |
| "tokens_seen": 589824000 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.00039807546812937393, | |
| "loss": 2.846, | |
| "theoretical_loss": 3.8445458770181116, | |
| "tokens_seen": 593100800 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.00039748439568753545, | |
| "loss": 2.8096, | |
| "theoretical_loss": 3.842332801316254, | |
| "tokens_seen": 596377600 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.00039689332324569697, | |
| "loss": 2.81, | |
| "theoretical_loss": 3.840135235626204, | |
| "tokens_seen": 599654400 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0003963022508038585, | |
| "loss": 2.7778, | |
| "theoretical_loss": 3.8379529873807945, | |
| "tokens_seen": 602931200 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.00039571117836202, | |
| "loss": 2.8046, | |
| "theoretical_loss": 3.835785867433705, | |
| "tokens_seen": 606208000 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0003951201059201816, | |
| "loss": 2.7609, | |
| "theoretical_loss": 3.8336336899805916, | |
| "tokens_seen": 609484800 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0003945290334783431, | |
| "loss": 2.7938, | |
| "theoretical_loss": 3.8314962724824495, | |
| "tokens_seen": 612761600 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.00039393796103650463, | |
| "loss": 2.8098, | |
| "theoretical_loss": 3.8293734355911297, | |
| "tokens_seen": 616038400 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.00039334688859466615, | |
| "loss": 2.7622, | |
| "theoretical_loss": 3.8272650030769433, | |
| "tokens_seen": 619315200 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.00039275581615282767, | |
| "loss": 2.7791, | |
| "theoretical_loss": 3.8251708017582793, | |
| "tokens_seen": 622592000 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0003921647437109892, | |
| "loss": 2.7272, | |
| "theoretical_loss": 3.8230906614331737, | |
| "tokens_seen": 625868800 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.00039157367126915077, | |
| "loss": 2.7712, | |
| "theoretical_loss": 3.821024414812761, | |
| "tokens_seen": 629145600 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0003909825988273123, | |
| "loss": 2.7899, | |
| "theoretical_loss": 3.818971897456552, | |
| "tokens_seen": 632422400 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0003903915263854738, | |
| "loss": 2.8311, | |
| "theoretical_loss": 3.8169329477094784, | |
| "tokens_seen": 635699200 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.00038980045394363533, | |
| "loss": 2.8424, | |
| "theoretical_loss": 3.814907406640639, | |
| "tokens_seen": 638976000 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.00038920938150179685, | |
| "loss": 2.8391, | |
| "theoretical_loss": 3.8128951179837056, | |
| "tokens_seen": 642252800 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.00038861830905995837, | |
| "loss": 2.8258, | |
| "theoretical_loss": 3.8108959280789243, | |
| "tokens_seen": 645529600 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.00038802723661811994, | |
| "loss": 2.8064, | |
| "theoretical_loss": 3.8089096858166718, | |
| "tokens_seen": 648806400 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.00038743616417628147, | |
| "loss": 2.8018, | |
| "theoretical_loss": 3.8069362425825037, | |
| "tokens_seen": 652083200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.5495775421806781, | |
| "debugging/distinct-1-grams": 0.7858205702842564, | |
| "debugging/distinct-2-grams": 0.9509135221332653, | |
| "debugging/entropy-1-grams": 6.135781796217266, | |
| "debugging/entropy-2-grams": 7.025177223786267, | |
| "debugging/length": 509.3888888888889, | |
| "debugging/num_segments": 18, | |
| "epoch": 0.23, | |
| "objective/train/docs_used": 369170, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.964120864868164, | |
| "objective/train/theoretical_loss": 3.8049754522036645, | |
| "objective/train/tokens_used": 675820000, | |
| "theoretical_loss": 3.8049754522036645, | |
| "tokens_seen": 655360000 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.000386845091734443, | |
| "loss": 2.7933, | |
| "theoretical_loss": 3.8049754522036645, | |
| "tokens_seen": 655360000 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003862540192926045, | |
| "loss": 2.8017, | |
| "theoretical_loss": 3.8030271708970003, | |
| "tokens_seen": 658636800 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.00038566294685076603, | |
| "loss": 2.7857, | |
| "theoretical_loss": 3.801091257218237, | |
| "tokens_seen": 661913600 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.00038507187440892755, | |
| "loss": 2.7687, | |
| "theoretical_loss": 3.79916757201258, | |
| "tokens_seen": 665190400 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003844926234159259, | |
| "loss": 2.7757, | |
| "theoretical_loss": 3.7972559783665965, | |
| "tokens_seen": 668467200 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003839015509740874, | |
| "loss": 2.829, | |
| "theoretical_loss": 3.7953563415613325, | |
| "tokens_seen": 671744000 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003833104785322489, | |
| "loss": 2.8231, | |
| "theoretical_loss": 3.7934685290266454, | |
| "tokens_seen": 675020800 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.00038271940609041044, | |
| "loss": 2.8413, | |
| "theoretical_loss": 3.7915924102966914, | |
| "tokens_seen": 678297600 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.00038212833364857196, | |
| "loss": 2.8301, | |
| "theoretical_loss": 3.789727856966552, | |
| "tokens_seen": 681574400 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003815372612067335, | |
| "loss": 2.8133, | |
| "theoretical_loss": 3.787874742649958, | |
| "tokens_seen": 684851200 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.000380946188764895, | |
| "loss": 2.8341, | |
| "theoretical_loss": 3.786032942938073, | |
| "tokens_seen": 688128000 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0003803551163230566, | |
| "loss": 2.8079, | |
| "theoretical_loss": 3.784202335359316, | |
| "tokens_seen": 691404800 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0003797640438812181, | |
| "loss": 2.812, | |
| "theoretical_loss": 3.7823827993401844, | |
| "tokens_seen": 694681600 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0003791729714393796, | |
| "loss": 2.7577, | |
| "theoretical_loss": 3.7805742161670466, | |
| "tokens_seen": 697958400 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.00037858189899754114, | |
| "loss": 2.7862, | |
| "theoretical_loss": 3.7787764689488847, | |
| "tokens_seen": 701235200 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.00037799082655570266, | |
| "loss": 2.7499, | |
| "theoretical_loss": 3.77698944258095, | |
| "tokens_seen": 704512000 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0003773997541138642, | |
| "loss": 2.7967, | |
| "theoretical_loss": 3.7752130237093064, | |
| "tokens_seen": 707788800 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.00037680868167202575, | |
| "loss": 2.7918, | |
| "theoretical_loss": 3.773447100696245, | |
| "tokens_seen": 711065600 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0003762176092301873, | |
| "loss": 2.8046, | |
| "theoretical_loss": 3.771691563586529, | |
| "tokens_seen": 714342400 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0003756265367883488, | |
| "loss": 2.7732, | |
| "theoretical_loss": 3.7699463040744616, | |
| "tokens_seen": 717619200 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0003750354643465103, | |
| "loss": 2.7373, | |
| "theoretical_loss": 3.768211215471741, | |
| "tokens_seen": 720896000 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.00037444439190467184, | |
| "loss": 2.7512, | |
| "theoretical_loss": 3.766486192676084, | |
| "tokens_seen": 724172800 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.00037385331946283336, | |
| "loss": 2.7115, | |
| "theoretical_loss": 3.764771132140602, | |
| "tokens_seen": 727449600 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.00037326224702099493, | |
| "loss": 2.7214, | |
| "theoretical_loss": 3.763065931843898, | |
| "tokens_seen": 730726400 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.00037267117457915645, | |
| "loss": 2.7296, | |
| "theoretical_loss": 3.7613704912608723, | |
| "tokens_seen": 734003200 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.000372080102137318, | |
| "loss": 2.7715, | |
| "theoretical_loss": 3.759684711334213, | |
| "tokens_seen": 737280000 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0003714890296954795, | |
| "loss": 2.794, | |
| "theoretical_loss": 3.7580084944465555, | |
| "tokens_seen": 740556800 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.000370897957253641, | |
| "loss": 2.785, | |
| "theoretical_loss": 3.7563417443932905, | |
| "tokens_seen": 743833600 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.00037030688481180254, | |
| "loss": 2.7882, | |
| "theoretical_loss": 3.754684366355999, | |
| "tokens_seen": 747110400 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.00036971581236996406, | |
| "loss": 2.7543, | |
| "theoretical_loss": 3.753036266876505, | |
| "tokens_seen": 750387200 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.00036912473992812563, | |
| "loss": 2.7509, | |
| "theoretical_loss": 3.751397353831524, | |
| "tokens_seen": 753664000 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.00036853366748628715, | |
| "loss": 2.7483, | |
| "theoretical_loss": 3.749767536407891, | |
| "tokens_seen": 756940800 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0003679425950444487, | |
| "loss": 2.7265, | |
| "theoretical_loss": 3.7481467250783504, | |
| "tokens_seen": 760217600 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0003673515226026102, | |
| "loss": 2.7443, | |
| "theoretical_loss": 3.746534831577904, | |
| "tokens_seen": 763494400 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0003667604501607717, | |
| "loss": 2.6939, | |
| "theoretical_loss": 3.744931768880681, | |
| "tokens_seen": 766771200 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.00036616937771893324, | |
| "loss": 2.7158, | |
| "theoretical_loss": 3.743337451177343, | |
| "tokens_seen": 770048000 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0003655783052770948, | |
| "loss": 2.7014, | |
| "theoretical_loss": 3.74175179385298, | |
| "tokens_seen": 773324800 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.00036498723283525633, | |
| "loss": 2.7803, | |
| "theoretical_loss": 3.740174713465512, | |
| "tokens_seen": 776601600 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.00036439616039341785, | |
| "loss": 2.7231, | |
| "theoretical_loss": 3.7386061277245655, | |
| "tokens_seen": 779878400 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0003638050879515794, | |
| "loss": 2.7733, | |
| "theoretical_loss": 3.7370459554708146, | |
| "tokens_seen": 783155200 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0003632140155097409, | |
| "loss": 2.804, | |
| "theoretical_loss": 3.735494116655784, | |
| "tokens_seen": 786432000 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0003626229430679024, | |
| "loss": 2.7704, | |
| "theoretical_loss": 3.733950532322087, | |
| "tokens_seen": 789708800 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.000362031870626064, | |
| "loss": 2.7873, | |
| "theoretical_loss": 3.7324151245841044, | |
| "tokens_seen": 792985600 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.00036144079818422546, | |
| "loss": 2.7787, | |
| "theoretical_loss": 3.730887816609077, | |
| "tokens_seen": 796262400 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.000360849725742387, | |
| "loss": 2.7795, | |
| "theoretical_loss": 3.729368532598609, | |
| "tokens_seen": 799539200 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0003602586533005485, | |
| "loss": 2.7645, | |
| "theoretical_loss": 3.7278571977705734, | |
| "tokens_seen": 802816000 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.00035966758085871, | |
| "loss": 2.7904, | |
| "theoretical_loss": 3.7263537383414023, | |
| "tokens_seen": 806092800 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.00035907650841687154, | |
| "loss": 2.7791, | |
| "theoretical_loss": 3.7248580815087626, | |
| "tokens_seen": 809369600 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.00035848543597503306, | |
| "loss": 2.7984, | |
| "theoretical_loss": 3.7233701554345924, | |
| "tokens_seen": 812646400 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.00035790618498203144, | |
| "loss": 2.7796, | |
| "theoretical_loss": 3.721889889228506, | |
| "tokens_seen": 815923200 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "objective/train/docs_used": 458185, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 3.0822372436523438, | |
| "objective/train/theoretical_loss": 3.720417212931543, | |
| "objective/train/tokens_used": 839660000, | |
| "theoretical_loss": 3.720417212931543, | |
| "tokens_seen": 819200000 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0003573269339890297, | |
| "loss": 2.761, | |
| "theoretical_loss": 3.720417212931543, | |
| "tokens_seen": 819200000 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.00035673586154719123, | |
| "loss": 2.7429, | |
| "theoretical_loss": 3.7189520575002666, | |
| "tokens_seen": 822476800 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.00035614478910535275, | |
| "loss": 2.7242, | |
| "theoretical_loss": 3.717494354791188, | |
| "tokens_seen": 825753600 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0003555537166635143, | |
| "loss": 2.7352, | |
| "theoretical_loss": 3.716044037545523, | |
| "tokens_seen": 829030400 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0003549626442216758, | |
| "loss": 2.7653, | |
| "theoretical_loss": 3.714601039374263, | |
| "tokens_seen": 832307200 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.00035437157177983737, | |
| "loss": 2.7024, | |
| "theoretical_loss": 3.7131652947435536, | |
| "tokens_seen": 835584000 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0003537804993379989, | |
| "loss": 2.7041, | |
| "theoretical_loss": 3.7117367389603793, | |
| "tokens_seen": 838860800 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0003531894268961604, | |
| "loss": 2.7092, | |
| "theoretical_loss": 3.710315308158541, | |
| "tokens_seen": 842137600 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.00035259835445432193, | |
| "loss": 2.7506, | |
| "theoretical_loss": 3.7089009392849173, | |
| "tokens_seen": 845414400 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.00035200728201248345, | |
| "loss": 2.7066, | |
| "theoretical_loss": 3.7074935700860143, | |
| "tokens_seen": 848691200 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.000351416209570645, | |
| "loss": 2.7087, | |
| "theoretical_loss": 3.706093139094781, | |
| "tokens_seen": 851968000 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00035082513712880655, | |
| "loss": 2.7168, | |
| "theoretical_loss": 3.7046995856176954, | |
| "tokens_seen": 855244800 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00035023406468696807, | |
| "loss": 2.76, | |
| "theoretical_loss": 3.703312849722111, | |
| "tokens_seen": 858521600 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.0003496429922451296, | |
| "loss": 2.7042, | |
| "theoretical_loss": 3.701932872223858, | |
| "tokens_seen": 861798400 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.0003490519198032911, | |
| "loss": 2.6904, | |
| "theoretical_loss": 3.7005595946750924, | |
| "tokens_seen": 865075200 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00034846084736145263, | |
| "loss": 2.7039, | |
| "theoretical_loss": 3.699192959352386, | |
| "tokens_seen": 868352000 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00034786977491961415, | |
| "loss": 2.6842, | |
| "theoretical_loss": 3.6978329092450557, | |
| "tokens_seen": 871628800 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00034727870247777573, | |
| "loss": 2.7035, | |
| "theoretical_loss": 3.6964793880437226, | |
| "tokens_seen": 874905600 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00034668763003593725, | |
| "loss": 2.7266, | |
| "theoretical_loss": 3.6951323401290974, | |
| "tokens_seen": 878182400 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00034609655759409877, | |
| "loss": 2.7197, | |
| "theoretical_loss": 3.6937917105609834, | |
| "tokens_seen": 881459200 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0003455054851522603, | |
| "loss": 2.7329, | |
| "theoretical_loss": 3.692457445067501, | |
| "tokens_seen": 884736000 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0003449144127104218, | |
| "loss": 2.735, | |
| "theoretical_loss": 3.6911294900345166, | |
| "tokens_seen": 888012800 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.00034432334026858333, | |
| "loss": 2.6776, | |
| "theoretical_loss": 3.6898077924952775, | |
| "tokens_seen": 891289600 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.00034373226782674485, | |
| "loss": 2.725, | |
| "theoretical_loss": 3.6884923001202505, | |
| "tokens_seen": 894566400 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.00034314119538490643, | |
| "loss": 2.6866, | |
| "theoretical_loss": 3.6871829612071583, | |
| "tokens_seen": 897843200 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0003425501229430679, | |
| "loss": 2.6789, | |
| "theoretical_loss": 3.6858797246711976, | |
| "tokens_seen": 901120000 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0003419590505012294, | |
| "loss": 2.6709, | |
| "theoretical_loss": 3.684582540035456, | |
| "tokens_seen": 904396800 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.00034136797805939094, | |
| "loss": 2.6846, | |
| "theoretical_loss": 3.683291357421508, | |
| "tokens_seen": 907673600 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00034077690561755246, | |
| "loss": 2.7284, | |
| "theoretical_loss": 3.682006127540184, | |
| "tokens_seen": 910950400 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.000340185833175714, | |
| "loss": 2.7195, | |
| "theoretical_loss": 3.680726801682522, | |
| "tokens_seen": 914227200 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00033959476073387555, | |
| "loss": 2.7211, | |
| "theoretical_loss": 3.679453331710889, | |
| "tokens_seen": 917504000 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0003390036882920371, | |
| "loss": 2.7229, | |
| "theoretical_loss": 3.6781856700502646, | |
| "tokens_seen": 920780800 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0003384126158501986, | |
| "loss": 2.7281, | |
| "theoretical_loss": 3.6769237696796933, | |
| "tokens_seen": 924057600 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0003378215434083601, | |
| "loss": 2.7212, | |
| "theoretical_loss": 3.6756675841238913, | |
| "tokens_seen": 927334400 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00033723047096652164, | |
| "loss": 2.7393, | |
| "theoretical_loss": 3.6744170674450176, | |
| "tokens_seen": 930611200 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00033663939852468316, | |
| "loss": 2.7324, | |
| "theoretical_loss": 3.673172174234587, | |
| "tokens_seen": 933888000 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00033604832608284473, | |
| "loss": 2.7316, | |
| "theoretical_loss": 3.6719328596055423, | |
| "tokens_seen": 937164800 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00033545725364100625, | |
| "loss": 2.7062, | |
| "theoretical_loss": 3.670699079184467, | |
| "tokens_seen": 940441600 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0003348661811991678, | |
| "loss": 2.7195, | |
| "theoretical_loss": 3.669470789103942, | |
| "tokens_seen": 943718400 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0003342751087573293, | |
| "loss": 2.7238, | |
| "theoretical_loss": 3.6682479459950446, | |
| "tokens_seen": 946995200 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0003336840363154908, | |
| "loss": 2.6712, | |
| "theoretical_loss": 3.6670305069799785, | |
| "tokens_seen": 950272000 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00033309296387365234, | |
| "loss": 2.6672, | |
| "theoretical_loss": 3.6658184296648457, | |
| "tokens_seen": 953548800 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00033250189143181386, | |
| "loss": 2.6387, | |
| "theoretical_loss": 3.6646116721325415, | |
| "tokens_seen": 956825600 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00033191081898997543, | |
| "loss": 2.6132, | |
| "theoretical_loss": 3.6634101929357836, | |
| "tokens_seen": 960102400 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0003313315679969737, | |
| "loss": 2.6409, | |
| "theoretical_loss": 3.6622139510902625, | |
| "tokens_seen": 963379200 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003307404955551352, | |
| "loss": 2.6332, | |
| "theoretical_loss": 3.6610229060679167, | |
| "tokens_seen": 966656000 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00033014942311329675, | |
| "loss": 2.6786, | |
| "theoretical_loss": 3.659837017790328, | |
| "tokens_seen": 969932800 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00032955835067145827, | |
| "loss": 2.683, | |
| "theoretical_loss": 3.658656246622233, | |
| "tokens_seen": 973209600 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003289672782296198, | |
| "loss": 2.7089, | |
| "theoretical_loss": 3.6574805533651515, | |
| "tokens_seen": 976486400 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00032837620578778136, | |
| "loss": 2.6247, | |
| "theoretical_loss": 3.6563098992511267, | |
| "tokens_seen": 979763200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.44848857539964976, | |
| "debugging/distinct-1-grams": 0.7781029350046207, | |
| "debugging/distinct-2-grams": 0.9575994862285143, | |
| "debugging/entropy-1-grams": 5.851953292865711, | |
| "debugging/entropy-2-grams": 6.601563630419801, | |
| "debugging/length": 518.3636363636364, | |
| "debugging/num_segments": 11, | |
| "epoch": 0.35, | |
| "objective/train/docs_used": 548797, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.4279284477233887, | |
| "objective/train/theoretical_loss": 3.655144245936574, | |
| "objective/train/tokens_used": 1003500000, | |
| "theoretical_loss": 3.655144245936574, | |
| "tokens_seen": 983040000 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003277851333459429, | |
| "loss": 2.7072, | |
| "theoretical_loss": 3.655144245936574, | |
| "tokens_seen": 983040000 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003271940609041044, | |
| "loss": 2.6516, | |
| "theoretical_loss": 3.653983555496242, | |
| "tokens_seen": 986316800 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003266029884622659, | |
| "loss": 2.6659, | |
| "theoretical_loss": 3.6528277904172755, | |
| "tokens_seen": 989593600 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00032601191602042745, | |
| "loss": 2.705, | |
| "theoretical_loss": 3.6516769135933815, | |
| "tokens_seen": 992870400 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.00032542084357858897, | |
| "loss": 2.7945, | |
| "theoretical_loss": 3.650530888319103, | |
| "tokens_seen": 996147200 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.00032482977113675054, | |
| "loss": 2.7835, | |
| "theoretical_loss": 3.649389678284182, | |
| "tokens_seen": 999424000 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.00032423869869491206, | |
| "loss": 2.803, | |
| "theoretical_loss": 3.6482532475680287, | |
| "tokens_seen": 1002700800 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0003236476262530736, | |
| "loss": 2.8145, | |
| "theoretical_loss": 3.6471215606342833, | |
| "tokens_seen": 1005977600 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.00032306837526007185, | |
| "loss": 2.7892, | |
| "theoretical_loss": 3.645994582325468, | |
| "tokens_seen": 1009254400 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0003224773028182334, | |
| "loss": 2.7912, | |
| "theoretical_loss": 3.6448722778577327, | |
| "tokens_seen": 1012531200 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0003218862303763949, | |
| "loss": 2.7827, | |
| "theoretical_loss": 3.6437546128156946, | |
| "tokens_seen": 1015808000 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.00032129515793455647, | |
| "loss": 2.739, | |
| "theoretical_loss": 3.6426415531473566, | |
| "tokens_seen": 1019084800 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.000320704085492718, | |
| "loss": 2.7624, | |
| "theoretical_loss": 3.641533065159118, | |
| "tokens_seen": 1022361600 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0003201130130508795, | |
| "loss": 2.7213, | |
| "theoretical_loss": 3.6404291155108712, | |
| "tokens_seen": 1025638400 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00031952194060904103, | |
| "loss": 2.7456, | |
| "theoretical_loss": 3.639329671211173, | |
| "tokens_seen": 1028915200 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00031893086816720255, | |
| "loss": 2.7553, | |
| "theoretical_loss": 3.6382346996125055, | |
| "tokens_seen": 1032192000 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0003183397957253641, | |
| "loss": 2.7371, | |
| "theoretical_loss": 3.6371441684066097, | |
| "tokens_seen": 1035468800 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0003177487232835256, | |
| "loss": 2.761, | |
| "theoretical_loss": 3.6360580456199036, | |
| "tokens_seen": 1038745600 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00031715765084168717, | |
| "loss": 2.7745, | |
| "theoretical_loss": 3.6349762996089683, | |
| "tokens_seen": 1042022400 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0003165665783998487, | |
| "loss": 2.7151, | |
| "theoretical_loss": 3.633898899056115, | |
| "tokens_seen": 1045299200 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0003159755059580102, | |
| "loss": 2.7723, | |
| "theoretical_loss": 3.6328258129650246, | |
| "tokens_seen": 1048576000 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.00031538443351617173, | |
| "loss": 2.748, | |
| "theoretical_loss": 3.6317570106564565, | |
| "tokens_seen": 1051852800 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.00031479336107433325, | |
| "loss": 2.7319, | |
| "theoretical_loss": 3.6306924617640295, | |
| "tokens_seen": 1055129600 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0003142022886324948, | |
| "loss": 2.7559, | |
| "theoretical_loss": 3.6296321362300716, | |
| "tokens_seen": 1058406400 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.00031361121619065635, | |
| "loss": 2.7457, | |
| "theoretical_loss": 3.6285760043015385, | |
| "tokens_seen": 1061683200 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.00031302014374881787, | |
| "loss": 2.7373, | |
| "theoretical_loss": 3.6275240365259958, | |
| "tokens_seen": 1064960000 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0003124290713069794, | |
| "loss": 2.693, | |
| "theoretical_loss": 3.6264762037476683, | |
| "tokens_seen": 1068236800 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0003118379988651409, | |
| "loss": 2.6899, | |
| "theoretical_loss": 3.625432477103554, | |
| "tokens_seen": 1071513600 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.00031124692642330243, | |
| "loss": 2.6807, | |
| "theoretical_loss": 3.6243928280195976, | |
| "tokens_seen": 1074790400 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00031065585398146395, | |
| "loss": 2.6832, | |
| "theoretical_loss": 3.62335722820693, | |
| "tokens_seen": 1078067200 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00031006478153962553, | |
| "loss": 2.6918, | |
| "theoretical_loss": 3.6223256496581637, | |
| "tokens_seen": 1081344000 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00030947370909778705, | |
| "loss": 2.7096, | |
| "theoretical_loss": 3.6212980646437485, | |
| "tokens_seen": 1084620800 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00030888263665594857, | |
| "loss": 2.6864, | |
| "theoretical_loss": 3.6202744457083877, | |
| "tokens_seen": 1087897600 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003082915642141101, | |
| "loss": 2.7229, | |
| "theoretical_loss": 3.6192547656675083, | |
| "tokens_seen": 1091174400 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003077004917722716, | |
| "loss": 2.7171, | |
| "theoretical_loss": 3.618238997603788, | |
| "tokens_seen": 1094451200 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00030710941933043313, | |
| "loss": 2.7049, | |
| "theoretical_loss": 3.617227114863738, | |
| "tokens_seen": 1097728000 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00030651834688859465, | |
| "loss": 2.6875, | |
| "theoretical_loss": 3.6162190910543366, | |
| "tokens_seen": 1101004800 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00030592727444675623, | |
| "loss": 2.6679, | |
| "theoretical_loss": 3.615214900039721, | |
| "tokens_seen": 1104281600 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00030533620200491775, | |
| "loss": 2.6854, | |
| "theoretical_loss": 3.614214515937924, | |
| "tokens_seen": 1107558400 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00030474512956307927, | |
| "loss": 2.6916, | |
| "theoretical_loss": 3.613217913117667, | |
| "tokens_seen": 1110835200 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0003041540571212408, | |
| "loss": 2.6977, | |
| "theoretical_loss": 3.612225066195201, | |
| "tokens_seen": 1114112000 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0003035629846794023, | |
| "loss": 2.7172, | |
| "theoretical_loss": 3.611235950031194, | |
| "tokens_seen": 1117388800 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00030297191223756383, | |
| "loss": 2.7051, | |
| "theoretical_loss": 3.6102505397276743, | |
| "tokens_seen": 1120665600 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0003023808397957254, | |
| "loss": 2.687, | |
| "theoretical_loss": 3.60926881062501, | |
| "tokens_seen": 1123942400 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00030178976735388693, | |
| "loss": 2.6438, | |
| "theoretical_loss": 3.608290738298942, | |
| "tokens_seen": 1127219200 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00030119869491204845, | |
| "loss": 2.6615, | |
| "theoretical_loss": 3.6073162985576643, | |
| "tokens_seen": 1130496000 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00030060762247020997, | |
| "loss": 2.6684, | |
| "theoretical_loss": 3.606345467438941, | |
| "tokens_seen": 1133772800 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0003000165500283715, | |
| "loss": 2.6597, | |
| "theoretical_loss": 3.6053782212072747, | |
| "tokens_seen": 1137049600 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.000299425477586533, | |
| "loss": 2.6721, | |
| "theoretical_loss": 3.604414536351113, | |
| "tokens_seen": 1140326400 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0002988344051446946, | |
| "loss": 2.6791, | |
| "theoretical_loss": 3.6034543895801017, | |
| "tokens_seen": 1143603200 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "objective/train/docs_used": 634175, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.4255619049072266, | |
| "objective/train/theoretical_loss": 3.6024977578223742, | |
| "objective/train/tokens_used": 1167340000, | |
| "theoretical_loss": 3.6024977578223742, | |
| "tokens_seen": 1146880000 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0002982433327028561, | |
| "loss": 2.6508, | |
| "theoretical_loss": 3.6024977578223742, | |
| "tokens_seen": 1146880000 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00029765226026101763, | |
| "loss": 2.7451, | |
| "theoretical_loss": 3.6015446182218875, | |
| "tokens_seen": 1150156800 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00029706118781917915, | |
| "loss": 2.7231, | |
| "theoretical_loss": 3.600594948135793, | |
| "tokens_seen": 1153433600 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00029647011537734067, | |
| "loss": 2.6868, | |
| "theoretical_loss": 3.59964872513185, | |
| "tokens_seen": 1156710400 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00029587904293550214, | |
| "loss": 2.7225, | |
| "theoretical_loss": 3.5987059269858763, | |
| "tokens_seen": 1159987200 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00029528797049366366, | |
| "loss": 2.7105, | |
| "theoretical_loss": 3.5977665316792375, | |
| "tokens_seen": 1163264000 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00029469689805182523, | |
| "loss": 2.6838, | |
| "theoretical_loss": 3.5968305173963744, | |
| "tokens_seen": 1166540800 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00029410582560998675, | |
| "loss": 2.6623, | |
| "theoretical_loss": 3.5958978625223628, | |
| "tokens_seen": 1169817600 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0002935147531681483, | |
| "loss": 2.6717, | |
| "theoretical_loss": 3.5949685456405165, | |
| "tokens_seen": 1173094400 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0002929236807263098, | |
| "loss": 2.6992, | |
| "theoretical_loss": 3.5940425455300176, | |
| "tokens_seen": 1176371200 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0002923326082844713, | |
| "loss": 2.6954, | |
| "theoretical_loss": 3.593119841163589, | |
| "tokens_seen": 1179648000 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00029174153584263284, | |
| "loss": 2.6819, | |
| "theoretical_loss": 3.5922004117051944, | |
| "tokens_seen": 1182924800 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0002911504634007944, | |
| "loss": 2.6886, | |
| "theoretical_loss": 3.5912842365077777, | |
| "tokens_seen": 1186201600 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00029055939095895593, | |
| "loss": 2.6893, | |
| "theoretical_loss": 3.5903712951110305, | |
| "tokens_seen": 1189478400 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.00028996831851711745, | |
| "loss": 2.6813, | |
| "theoretical_loss": 3.5894615672391947, | |
| "tokens_seen": 1192755200 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.000289377246075279, | |
| "loss": 2.7106, | |
| "theoretical_loss": 3.5885550327988973, | |
| "tokens_seen": 1196032000 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0002887861736334405, | |
| "loss": 2.6777, | |
| "theoretical_loss": 3.587651671877014, | |
| "tokens_seen": 1199308800 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.000288195101191602, | |
| "loss": 2.6959, | |
| "theoretical_loss": 3.5867514647385663, | |
| "tokens_seen": 1202585600 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0002876040287497636, | |
| "loss": 2.6972, | |
| "theoretical_loss": 3.585854391824647, | |
| "tokens_seen": 1205862400 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0002870129563079251, | |
| "loss": 2.6602, | |
| "theoretical_loss": 3.584960433750375, | |
| "tokens_seen": 1209139200 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.00028642188386608663, | |
| "loss": 2.6846, | |
| "theoretical_loss": 3.5840695713028827, | |
| "tokens_seen": 1212416000 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.00028583081142424815, | |
| "loss": 2.6813, | |
| "theoretical_loss": 3.5831817854393266, | |
| "tokens_seen": 1215692800 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00028523973898240967, | |
| "loss": 2.6898, | |
| "theoretical_loss": 3.582297057284933, | |
| "tokens_seen": 1218969600 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002846486665405712, | |
| "loss": 2.7131, | |
| "theoretical_loss": 3.5814153681310623, | |
| "tokens_seen": 1222246400 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002840575940987327, | |
| "loss": 2.7068, | |
| "theoretical_loss": 3.5805366994333125, | |
| "tokens_seen": 1225523200 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002834665216568943, | |
| "loss": 2.6921, | |
| "theoretical_loss": 3.5796610328096365, | |
| "tokens_seen": 1228800000 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002828754492150558, | |
| "loss": 2.6975, | |
| "theoretical_loss": 3.578788350038497, | |
| "tokens_seen": 1232076800 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00028228437677321733, | |
| "loss": 2.6815, | |
| "theoretical_loss": 3.5779186330570405, | |
| "tokens_seen": 1235353600 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00028169330433137885, | |
| "loss": 2.6759, | |
| "theoretical_loss": 3.5770518639592983, | |
| "tokens_seen": 1238630400 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00028110223188954037, | |
| "loss": 2.7393, | |
| "theoretical_loss": 3.5761880249944147, | |
| "tokens_seen": 1241907200 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002805111594477019, | |
| "loss": 2.7223, | |
| "theoretical_loss": 3.5753270985648973, | |
| "tokens_seen": 1245184000 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00027992008700586347, | |
| "loss": 2.716, | |
| "theoretical_loss": 3.574469067224892, | |
| "tokens_seen": 1248460800 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.000279329014564025, | |
| "loss": 2.7334, | |
| "theoretical_loss": 3.573613913678484, | |
| "tokens_seen": 1251737600 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.0002787379421221865, | |
| "loss": 2.7213, | |
| "theoretical_loss": 3.57276162077802, | |
| "tokens_seen": 1255014400 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00027814686968034803, | |
| "loss": 2.712, | |
| "theoretical_loss": 3.5719121715224524, | |
| "tokens_seen": 1258291200 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00027755579723850955, | |
| "loss": 2.6879, | |
| "theoretical_loss": 3.571065549055712, | |
| "tokens_seen": 1261568000 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00027696472479667107, | |
| "loss": 2.6854, | |
| "theoretical_loss": 3.5702217366650935, | |
| "tokens_seen": 1264844800 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00027637365235483265, | |
| "loss": 2.6818, | |
| "theoretical_loss": 3.5693807177796737, | |
| "tokens_seen": 1268121600 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00027578257991299417, | |
| "loss": 2.6228, | |
| "theoretical_loss": 3.5685424759687434, | |
| "tokens_seen": 1271398400 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0002751915074711557, | |
| "loss": 2.6779, | |
| "theoretical_loss": 3.567706994940263, | |
| "tokens_seen": 1274675200 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0002746004350293172, | |
| "loss": 2.6876, | |
| "theoretical_loss": 3.5668742585393405, | |
| "tokens_seen": 1277952000 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00027400936258747873, | |
| "loss": 2.6531, | |
| "theoretical_loss": 3.566044250746728, | |
| "tokens_seen": 1281228800 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00027341829014564025, | |
| "loss": 2.6545, | |
| "theoretical_loss": 3.5652169556773403, | |
| "tokens_seen": 1284505600 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0002728272177038018, | |
| "loss": 2.6209, | |
| "theoretical_loss": 3.5643923575787912, | |
| "tokens_seen": 1287782400 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00027223614526196335, | |
| "loss": 2.6529, | |
| "theoretical_loss": 3.563570440829951, | |
| "tokens_seen": 1291059200 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00027164507282012487, | |
| "loss": 2.6148, | |
| "theoretical_loss": 3.562751189939524, | |
| "tokens_seen": 1294336000 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0002710540003782864, | |
| "loss": 2.6728, | |
| "theoretical_loss": 3.5619345895446424, | |
| "tokens_seen": 1297612800 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0002704629279364479, | |
| "loss": 2.6349, | |
| "theoretical_loss": 3.561120624409482, | |
| "tokens_seen": 1300889600 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00026987185549460943, | |
| "loss": 2.6384, | |
| "theoretical_loss": 3.560309279423894, | |
| "tokens_seen": 1304166400 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00026928078305277095, | |
| "loss": 2.6076, | |
| "theoretical_loss": 3.5595005396020554, | |
| "tokens_seen": 1307443200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.5082619340855203, | |
| "debugging/distinct-1-grams": 0.7794681642048736, | |
| "debugging/distinct-2-grams": 0.9351864119489364, | |
| "debugging/entropy-1-grams": 6.068338078693592, | |
| "debugging/entropy-2-grams": 6.8266304879748, | |
| "debugging/length": 505.06666666666666, | |
| "debugging/num_segments": 15, | |
| "epoch": 0.47, | |
| "objective/train/docs_used": 722411, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.4245951175689697, | |
| "objective/train/theoretical_loss": 3.558694390081137, | |
| "objective/train/tokens_used": 1331180000, | |
| "theoretical_loss": 3.558694390081137, | |
| "tokens_seen": 1310720000 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0002686897106109325, | |
| "loss": 2.6127, | |
| "theoretical_loss": 3.558694390081137, | |
| "tokens_seen": 1310720000 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00026809863816909405, | |
| "loss": 2.5874, | |
| "theoretical_loss": 3.5578908161199934, | |
| "tokens_seen": 1313996800 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00026750756572725557, | |
| "loss": 2.6153, | |
| "theoretical_loss": 3.5570898030978584, | |
| "tokens_seen": 1317273600 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0002669164932854171, | |
| "loss": 2.6265, | |
| "theoretical_loss": 3.556291336513074, | |
| "tokens_seen": 1320550400 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00026632542084357855, | |
| "loss": 2.6162, | |
| "theoretical_loss": 3.5554954019818235, | |
| "tokens_seen": 1323827200 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0002657343484017401, | |
| "loss": 2.612, | |
| "theoretical_loss": 3.554701985236883, | |
| "tokens_seen": 1327104000 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026514327595990165, | |
| "loss": 2.6526, | |
| "theoretical_loss": 3.553911072126394, | |
| "tokens_seen": 1330380800 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026455220351806317, | |
| "loss": 2.6474, | |
| "theoretical_loss": 3.5531226486126504, | |
| "tokens_seen": 1333657600 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.0002639611310762247, | |
| "loss": 2.6462, | |
| "theoretical_loss": 3.552336700770896, | |
| "tokens_seen": 1336934400 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.0002633700586343862, | |
| "loss": 2.6368, | |
| "theoretical_loss": 3.5515532147881443, | |
| "tokens_seen": 1340211200 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026277898619254773, | |
| "loss": 2.6594, | |
| "theoretical_loss": 3.5507721769620098, | |
| "tokens_seen": 1343488000 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026218791375070925, | |
| "loss": 2.6582, | |
| "theoretical_loss": 3.549993573699556, | |
| "tokens_seen": 1346764800 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026159684130887083, | |
| "loss": 2.6446, | |
| "theoretical_loss": 3.5492173915161565, | |
| "tokens_seen": 1350041600 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026100576886703235, | |
| "loss": 2.6281, | |
| "theoretical_loss": 3.548443617034371, | |
| "tokens_seen": 1353318400 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026041469642519387, | |
| "loss": 2.6538, | |
| "theoretical_loss": 3.547672236982839, | |
| "tokens_seen": 1356595200 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.0002598236239833554, | |
| "loss": 2.6518, | |
| "theoretical_loss": 3.5469032381951804, | |
| "tokens_seen": 1359872000 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.0002592325515415169, | |
| "loss": 2.6576, | |
| "theoretical_loss": 3.5461366076089202, | |
| "tokens_seen": 1363148800 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00025864147909967843, | |
| "loss": 2.6604, | |
| "theoretical_loss": 3.5453723322644146, | |
| "tokens_seen": 1366425600 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00025805040665783995, | |
| "loss": 2.6435, | |
| "theoretical_loss": 3.544610399303803, | |
| "tokens_seen": 1369702400 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00025745933421600153, | |
| "loss": 2.6422, | |
| "theoretical_loss": 3.5438507959699637, | |
| "tokens_seen": 1372979200 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00025686826177416305, | |
| "loss": 2.6572, | |
| "theoretical_loss": 3.5430935096054883, | |
| "tokens_seen": 1376256000 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00025627718933232457, | |
| "loss": 2.6133, | |
| "theoretical_loss": 3.5423385276516663, | |
| "tokens_seen": 1379532800 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.0002556861168904861, | |
| "loss": 2.6267, | |
| "theoretical_loss": 3.5415858376474825, | |
| "tokens_seen": 1382809600 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.0002550950444486476, | |
| "loss": 2.5951, | |
| "theoretical_loss": 3.5408354272286298, | |
| "tokens_seen": 1386086400 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.00025450397200680913, | |
| "loss": 2.6355, | |
| "theoretical_loss": 3.540087284126531, | |
| "tokens_seen": 1389363200 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.0002539128995649707, | |
| "loss": 2.6396, | |
| "theoretical_loss": 3.539341396167372, | |
| "tokens_seen": 1392640000 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.00025332182712313223, | |
| "loss": 2.6724, | |
| "theoretical_loss": 3.538597751271153, | |
| "tokens_seen": 1395916800 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.00025273075468129375, | |
| "loss": 2.6724, | |
| "theoretical_loss": 3.5378563374507443, | |
| "tokens_seen": 1399193600 | |
| } | |
| ], | |
| "max_steps": 42724, | |
| "num_train_epochs": 9223372036854775807, | |
| "total_flos": 7.14460209610752e+17, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |