| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.0, | |
| "global_step": 42724, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 1.1682242990654204e-06, | |
| "loss": 10.8813, | |
| "theoretical_loss": 20.81281176760504, | |
| "tokens_seen": 65536 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 5.841121495327103e-05, | |
| "loss": 8.9731, | |
| "theoretical_loss": 8.563476630668958, | |
| "tokens_seen": 3276800 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00011682242990654206, | |
| "loss": 6.803, | |
| "theoretical_loss": 7.477752684105921, | |
| "tokens_seen": 6553600 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00017523364485981307, | |
| "loss": 5.89, | |
| "theoretical_loss": 6.9337484549527915, | |
| "tokens_seen": 9830400 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00023364485981308412, | |
| "loss": 5.4842, | |
| "theoretical_loss": 6.5835601944843045, | |
| "tokens_seen": 13107200 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00029205607476635517, | |
| "loss": 5.2294, | |
| "theoretical_loss": 6.3307075311739744, | |
| "tokens_seen": 16384000 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00035046728971962614, | |
| "loss": 5.0511, | |
| "theoretical_loss": 6.135523197998216, | |
| "tokens_seen": 19660800 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004088785046728972, | |
| "loss": 4.9053, | |
| "theoretical_loss": 5.978095549927499, | |
| "tokens_seen": 22937600 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00046728971962616824, | |
| "loss": 4.8028, | |
| "theoretical_loss": 5.847111292323815, | |
| "tokens_seen": 26214400 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000499739928125591, | |
| "loss": 4.7268, | |
| "theoretical_loss": 5.735570781940016, | |
| "tokens_seen": 29491200 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004991488556837526, | |
| "loss": 4.5965, | |
| "theoretical_loss": 5.638864110129244, | |
| "tokens_seen": 32768000 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004985577832419141, | |
| "loss": 4.4507, | |
| "theoretical_loss": 5.553806347902798, | |
| "tokens_seen": 36044800 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004979667108000757, | |
| "loss": 4.3796, | |
| "theoretical_loss": 5.478112046614329, | |
| "tokens_seen": 39321600 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004973756383582371, | |
| "loss": 4.2971, | |
| "theoretical_loss": 5.410089925637252, | |
| "tokens_seen": 42598400 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004967845659163987, | |
| "loss": 4.1952, | |
| "theoretical_loss": 5.348456049793725, | |
| "tokens_seen": 45875200 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004961934934745603, | |
| "loss": 4.1713, | |
| "theoretical_loss": 5.292214532995457, | |
| "tokens_seen": 49152000 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004956024210327218, | |
| "loss": 4.065, | |
| "theoretical_loss": 5.240578591827869, | |
| "tokens_seen": 52428800 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004950113485908833, | |
| "loss": 3.9725, | |
| "theoretical_loss": 5.192916690583679, | |
| "tokens_seen": 55705600 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004944202761490448, | |
| "loss": 3.9811, | |
| "theoretical_loss": 5.148714829414331, | |
| "tokens_seen": 58982400 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004938292037072064, | |
| "loss": 3.8862, | |
| "theoretical_loss": 5.107549528462992, | |
| "tokens_seen": 62259200 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004932381312653678, | |
| "loss": 3.7569, | |
| "theoretical_loss": 5.069068083201136, | |
| "tokens_seen": 65536000 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004926470588235294, | |
| "loss": 3.6979, | |
| "theoretical_loss": 5.032973875895897, | |
| "tokens_seen": 68812800 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.000492055986381691, | |
| "loss": 3.5973, | |
| "theoretical_loss": 4.999015274282555, | |
| "tokens_seen": 72089600 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004914649139398525, | |
| "loss": 3.5246, | |
| "theoretical_loss": 4.966977121409853, | |
| "tokens_seen": 75366400 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.000490873841498014, | |
| "loss": 3.5081, | |
| "theoretical_loss": 4.936674127683027, | |
| "tokens_seen": 78643200 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004902827690561755, | |
| "loss": 3.4698, | |
| "theoretical_loss": 4.907945679887972, | |
| "tokens_seen": 81920000 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004896916966143371, | |
| "loss": 3.4115, | |
| "theoretical_loss": 4.8806517198708175, | |
| "tokens_seen": 85196800 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004891006241724985, | |
| "loss": 3.4193, | |
| "theoretical_loss": 4.85466944053967, | |
| "tokens_seen": 88473600 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004885095517306601, | |
| "loss": 3.3934, | |
| "theoretical_loss": 4.829890613366704, | |
| "tokens_seen": 91750400 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004879184792888217, | |
| "loss": 3.35, | |
| "theoretical_loss": 4.806219408835812, | |
| "tokens_seen": 95027200 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004873274068469832, | |
| "loss": 3.2861, | |
| "theoretical_loss": 4.783570605334148, | |
| "tokens_seen": 98304000 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.00048673633440514467, | |
| "loss": 3.2664, | |
| "theoretical_loss": 4.761868106830299, | |
| "tokens_seen": 101580800 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004861452619633062, | |
| "loss": 3.3073, | |
| "theoretical_loss": 4.741043708020364, | |
| "tokens_seen": 104857600 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004855541895214677, | |
| "loss": 3.2906, | |
| "theoretical_loss": 4.721036059306941, | |
| "tokens_seen": 108134400 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004849631170796293, | |
| "loss": 3.2249, | |
| "theoretical_loss": 4.701789794289756, | |
| "tokens_seen": 111411200 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004843720446377908, | |
| "loss": 3.2799, | |
| "theoretical_loss": 4.68325479029382, | |
| "tokens_seen": 114688000 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.00048378097219595233, | |
| "loss": 3.229, | |
| "theoretical_loss": 4.6653855384841725, | |
| "tokens_seen": 117964800 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.00048318989975411385, | |
| "loss": 3.2623, | |
| "theoretical_loss": 4.6481406047776295, | |
| "tokens_seen": 121241600 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.00048259882731227537, | |
| "loss": 3.2475, | |
| "theoretical_loss": 4.631482166397534, | |
| "tokens_seen": 124518400 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0004820077548704369, | |
| "loss": 3.1819, | |
| "theoretical_loss": 4.615375611773608, | |
| "tokens_seen": 127795200 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00048141668242859847, | |
| "loss": 3.2083, | |
| "theoretical_loss": 4.5997891937483955, | |
| "tokens_seen": 131072000 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00048082560998676, | |
| "loss": 3.189, | |
| "theoretical_loss": 4.584693727850565, | |
| "tokens_seen": 134348800 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0004802345375449215, | |
| "loss": 3.1967, | |
| "theoretical_loss": 4.570062328836407, | |
| "tokens_seen": 137625600 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00047964346510308303, | |
| "loss": 3.1635, | |
| "theoretical_loss": 4.5558701798619285, | |
| "tokens_seen": 140902400 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00047905239266124455, | |
| "loss": 3.1347, | |
| "theoretical_loss": 4.542094329588689, | |
| "tokens_seen": 144179200 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00047846132021940607, | |
| "loss": 3.1606, | |
| "theoretical_loss": 4.528713513292708, | |
| "tokens_seen": 147456000 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0004778702477775676, | |
| "loss": 3.0966, | |
| "theoretical_loss": 4.515707994672887, | |
| "tokens_seen": 150732800 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047727917533572917, | |
| "loss": 3.1029, | |
| "theoretical_loss": 4.503059425571229, | |
| "tokens_seen": 154009600 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0004766881028938907, | |
| "loss": 3.1398, | |
| "theoretical_loss": 4.490750721243157, | |
| "tokens_seen": 157286400 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0004760970304520522, | |
| "loss": 3.0796, | |
| "theoretical_loss": 4.478765949169858, | |
| "tokens_seen": 160563200 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "objective/train/docs_used": 96518, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 3.2286458015441895, | |
| "objective/train/theoretical_loss": 4.46709022969911, | |
| "objective/train/tokens_used": 184300000, | |
| "theoretical_loss": 4.46709022969911, | |
| "tokens_seen": 163840000 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047550595801021373, | |
| "loss": 3.0725, | |
| "theoretical_loss": 4.46709022969911, | |
| "tokens_seen": 163840000 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047491488556837525, | |
| "loss": 3.0736, | |
| "theoretical_loss": 4.455709647047437, | |
| "tokens_seen": 167116800 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047432381312653677, | |
| "loss": 3.0957, | |
| "theoretical_loss": 4.444611169403321, | |
| "tokens_seen": 170393600 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047373274068469835, | |
| "loss": 3.0505, | |
| "theoretical_loss": 4.4337825770455375, | |
| "tokens_seen": 173670400 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047314166824285987, | |
| "loss": 3.0771, | |
| "theoretical_loss": 4.423212397538051, | |
| "tokens_seen": 176947200 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0004725505958010214, | |
| "loss": 3.0805, | |
| "theoretical_loss": 4.4128898471880325, | |
| "tokens_seen": 180224000 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0004719595233591829, | |
| "loss": 3.0562, | |
| "theoretical_loss": 4.40280477805997, | |
| "tokens_seen": 183500800 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00047136845091734443, | |
| "loss": 3.0368, | |
| "theoretical_loss": 4.392947629929752, | |
| "tokens_seen": 186777600 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00047077737847550595, | |
| "loss": 3.0456, | |
| "theoretical_loss": 4.383309386640423, | |
| "tokens_seen": 190054400 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0004701863060336675, | |
| "loss": 3.0098, | |
| "theoretical_loss": 4.373881536388167, | |
| "tokens_seen": 193331200 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00046959523359182905, | |
| "loss": 3.0174, | |
| "theoretical_loss": 4.364656035524595, | |
| "tokens_seen": 196608000 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00046900416114999057, | |
| "loss": 3.0129, | |
| "theoretical_loss": 4.355625275511174, | |
| "tokens_seen": 199884800 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0004684130887081521, | |
| "loss": 2.9953, | |
| "theoretical_loss": 4.346782052704563, | |
| "tokens_seen": 203161600 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0004678220162663136, | |
| "loss": 2.9867, | |
| "theoretical_loss": 4.338119540689052, | |
| "tokens_seen": 206438400 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00046723094382447513, | |
| "loss": 3.0165, | |
| "theoretical_loss": 4.329631264904703, | |
| "tokens_seen": 209715200 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.00046663987138263665, | |
| "loss": 3.0017, | |
| "theoretical_loss": 4.321311079348144, | |
| "tokens_seen": 212992000 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0004660487989407982, | |
| "loss": 2.9951, | |
| "theoretical_loss": 4.313153145147683, | |
| "tokens_seen": 216268800 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.00046545772649895975, | |
| "loss": 2.9689, | |
| "theoretical_loss": 4.305151910836119, | |
| "tokens_seen": 219545600 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.00046486665405712127, | |
| "loss": 2.9826, | |
| "theoretical_loss": 4.2973020941635784, | |
| "tokens_seen": 222822400 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0004642755816152828, | |
| "loss": 2.9874, | |
| "theoretical_loss": 4.28959866530949, | |
| "tokens_seen": 226099200 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0004636845091734443, | |
| "loss": 2.9269, | |
| "theoretical_loss": 4.282036831367506, | |
| "tokens_seen": 229376000 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.00046309343673160583, | |
| "loss": 2.9501, | |
| "theoretical_loss": 4.274612021990189, | |
| "tokens_seen": 232652800 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0004625023642897674, | |
| "loss": 2.9073, | |
| "theoretical_loss": 4.267319876091788, | |
| "tokens_seen": 235929600 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004619112918479289, | |
| "loss": 2.9524, | |
| "theoretical_loss": 4.260156229517635, | |
| "tokens_seen": 239206400 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.00046132021940609044, | |
| "loss": 2.9937, | |
| "theoretical_loss": 4.253117103597704, | |
| "tokens_seen": 242483200 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.00046072914696425197, | |
| "loss": 2.9917, | |
| "theoretical_loss": 4.246198694509945, | |
| "tokens_seen": 245760000 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004601380745224135, | |
| "loss": 2.967, | |
| "theoretical_loss": 4.239397363386152, | |
| "tokens_seen": 249036800 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.000459547002080575, | |
| "loss": 2.9536, | |
| "theoretical_loss": 4.232709627099522, | |
| "tokens_seen": 252313600 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004589559296387366, | |
| "loss": 2.9099, | |
| "theoretical_loss": 4.226132149678757, | |
| "tokens_seen": 255590400 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004583648571968981, | |
| "loss": 2.9495, | |
| "theoretical_loss": 4.219661734298666, | |
| "tokens_seen": 258867200 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004577737847550596, | |
| "loss": 2.9228, | |
| "theoretical_loss": 4.213295315801815, | |
| "tokens_seen": 262144000 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004571827123132211, | |
| "loss": 2.914, | |
| "theoretical_loss": 4.207029953709861, | |
| "tokens_seen": 265420800 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0004565916398713826, | |
| "loss": 2.9065, | |
| "theoretical_loss": 4.200862825686893, | |
| "tokens_seen": 268697600 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.00045600056742954413, | |
| "loss": 2.9208, | |
| "theoretical_loss": 4.19479122142044, | |
| "tokens_seen": 271974400 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0004554094949877057, | |
| "loss": 2.8755, | |
| "theoretical_loss": 4.188812536888775, | |
| "tokens_seen": 275251200 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.00045481842254586723, | |
| "loss": 2.911, | |
| "theoretical_loss": 4.182924268985855, | |
| "tokens_seen": 278528000 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.00045422735010402875, | |
| "loss": 2.8592, | |
| "theoretical_loss": 4.177124010477671, | |
| "tokens_seen": 281804800 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.00045363627766219027, | |
| "loss": 2.9063, | |
| "theoretical_loss": 4.171409445265983, | |
| "tokens_seen": 285081600 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0004530452052203518, | |
| "loss": 2.8809, | |
| "theoretical_loss": 4.165778343937409, | |
| "tokens_seen": 288358400 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0004524541327785133, | |
| "loss": 2.8394, | |
| "theoretical_loss": 4.160228559577659, | |
| "tokens_seen": 291635200 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.00045186306033667483, | |
| "loss": 2.8306, | |
| "theoretical_loss": 4.15475802383233, | |
| "tokens_seen": 294912000 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0004512719878948364, | |
| "loss": 2.8102, | |
| "theoretical_loss": 4.149364743197177, | |
| "tokens_seen": 298188800 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.00045068091545299793, | |
| "loss": 2.8491, | |
| "theoretical_loss": 4.14404679552214, | |
| "tokens_seen": 301465600 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.00045008984301115945, | |
| "loss": 2.8152, | |
| "theoretical_loss": 4.138802326714632, | |
| "tokens_seen": 304742400 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.00044949877056932097, | |
| "loss": 2.8902, | |
| "theoretical_loss": 4.133629547628726, | |
| "tokens_seen": 308019200 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0004489076981274825, | |
| "loss": 2.8874, | |
| "theoretical_loss": 4.128526731127894, | |
| "tokens_seen": 311296000 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.000448316625685644, | |
| "loss": 2.8282, | |
| "theoretical_loss": 4.123492209309923, | |
| "tokens_seen": 314572800 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0004477255532438056, | |
| "loss": 2.8807, | |
| "theoretical_loss": 4.118524370883447, | |
| "tokens_seen": 317849600 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0004471344808019671, | |
| "loss": 2.8404, | |
| "theoretical_loss": 4.113621658686355, | |
| "tokens_seen": 321126400 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.00044654340836012863, | |
| "loss": 2.8468, | |
| "theoretical_loss": 4.108782567337039, | |
| "tokens_seen": 324403200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.5179236306635556, | |
| "debugging/distinct-1-grams": 0.7868809037747271, | |
| "debugging/distinct-2-grams": 0.95444323584591, | |
| "debugging/entropy-1-grams": 5.810297153205875, | |
| "debugging/entropy-2-grams": 6.5112113932041344, | |
| "debugging/length": 540.5, | |
| "debugging/num_segments": 12, | |
| "epoch": 0.12, | |
| "objective/train/docs_used": 182435, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.7323758602142334, | |
| "objective/train/theoretical_loss": 4.104005641010112, | |
| "objective/train/tokens_used": 348140000, | |
| "theoretical_loss": 4.104005641010112, | |
| "tokens_seen": 327680000 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.00044595233591829015, | |
| "loss": 2.8315, | |
| "theoretical_loss": 4.104005641010112, | |
| "tokens_seen": 327680000 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.00044536126347645167, | |
| "loss": 2.7719, | |
| "theoretical_loss": 4.099289471328812, | |
| "tokens_seen": 330956800 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0004447701910346132, | |
| "loss": 2.8239, | |
| "theoretical_loss": 4.094632695366921, | |
| "tokens_seen": 334233600 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.00044417911859277476, | |
| "loss": 2.7712, | |
| "theoretical_loss": 4.090033993753448, | |
| "tokens_seen": 337510400 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0004435880461509363, | |
| "loss": 2.7701, | |
| "theoretical_loss": 4.085492088873883, | |
| "tokens_seen": 340787200 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0004429969737090978, | |
| "loss": 2.7771, | |
| "theoretical_loss": 4.081005743162224, | |
| "tokens_seen": 344064000 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0004424059012672593, | |
| "loss": 2.7947, | |
| "theoretical_loss": 4.076573757478361, | |
| "tokens_seen": 347340800 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00044181482882542085, | |
| "loss": 2.7985, | |
| "theoretical_loss": 4.072194969565807, | |
| "tokens_seen": 350617600 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00044122375638358237, | |
| "loss": 2.8102, | |
| "theoretical_loss": 4.067868252585089, | |
| "tokens_seen": 353894400 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0004406326839417439, | |
| "loss": 2.8096, | |
| "theoretical_loss": 4.063592513718411, | |
| "tokens_seen": 357171200 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00044004161149990546, | |
| "loss": 2.7988, | |
| "theoretical_loss": 4.059366692841521, | |
| "tokens_seen": 360448000 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.000439450539058067, | |
| "loss": 2.8074, | |
| "theoretical_loss": 4.055189761258959, | |
| "tokens_seen": 363724800 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0004388594666162285, | |
| "loss": 2.7982, | |
| "theoretical_loss": 4.051060720499127, | |
| "tokens_seen": 367001600 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00043826839417439, | |
| "loss": 2.7748, | |
| "theoretical_loss": 4.046978601165831, | |
| "tokens_seen": 370278400 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00043767732173255155, | |
| "loss": 2.7677, | |
| "theoretical_loss": 4.042942461843204, | |
| "tokens_seen": 373555200 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00043708624929071307, | |
| "loss": 2.7828, | |
| "theoretical_loss": 4.038951388051044, | |
| "tokens_seen": 376832000 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.00043649517684887464, | |
| "loss": 2.7974, | |
| "theoretical_loss": 4.035004491247873, | |
| "tokens_seen": 380108800 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.00043590410440703616, | |
| "loss": 2.7766, | |
| "theoretical_loss": 4.031100907879109, | |
| "tokens_seen": 383385600 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0004353130319651977, | |
| "loss": 2.8236, | |
| "theoretical_loss": 4.02723979846797, | |
| "tokens_seen": 386662400 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0004347219595233592, | |
| "loss": 2.829, | |
| "theoretical_loss": 4.023420346746835, | |
| "tokens_seen": 389939200 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0004341308870815207, | |
| "loss": 2.7938, | |
| "theoretical_loss": 4.019641758826938, | |
| "tokens_seen": 393216000 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.00043353981463968225, | |
| "loss": 2.7822, | |
| "theoretical_loss": 4.015903262404413, | |
| "tokens_seen": 396492800 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0004329487421978438, | |
| "loss": 2.7694, | |
| "theoretical_loss": 4.012204106000786, | |
| "tokens_seen": 399769600 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.00043235766975600534, | |
| "loss": 2.7842, | |
| "theoretical_loss": 4.008543558236181, | |
| "tokens_seen": 403046400 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.00043176659731416686, | |
| "loss": 2.7663, | |
| "theoretical_loss": 4.004920907133565, | |
| "tokens_seen": 406323200 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004311755248723284, | |
| "loss": 2.7278, | |
| "theoretical_loss": 4.001335459452449, | |
| "tokens_seen": 409600000 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004305844524304899, | |
| "loss": 2.7589, | |
| "theoretical_loss": 3.997786540050617, | |
| "tokens_seen": 412876800 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004299933799886514, | |
| "loss": 2.7719, | |
| "theoretical_loss": 3.9942734912724456, | |
| "tokens_seen": 416153600 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.00042940230754681295, | |
| "loss": 2.7241, | |
| "theoretical_loss": 3.9907956723625375, | |
| "tokens_seen": 419430400 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004288112351049745, | |
| "loss": 2.7718, | |
| "theoretical_loss": 3.9873524589034224, | |
| "tokens_seen": 422707200 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.000428220162663136, | |
| "loss": 2.765, | |
| "theoretical_loss": 3.9839432422761556, | |
| "tokens_seen": 425984000 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004276290902212975, | |
| "loss": 2.804, | |
| "theoretical_loss": 3.980567429142721, | |
| "tokens_seen": 429260800 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.00042703801777945903, | |
| "loss": 2.7606, | |
| "theoretical_loss": 3.977224440949197, | |
| "tokens_seen": 432537600 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042644694533762055, | |
| "loss": 2.772, | |
| "theoretical_loss": 3.9739137134486917, | |
| "tokens_seen": 435814400 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042585587289578207, | |
| "loss": 2.8015, | |
| "theoretical_loss": 3.9706346962431396, | |
| "tokens_seen": 439091200 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042526480045394365, | |
| "loss": 2.8007, | |
| "theoretical_loss": 3.9673868523430564, | |
| "tokens_seen": 442368000 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042467372801210517, | |
| "loss": 2.7835, | |
| "theoretical_loss": 3.9641696577444376, | |
| "tokens_seen": 445644800 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0004240826555702667, | |
| "loss": 2.7811, | |
| "theoretical_loss": 3.9609826010220033, | |
| "tokens_seen": 448921600 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0004234915831284282, | |
| "loss": 2.8235, | |
| "theoretical_loss": 3.9578251829380506, | |
| "tokens_seen": 452198400 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042290051068658973, | |
| "loss": 2.7819, | |
| "theoretical_loss": 3.954696916066199, | |
| "tokens_seen": 455475200 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042230943824475125, | |
| "loss": 2.7632, | |
| "theoretical_loss": 3.9515973244293643, | |
| "tokens_seen": 458752000 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0004217183658029128, | |
| "loss": 2.7585, | |
| "theoretical_loss": 3.948525943151326, | |
| "tokens_seen": 462028800 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.00042112729336107435, | |
| "loss": 2.7676, | |
| "theoretical_loss": 3.9454823181212815, | |
| "tokens_seen": 465305600 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.00042053622091923587, | |
| "loss": 2.7674, | |
| "theoretical_loss": 3.9424660056708167, | |
| "tokens_seen": 468582400 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0004199451484773974, | |
| "loss": 2.7598, | |
| "theoretical_loss": 3.939476572262754, | |
| "tokens_seen": 471859200 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0004193540760355589, | |
| "loss": 2.761, | |
| "theoretical_loss": 3.9365135941913563, | |
| "tokens_seen": 475136000 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.00041876300359372043, | |
| "loss": 2.7423, | |
| "theoretical_loss": 3.9335766572934023, | |
| "tokens_seen": 478412800 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.00041817193115188195, | |
| "loss": 2.7487, | |
| "theoretical_loss": 3.9306653566696603, | |
| "tokens_seen": 481689600 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0004175808587100435, | |
| "loss": 2.7297, | |
| "theoretical_loss": 3.927779296416332, | |
| "tokens_seen": 484966400 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.00041698978626820505, | |
| "loss": 2.7348, | |
| "theoretical_loss": 3.924918089366024, | |
| "tokens_seen": 488243200 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "objective/train/docs_used": 267104, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.6938159465789795, | |
| "objective/train/theoretical_loss": 3.9220813568378707, | |
| "objective/train/tokens_used": 511980000, | |
| "theoretical_loss": 3.9220813568378707, | |
| "tokens_seen": 491520000 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.00041639871382636657, | |
| "loss": 2.6909, | |
| "theoretical_loss": 3.9220813568378707, | |
| "tokens_seen": 491520000 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0004158076413845281, | |
| "loss": 2.6957, | |
| "theoretical_loss": 3.9192687283964096, | |
| "tokens_seen": 494796800 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0004152165689426896, | |
| "loss": 2.7163, | |
| "theoretical_loss": 3.9164798416188527, | |
| "tokens_seen": 498073600 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.00041462549650085113, | |
| "loss": 2.6846, | |
| "theoretical_loss": 3.913714341870409, | |
| "tokens_seen": 501350400 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0004140344240590127, | |
| "loss": 2.6939, | |
| "theoretical_loss": 3.9109718820873303, | |
| "tokens_seen": 504627200 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0004134433516171742, | |
| "loss": 2.7024, | |
| "theoretical_loss": 3.9082521225673625, | |
| "tokens_seen": 507904000 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.00041285227917533575, | |
| "loss": 2.6622, | |
| "theoretical_loss": 3.9055547307673075, | |
| "tokens_seen": 511180800 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.00041226120673349727, | |
| "loss": 2.7267, | |
| "theoretical_loss": 3.9028793811074056, | |
| "tokens_seen": 514457600 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0004116701342916588, | |
| "loss": 2.7517, | |
| "theoretical_loss": 3.900225754782274, | |
| "tokens_seen": 517734400 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0004110790618498203, | |
| "loss": 2.7289, | |
| "theoretical_loss": 3.897593539578138, | |
| "tokens_seen": 521011200 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0004104879894079819, | |
| "loss": 2.7181, | |
| "theoretical_loss": 3.8949824296961015, | |
| "tokens_seen": 524288000 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0004098969169661434, | |
| "loss": 2.7306, | |
| "theoretical_loss": 3.8923921255812353, | |
| "tokens_seen": 527564800 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0004093058445243049, | |
| "loss": 2.7015, | |
| "theoretical_loss": 3.8898223337572393, | |
| "tokens_seen": 530841600 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.00040871477208246645, | |
| "loss": 2.6978, | |
| "theoretical_loss": 3.88727276666648, | |
| "tokens_seen": 534118400 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.00040812369964062797, | |
| "loss": 2.6723, | |
| "theoretical_loss": 3.884743142515184, | |
| "tokens_seen": 537395200 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0004075326271987895, | |
| "loss": 2.6872, | |
| "theoretical_loss": 3.8822331851235985, | |
| "tokens_seen": 540672000 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.000406941554756951, | |
| "loss": 2.6913, | |
| "theoretical_loss": 3.87974262378093, | |
| "tokens_seen": 543948800 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004063504823151126, | |
| "loss": 2.6513, | |
| "theoretical_loss": 3.877271193104873, | |
| "tokens_seen": 547225600 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004057594098732741, | |
| "loss": 2.6908, | |
| "theoretical_loss": 3.8748186329055736, | |
| "tokens_seen": 550502400 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004051801588802724, | |
| "loss": 2.7044, | |
| "theoretical_loss": 3.87238468805384, | |
| "tokens_seen": 553779200 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004045890864384339, | |
| "loss": 2.6751, | |
| "theoretical_loss": 3.8699691083534633, | |
| "tokens_seen": 557056000 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004039980139965954, | |
| "loss": 2.6641, | |
| "theoretical_loss": 3.8675716484174907, | |
| "tokens_seen": 560332800 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.00040340694155475694, | |
| "loss": 2.6713, | |
| "theoretical_loss": 3.8651920675482936, | |
| "tokens_seen": 563609600 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004028158691129185, | |
| "loss": 2.6941, | |
| "theoretical_loss": 3.862830129621318, | |
| "tokens_seen": 566886400 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.00040222479667108003, | |
| "loss": 2.7425, | |
| "theoretical_loss": 3.8604856029723575, | |
| "tokens_seen": 570163200 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.00040163372422924155, | |
| "loss": 2.6816, | |
| "theoretical_loss": 3.8581582602882447, | |
| "tokens_seen": 573440000 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0004010426517874031, | |
| "loss": 2.7, | |
| "theoretical_loss": 3.8558478785008203, | |
| "tokens_seen": 576716800 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0004004515793455646, | |
| "loss": 2.7188, | |
| "theoretical_loss": 3.8535542386840778, | |
| "tokens_seen": 579993600 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0003998605069037261, | |
| "loss": 2.714, | |
| "theoretical_loss": 3.8512771259543586, | |
| "tokens_seen": 583270400 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0003992694344618877, | |
| "loss": 2.7144, | |
| "theoretical_loss": 3.8490163293735082, | |
| "tokens_seen": 586547200 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0003986783620200492, | |
| "loss": 2.7168, | |
| "theoretical_loss": 3.8467716418548648, | |
| "tokens_seen": 589824000 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.00039808728957821073, | |
| "loss": 2.7048, | |
| "theoretical_loss": 3.844542860072007, | |
| "tokens_seen": 593100800 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.00039749621713637225, | |
| "loss": 2.7227, | |
| "theoretical_loss": 3.8423297843701496, | |
| "tokens_seen": 596377600 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0003969051446945338, | |
| "loss": 2.7137, | |
| "theoretical_loss": 3.8401322186800995, | |
| "tokens_seen": 599654400 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0003963140722526953, | |
| "loss": 2.7505, | |
| "theoretical_loss": 3.83794997043469, | |
| "tokens_seen": 602931200 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.00039572299981085687, | |
| "loss": 2.7533, | |
| "theoretical_loss": 3.8357828504876004, | |
| "tokens_seen": 606208000 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0003951319273690184, | |
| "loss": 2.7222, | |
| "theoretical_loss": 3.833630673034487, | |
| "tokens_seen": 609484800 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0003945408549271799, | |
| "loss": 2.7234, | |
| "theoretical_loss": 3.831493255536345, | |
| "tokens_seen": 612761600 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.00039394978248534143, | |
| "loss": 2.7374, | |
| "theoretical_loss": 3.8293704186450253, | |
| "tokens_seen": 616038400 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.00039335871004350295, | |
| "loss": 2.7151, | |
| "theoretical_loss": 3.827261986130839, | |
| "tokens_seen": 619315200 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0003927676376016645, | |
| "loss": 2.6627, | |
| "theoretical_loss": 3.825167784812175, | |
| "tokens_seen": 622592000 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.000392176565159826, | |
| "loss": 2.69, | |
| "theoretical_loss": 3.823087644487069, | |
| "tokens_seen": 625868800 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.00039158549271798757, | |
| "loss": 2.6656, | |
| "theoretical_loss": 3.8210213978666565, | |
| "tokens_seen": 629145600 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0003909944202761491, | |
| "loss": 2.6679, | |
| "theoretical_loss": 3.8189688805104476, | |
| "tokens_seen": 632422400 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.00039040334783431056, | |
| "loss": 2.6709, | |
| "theoretical_loss": 3.816929930763374, | |
| "tokens_seen": 635699200 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0003898122753924721, | |
| "loss": 2.6848, | |
| "theoretical_loss": 3.8149043896945347, | |
| "tokens_seen": 638976000 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0003892212029506336, | |
| "loss": 2.6777, | |
| "theoretical_loss": 3.812892101037601, | |
| "tokens_seen": 642252800 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0003886301305087951, | |
| "loss": 2.6847, | |
| "theoretical_loss": 3.81089291113282, | |
| "tokens_seen": 645529600 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0003880390580669567, | |
| "loss": 2.6691, | |
| "theoretical_loss": 3.8089066688705673, | |
| "tokens_seen": 648806400 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0003874479856251182, | |
| "loss": 2.6569, | |
| "theoretical_loss": 3.8069332256363992, | |
| "tokens_seen": 652083200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.642861845651988, | |
| "debugging/distinct-1-grams": 0.8145672280225638, | |
| "debugging/distinct-2-grams": 0.980694270616048, | |
| "debugging/entropy-1-grams": 5.8621094281307204, | |
| "debugging/entropy-2-grams": 6.737099920464704, | |
| "debugging/length": 648.6666666666666, | |
| "debugging/num_segments": 12, | |
| "epoch": 0.23, | |
| "objective/train/docs_used": 351924, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.6956400871276855, | |
| "objective/train/theoretical_loss": 3.80497243525756, | |
| "objective/train/tokens_used": 675820000, | |
| "theoretical_loss": 3.80497243525756, | |
| "tokens_seen": 655360000 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.00038685691318327974, | |
| "loss": 2.6514, | |
| "theoretical_loss": 3.80497243525756, | |
| "tokens_seen": 655360000 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.00038626584074144126, | |
| "loss": 2.6215, | |
| "theoretical_loss": 3.8030241539508958, | |
| "tokens_seen": 658636800 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003856747682996028, | |
| "loss": 2.619, | |
| "theoretical_loss": 3.8010882402721324, | |
| "tokens_seen": 661913600 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003850836958577643, | |
| "loss": 2.656, | |
| "theoretical_loss": 3.7991645550664757, | |
| "tokens_seen": 665190400 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003844926234159259, | |
| "loss": 2.6818, | |
| "theoretical_loss": 3.797252961420492, | |
| "tokens_seen": 668467200 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003839015509740874, | |
| "loss": 2.6846, | |
| "theoretical_loss": 3.795353324615228, | |
| "tokens_seen": 671744000 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003833104785322489, | |
| "loss": 2.7187, | |
| "theoretical_loss": 3.793465512080541, | |
| "tokens_seen": 675020800 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.00038271940609041044, | |
| "loss": 2.7152, | |
| "theoretical_loss": 3.791589393350587, | |
| "tokens_seen": 678297600 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.00038212833364857196, | |
| "loss": 2.6926, | |
| "theoretical_loss": 3.7897248400204475, | |
| "tokens_seen": 681574400 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003815372612067335, | |
| "loss": 2.6837, | |
| "theoretical_loss": 3.7878717257038534, | |
| "tokens_seen": 684851200 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.000380946188764895, | |
| "loss": 2.6716, | |
| "theoretical_loss": 3.7860299259919685, | |
| "tokens_seen": 688128000 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0003803551163230566, | |
| "loss": 2.6557, | |
| "theoretical_loss": 3.7841993184132114, | |
| "tokens_seen": 691404800 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0003797640438812181, | |
| "loss": 2.6455, | |
| "theoretical_loss": 3.78237978239408, | |
| "tokens_seen": 694681600 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0003791729714393796, | |
| "loss": 2.6635, | |
| "theoretical_loss": 3.780571199220942, | |
| "tokens_seen": 697958400 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.00037858189899754114, | |
| "loss": 2.638, | |
| "theoretical_loss": 3.7787734520027803, | |
| "tokens_seen": 701235200 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.00037799082655570266, | |
| "loss": 2.6455, | |
| "theoretical_loss": 3.7769864256348455, | |
| "tokens_seen": 704512000 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0003773997541138642, | |
| "loss": 2.6937, | |
| "theoretical_loss": 3.775210006763202, | |
| "tokens_seen": 707788800 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.00037680868167202575, | |
| "loss": 2.6785, | |
| "theoretical_loss": 3.7734440837501406, | |
| "tokens_seen": 711065600 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0003762176092301873, | |
| "loss": 2.7156, | |
| "theoretical_loss": 3.7716885466404246, | |
| "tokens_seen": 714342400 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0003756265367883488, | |
| "loss": 2.7095, | |
| "theoretical_loss": 3.769943287128357, | |
| "tokens_seen": 717619200 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0003750354643465103, | |
| "loss": 2.7089, | |
| "theoretical_loss": 3.7682081985256364, | |
| "tokens_seen": 720896000 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.00037444439190467184, | |
| "loss": 2.7286, | |
| "theoretical_loss": 3.7664831757299795, | |
| "tokens_seen": 724172800 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.00037385331946283336, | |
| "loss": 2.6792, | |
| "theoretical_loss": 3.7647681151944976, | |
| "tokens_seen": 727449600 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.00037326224702099493, | |
| "loss": 2.6448, | |
| "theoretical_loss": 3.7630629148977937, | |
| "tokens_seen": 730726400 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.00037267117457915645, | |
| "loss": 2.6615, | |
| "theoretical_loss": 3.761367474314768, | |
| "tokens_seen": 734003200 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.000372080102137318, | |
| "loss": 2.6745, | |
| "theoretical_loss": 3.7596816943881084, | |
| "tokens_seen": 737280000 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0003714890296954795, | |
| "loss": 2.6448, | |
| "theoretical_loss": 3.758005477500451, | |
| "tokens_seen": 740556800 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.000370897957253641, | |
| "loss": 2.6424, | |
| "theoretical_loss": 3.756338727447186, | |
| "tokens_seen": 743833600 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.00037030688481180254, | |
| "loss": 2.6773, | |
| "theoretical_loss": 3.7546813494098945, | |
| "tokens_seen": 747110400 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.00036971581236996406, | |
| "loss": 2.6502, | |
| "theoretical_loss": 3.7530332499304007, | |
| "tokens_seen": 750387200 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.00036912473992812563, | |
| "loss": 2.673, | |
| "theoretical_loss": 3.7513943368854195, | |
| "tokens_seen": 753664000 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.00036853366748628715, | |
| "loss": 2.6322, | |
| "theoretical_loss": 3.7497645194617863, | |
| "tokens_seen": 756940800 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0003679425950444487, | |
| "loss": 2.602, | |
| "theoretical_loss": 3.748143708132246, | |
| "tokens_seen": 760217600 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0003673515226026102, | |
| "loss": 2.6187, | |
| "theoretical_loss": 3.7465318146317994, | |
| "tokens_seen": 763494400 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0003667604501607717, | |
| "loss": 2.587, | |
| "theoretical_loss": 3.7449287519345766, | |
| "tokens_seen": 766771200 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.00036616937771893324, | |
| "loss": 2.5946, | |
| "theoretical_loss": 3.7433344342312385, | |
| "tokens_seen": 770048000 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0003655783052770948, | |
| "loss": 2.6269, | |
| "theoretical_loss": 3.7417487769068756, | |
| "tokens_seen": 773324800 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.00036498723283525633, | |
| "loss": 2.6403, | |
| "theoretical_loss": 3.7401716965194076, | |
| "tokens_seen": 776601600 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.00036439616039341785, | |
| "loss": 2.6323, | |
| "theoretical_loss": 3.738603110778461, | |
| "tokens_seen": 779878400 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0003638050879515794, | |
| "loss": 2.6448, | |
| "theoretical_loss": 3.73704293852471, | |
| "tokens_seen": 783155200 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0003632140155097409, | |
| "loss": 2.6451, | |
| "theoretical_loss": 3.7354910997096793, | |
| "tokens_seen": 786432000 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0003626229430679024, | |
| "loss": 2.656, | |
| "theoretical_loss": 3.7339475153759825, | |
| "tokens_seen": 789708800 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.000362031870626064, | |
| "loss": 2.611, | |
| "theoretical_loss": 3.732412107638, | |
| "tokens_seen": 792985600 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.00036144079818422546, | |
| "loss": 2.6138, | |
| "theoretical_loss": 3.7308847996629724, | |
| "tokens_seen": 796262400 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.000360849725742387, | |
| "loss": 2.6362, | |
| "theoretical_loss": 3.7293655156525043, | |
| "tokens_seen": 799539200 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0003602586533005485, | |
| "loss": 2.6161, | |
| "theoretical_loss": 3.727854180824469, | |
| "tokens_seen": 802816000 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.00035966758085871, | |
| "loss": 2.5601, | |
| "theoretical_loss": 3.7263507213952978, | |
| "tokens_seen": 806092800 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.00035907650841687154, | |
| "loss": 2.584, | |
| "theoretical_loss": 3.724855064562658, | |
| "tokens_seen": 809369600 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.00035848543597503306, | |
| "loss": 2.6023, | |
| "theoretical_loss": 3.723367138488488, | |
| "tokens_seen": 812646400 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.00035789436353319464, | |
| "loss": 2.6156, | |
| "theoretical_loss": 3.7218868722824014, | |
| "tokens_seen": 815923200 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "objective/train/docs_used": 436767, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.712568759918213, | |
| "objective/train/theoretical_loss": 3.7204141959854384, | |
| "objective/train/tokens_used": 839660000, | |
| "theoretical_loss": 3.7204141959854384, | |
| "tokens_seen": 819200000 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.00035730329109135616, | |
| "loss": 2.6219, | |
| "theoretical_loss": 3.7204141959854384, | |
| "tokens_seen": 819200000 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0003567122186495177, | |
| "loss": 2.6477, | |
| "theoretical_loss": 3.718949040554162, | |
| "tokens_seen": 822476800 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0003561211462076792, | |
| "loss": 2.6576, | |
| "theoretical_loss": 3.7174913378450833, | |
| "tokens_seen": 825753600 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0003555300737658407, | |
| "loss": 2.6582, | |
| "theoretical_loss": 3.7160410205994183, | |
| "tokens_seen": 829030400 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.00035493900132400224, | |
| "loss": 2.6759, | |
| "theoretical_loss": 3.7145980224281585, | |
| "tokens_seen": 832307200 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0003543597503310006, | |
| "loss": 2.6768, | |
| "theoretical_loss": 3.713162277797449, | |
| "tokens_seen": 835584000 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.00035376867788916214, | |
| "loss": 2.6422, | |
| "theoretical_loss": 3.7117337220142748, | |
| "tokens_seen": 838860800 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0003531894268961604, | |
| "loss": 2.6515, | |
| "theoretical_loss": 3.7103122912124364, | |
| "tokens_seen": 842137600 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0003526101759031587, | |
| "loss": 2.6762, | |
| "theoretical_loss": 3.7088979223388128, | |
| "tokens_seen": 845414400 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0003520191034613202, | |
| "loss": 2.6563, | |
| "theoretical_loss": 3.70749055313991, | |
| "tokens_seen": 848691200 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0003514280310194817, | |
| "loss": 2.6461, | |
| "theoretical_loss": 3.7060901221486766, | |
| "tokens_seen": 851968000 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.0003508369585776433, | |
| "loss": 2.6557, | |
| "theoretical_loss": 3.704696568671591, | |
| "tokens_seen": 855244800 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.0003502458861358048, | |
| "loss": 2.6539, | |
| "theoretical_loss": 3.7033098327760063, | |
| "tokens_seen": 858521600 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00034965481369396634, | |
| "loss": 2.632, | |
| "theoretical_loss": 3.7019298552777533, | |
| "tokens_seen": 861798400 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00034906374125212786, | |
| "loss": 2.619, | |
| "theoretical_loss": 3.700556577728988, | |
| "tokens_seen": 865075200 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.0003484726688102894, | |
| "loss": 2.629, | |
| "theoretical_loss": 3.6991899424062815, | |
| "tokens_seen": 868352000 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.0003478815963684509, | |
| "loss": 2.609, | |
| "theoretical_loss": 3.697829892298951, | |
| "tokens_seen": 871628800 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.0003472905239266125, | |
| "loss": 2.6334, | |
| "theoretical_loss": 3.696476371097618, | |
| "tokens_seen": 874905600 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.000346699451484774, | |
| "loss": 2.6014, | |
| "theoretical_loss": 3.695129323182993, | |
| "tokens_seen": 878182400 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.0003461083790429355, | |
| "loss": 2.5987, | |
| "theoretical_loss": 3.693788693614879, | |
| "tokens_seen": 881459200 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.00034551730660109704, | |
| "loss": 2.5732, | |
| "theoretical_loss": 3.6924544281213967, | |
| "tokens_seen": 884736000 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.00034492623415925856, | |
| "loss": 2.5476, | |
| "theoretical_loss": 3.691126473088412, | |
| "tokens_seen": 888012800 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0003443351617174201, | |
| "loss": 2.5573, | |
| "theoretical_loss": 3.689804775549173, | |
| "tokens_seen": 891289600 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0003437440892755816, | |
| "loss": 2.5655, | |
| "theoretical_loss": 3.688489283174146, | |
| "tokens_seen": 894566400 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0003431530168337432, | |
| "loss": 2.6253, | |
| "theoretical_loss": 3.6871799442610538, | |
| "tokens_seen": 897843200 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0003425619443919047, | |
| "loss": 2.6272, | |
| "theoretical_loss": 3.685876707725093, | |
| "tokens_seen": 901120000 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0003419708719500662, | |
| "loss": 2.5663, | |
| "theoretical_loss": 3.6845795230893517, | |
| "tokens_seen": 904396800 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.00034137979950822774, | |
| "loss": 2.5763, | |
| "theoretical_loss": 3.6832883404754035, | |
| "tokens_seen": 907673600 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00034078872706638926, | |
| "loss": 2.5661, | |
| "theoretical_loss": 3.6820031105940796, | |
| "tokens_seen": 910950400 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0003401976546245508, | |
| "loss": 2.565, | |
| "theoretical_loss": 3.6807237847364176, | |
| "tokens_seen": 914227200 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00033960658218271236, | |
| "loss": 2.6169, | |
| "theoretical_loss": 3.6794503147647846, | |
| "tokens_seen": 917504000 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0003390155097408739, | |
| "loss": 2.6209, | |
| "theoretical_loss": 3.67818265310416, | |
| "tokens_seen": 920780800 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0003384244372990354, | |
| "loss": 2.602, | |
| "theoretical_loss": 3.6769207527335888, | |
| "tokens_seen": 924057600 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0003378333648571969, | |
| "loss": 2.597, | |
| "theoretical_loss": 3.675664567177787, | |
| "tokens_seen": 927334400 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00033724229241535844, | |
| "loss": 2.596, | |
| "theoretical_loss": 3.674414050498913, | |
| "tokens_seen": 930611200 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00033665121997351996, | |
| "loss": 2.5945, | |
| "theoretical_loss": 3.6731691572884824, | |
| "tokens_seen": 933888000 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00033606014753168154, | |
| "loss": 2.5671, | |
| "theoretical_loss": 3.671929842659438, | |
| "tokens_seen": 937164800 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00033546907508984306, | |
| "loss": 2.5612, | |
| "theoretical_loss": 3.6706960622383624, | |
| "tokens_seen": 940441600 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0003348780026480046, | |
| "loss": 2.5934, | |
| "theoretical_loss": 3.6694677721578377, | |
| "tokens_seen": 943718400 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0003342869302061661, | |
| "loss": 2.59, | |
| "theoretical_loss": 3.66824492904894, | |
| "tokens_seen": 946995200 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00033369585776432757, | |
| "loss": 2.5596, | |
| "theoretical_loss": 3.667027490033874, | |
| "tokens_seen": 950272000 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0003331047853224891, | |
| "loss": 2.5686, | |
| "theoretical_loss": 3.6658154127187412, | |
| "tokens_seen": 953548800 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0003325137128806506, | |
| "loss": 2.5853, | |
| "theoretical_loss": 3.664608655186437, | |
| "tokens_seen": 956825600 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0003319226404388122, | |
| "loss": 2.5926, | |
| "theoretical_loss": 3.663407175989679, | |
| "tokens_seen": 960102400 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0003313315679969737, | |
| "loss": 2.5611, | |
| "theoretical_loss": 3.662210934144158, | |
| "tokens_seen": 963379200 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003307404955551352, | |
| "loss": 2.587, | |
| "theoretical_loss": 3.661019889121812, | |
| "tokens_seen": 966656000 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00033014942311329675, | |
| "loss": 2.6005, | |
| "theoretical_loss": 3.6598340008442234, | |
| "tokens_seen": 969932800 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00032955835067145827, | |
| "loss": 2.5902, | |
| "theoretical_loss": 3.6586532296761285, | |
| "tokens_seen": 973209600 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003289672782296198, | |
| "loss": 2.6145, | |
| "theoretical_loss": 3.657477536419047, | |
| "tokens_seen": 976486400 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00032837620578778136, | |
| "loss": 2.6437, | |
| "theoretical_loss": 3.656306882305022, | |
| "tokens_seen": 979763200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.6198785821735879, | |
| "debugging/distinct-1-grams": 0.7707728209964196, | |
| "debugging/distinct-2-grams": 0.9604165709732011, | |
| "debugging/entropy-1-grams": 5.836498792012382, | |
| "debugging/entropy-2-grams": 6.692653651879306, | |
| "debugging/length": 640.8333333333334, | |
| "debugging/num_segments": 12, | |
| "epoch": 0.35, | |
| "objective/train/docs_used": 522194, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.5617499351501465, | |
| "objective/train/theoretical_loss": 3.6551412289904697, | |
| "objective/train/tokens_used": 1003500000, | |
| "theoretical_loss": 3.6551412289904697, | |
| "tokens_seen": 983040000 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003277851333459429, | |
| "loss": 2.5923, | |
| "theoretical_loss": 3.6551412289904697, | |
| "tokens_seen": 983040000 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003271940609041044, | |
| "loss": 2.5912, | |
| "theoretical_loss": 3.6539805385501376, | |
| "tokens_seen": 986316800 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003266029884622659, | |
| "loss": 2.6037, | |
| "theoretical_loss": 3.652824773471171, | |
| "tokens_seen": 989593600 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00032601191602042745, | |
| "loss": 2.5777, | |
| "theoretical_loss": 3.651673896647277, | |
| "tokens_seen": 992870400 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.00032542084357858897, | |
| "loss": 2.5642, | |
| "theoretical_loss": 3.6505278713729985, | |
| "tokens_seen": 996147200 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.00032482977113675054, | |
| "loss": 2.5653, | |
| "theoretical_loss": 3.6493866613380774, | |
| "tokens_seen": 999424000 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.00032423869869491206, | |
| "loss": 2.5584, | |
| "theoretical_loss": 3.648250230621924, | |
| "tokens_seen": 1002700800 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0003236476262530736, | |
| "loss": 2.5586, | |
| "theoretical_loss": 3.647118543688179, | |
| "tokens_seen": 1005977600 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0003230565538112351, | |
| "loss": 2.5333, | |
| "theoretical_loss": 3.6459915653793633, | |
| "tokens_seen": 1009254400 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0003224654813693966, | |
| "loss": 2.5645, | |
| "theoretical_loss": 3.644869260911628, | |
| "tokens_seen": 1012531200 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.00032187440892755814, | |
| "loss": 2.5094, | |
| "theoretical_loss": 3.64375159586959, | |
| "tokens_seen": 1015808000 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0003212833364857197, | |
| "loss": 2.5334, | |
| "theoretical_loss": 3.642638536201252, | |
| "tokens_seen": 1019084800 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00032069226404388124, | |
| "loss": 2.5607, | |
| "theoretical_loss": 3.6415300482130135, | |
| "tokens_seen": 1022361600 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00032010119160204276, | |
| "loss": 2.5815, | |
| "theoretical_loss": 3.6404260985647667, | |
| "tokens_seen": 1025638400 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0003195101191602043, | |
| "loss": 2.531, | |
| "theoretical_loss": 3.6393266542650684, | |
| "tokens_seen": 1028915200 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0003189190467183658, | |
| "loss": 2.5341, | |
| "theoretical_loss": 3.638231682666401, | |
| "tokens_seen": 1032192000 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0003183279742765273, | |
| "loss": 2.5545, | |
| "theoretical_loss": 3.637141151460505, | |
| "tokens_seen": 1035468800 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00031773690183468884, | |
| "loss": 2.5548, | |
| "theoretical_loss": 3.636055028673799, | |
| "tokens_seen": 1038745600 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0003171458293928504, | |
| "loss": 2.5728, | |
| "theoretical_loss": 3.634973282662864, | |
| "tokens_seen": 1042022400 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00031655475695101194, | |
| "loss": 2.6221, | |
| "theoretical_loss": 3.6338958821100107, | |
| "tokens_seen": 1045299200 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00031596368450917346, | |
| "loss": 2.6254, | |
| "theoretical_loss": 3.63282279601892, | |
| "tokens_seen": 1048576000 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.000315372612067335, | |
| "loss": 2.6667, | |
| "theoretical_loss": 3.631753993710352, | |
| "tokens_seen": 1051852800 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0003147815396254965, | |
| "loss": 2.6569, | |
| "theoretical_loss": 3.630689444817925, | |
| "tokens_seen": 1055129600 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.000314190467183658, | |
| "loss": 2.648, | |
| "theoretical_loss": 3.629629119283967, | |
| "tokens_seen": 1058406400 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0003135993947418196, | |
| "loss": 2.668, | |
| "theoretical_loss": 3.628572987355434, | |
| "tokens_seen": 1061683200 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0003130083222999811, | |
| "loss": 2.6364, | |
| "theoretical_loss": 3.6275210195798913, | |
| "tokens_seen": 1064960000 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.00031241724985814264, | |
| "loss": 2.6346, | |
| "theoretical_loss": 3.626473186801564, | |
| "tokens_seen": 1068236800 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.00031182617741630416, | |
| "loss": 2.6531, | |
| "theoretical_loss": 3.6254294601574495, | |
| "tokens_seen": 1071513600 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0003112351049744657, | |
| "loss": 2.628, | |
| "theoretical_loss": 3.624389811073493, | |
| "tokens_seen": 1074790400 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003106440325326272, | |
| "loss": 2.6233, | |
| "theoretical_loss": 3.6233542112608257, | |
| "tokens_seen": 1078067200 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003100529600907888, | |
| "loss": 2.6271, | |
| "theoretical_loss": 3.6223226327120592, | |
| "tokens_seen": 1081344000 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003094618876489503, | |
| "loss": 2.6132, | |
| "theoretical_loss": 3.621295047697644, | |
| "tokens_seen": 1084620800 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003088708152071118, | |
| "loss": 2.6227, | |
| "theoretical_loss": 3.6202714287622833, | |
| "tokens_seen": 1087897600 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00030827974276527334, | |
| "loss": 2.6147, | |
| "theoretical_loss": 3.6192517487214038, | |
| "tokens_seen": 1091174400 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00030768867032343486, | |
| "loss": 2.5954, | |
| "theoretical_loss": 3.6182359806576834, | |
| "tokens_seen": 1094451200 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003070975978815964, | |
| "loss": 2.6142, | |
| "theoretical_loss": 3.6172240979176333, | |
| "tokens_seen": 1097728000 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003065065254397579, | |
| "loss": 2.5959, | |
| "theoretical_loss": 3.616216074108232, | |
| "tokens_seen": 1101004800 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003059154529979195, | |
| "loss": 2.6174, | |
| "theoretical_loss": 3.6152118830936164, | |
| "tokens_seen": 1104281600 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.000305324380556081, | |
| "loss": 2.6236, | |
| "theoretical_loss": 3.6142114989918195, | |
| "tokens_seen": 1107558400 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00030473330811424246, | |
| "loss": 2.6293, | |
| "theoretical_loss": 3.6132148961715624, | |
| "tokens_seen": 1110835200 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.000304142235672404, | |
| "loss": 2.5852, | |
| "theoretical_loss": 3.6122220492490964, | |
| "tokens_seen": 1114112000 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0003035511632305655, | |
| "loss": 2.5782, | |
| "theoretical_loss": 3.6112329330850894, | |
| "tokens_seen": 1117388800 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.000302960090788727, | |
| "loss": 2.607, | |
| "theoretical_loss": 3.61024752278157, | |
| "tokens_seen": 1120665600 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0003023690183468886, | |
| "loss": 2.56, | |
| "theoretical_loss": 3.6092657936789054, | |
| "tokens_seen": 1123942400 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0003017779459050501, | |
| "loss": 2.5852, | |
| "theoretical_loss": 3.6082877213528377, | |
| "tokens_seen": 1127219200 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00030118687346321164, | |
| "loss": 2.5555, | |
| "theoretical_loss": 3.60731328161156, | |
| "tokens_seen": 1130496000 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00030059580102137316, | |
| "loss": 2.5653, | |
| "theoretical_loss": 3.6063424504928365, | |
| "tokens_seen": 1133772800 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0003000047285795347, | |
| "loss": 2.5872, | |
| "theoretical_loss": 3.60537520426117, | |
| "tokens_seen": 1137049600 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0002994136561376962, | |
| "loss": 2.5925, | |
| "theoretical_loss": 3.6044115194050086, | |
| "tokens_seen": 1140326400 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0002988225836958578, | |
| "loss": 2.5761, | |
| "theoretical_loss": 3.603451372633997, | |
| "tokens_seen": 1143603200 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "objective/train/docs_used": 604425, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.565331220626831, | |
| "objective/train/theoretical_loss": 3.6024947408762698, | |
| "objective/train/tokens_used": 1167340000, | |
| "theoretical_loss": 3.6024947408762698, | |
| "tokens_seen": 1146880000 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0002982315112540193, | |
| "loss": 2.6043, | |
| "theoretical_loss": 3.6024947408762698, | |
| "tokens_seen": 1146880000 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0002976404388121808, | |
| "loss": 2.5725, | |
| "theoretical_loss": 3.601541601275783, | |
| "tokens_seen": 1150156800 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00029704936637034234, | |
| "loss": 2.548, | |
| "theoretical_loss": 3.6005919311896886, | |
| "tokens_seen": 1153433600 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00029647011537734067, | |
| "loss": 2.5699, | |
| "theoretical_loss": 3.5996457081857454, | |
| "tokens_seen": 1156710400 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00029587904293550214, | |
| "loss": 2.5709, | |
| "theoretical_loss": 3.598702910039772, | |
| "tokens_seen": 1159987200 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00029528797049366366, | |
| "loss": 2.5726, | |
| "theoretical_loss": 3.597763514733133, | |
| "tokens_seen": 1163264000 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00029469689805182523, | |
| "loss": 2.5542, | |
| "theoretical_loss": 3.59682750045027, | |
| "tokens_seen": 1166540800 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00029410582560998675, | |
| "loss": 2.5631, | |
| "theoretical_loss": 3.5958948455762583, | |
| "tokens_seen": 1169817600 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0002935147531681483, | |
| "loss": 2.5865, | |
| "theoretical_loss": 3.594965528694412, | |
| "tokens_seen": 1173094400 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0002929236807263098, | |
| "loss": 2.5499, | |
| "theoretical_loss": 3.594039528583913, | |
| "tokens_seen": 1176371200 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0002923326082844713, | |
| "loss": 2.559, | |
| "theoretical_loss": 3.5931168242174847, | |
| "tokens_seen": 1179648000 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00029174153584263284, | |
| "loss": 2.5468, | |
| "theoretical_loss": 3.59219739475909, | |
| "tokens_seen": 1182924800 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0002911504634007944, | |
| "loss": 2.5555, | |
| "theoretical_loss": 3.5912812195616732, | |
| "tokens_seen": 1186201600 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00029055939095895593, | |
| "loss": 2.5321, | |
| "theoretical_loss": 3.590368278164926, | |
| "tokens_seen": 1189478400 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.00028996831851711745, | |
| "loss": 2.5672, | |
| "theoretical_loss": 3.5894585502930902, | |
| "tokens_seen": 1192755200 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.000289377246075279, | |
| "loss": 2.5742, | |
| "theoretical_loss": 3.588552015852793, | |
| "tokens_seen": 1196032000 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0002887861736334405, | |
| "loss": 2.5502, | |
| "theoretical_loss": 3.5876486549309097, | |
| "tokens_seen": 1199308800 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.000288195101191602, | |
| "loss": 2.5514, | |
| "theoretical_loss": 3.586748447792462, | |
| "tokens_seen": 1202585600 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0002876040287497636, | |
| "loss": 2.5578, | |
| "theoretical_loss": 3.5858513748785423, | |
| "tokens_seen": 1205862400 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0002870129563079251, | |
| "loss": 2.6184, | |
| "theoretical_loss": 3.5849574168042704, | |
| "tokens_seen": 1209139200 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.00028642188386608663, | |
| "loss": 2.5991, | |
| "theoretical_loss": 3.5840665543567782, | |
| "tokens_seen": 1212416000 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.00028583081142424815, | |
| "loss": 2.5757, | |
| "theoretical_loss": 3.583178768493222, | |
| "tokens_seen": 1215692800 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00028523973898240967, | |
| "loss": 2.5659, | |
| "theoretical_loss": 3.5822940403388284, | |
| "tokens_seen": 1218969600 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002846486665405712, | |
| "loss": 2.5729, | |
| "theoretical_loss": 3.581412351184958, | |
| "tokens_seen": 1222246400 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002840575940987327, | |
| "loss": 2.5652, | |
| "theoretical_loss": 3.580533682487208, | |
| "tokens_seen": 1225523200 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002834665216568943, | |
| "loss": 2.5672, | |
| "theoretical_loss": 3.579658015863532, | |
| "tokens_seen": 1228800000 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002828754492150558, | |
| "loss": 2.5836, | |
| "theoretical_loss": 3.5787853330923927, | |
| "tokens_seen": 1232076800 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00028228437677321733, | |
| "loss": 2.5493, | |
| "theoretical_loss": 3.577915616110936, | |
| "tokens_seen": 1235353600 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00028169330433137885, | |
| "loss": 2.5739, | |
| "theoretical_loss": 3.577048847013194, | |
| "tokens_seen": 1238630400 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00028110223188954037, | |
| "loss": 2.5689, | |
| "theoretical_loss": 3.57618500804831, | |
| "tokens_seen": 1241907200 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002805111594477019, | |
| "loss": 2.5232, | |
| "theoretical_loss": 3.575324081618793, | |
| "tokens_seen": 1245184000 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00027992008700586347, | |
| "loss": 2.5686, | |
| "theoretical_loss": 3.5744660502787875, | |
| "tokens_seen": 1248460800 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.000279329014564025, | |
| "loss": 2.5236, | |
| "theoretical_loss": 3.5736108967323794, | |
| "tokens_seen": 1251737600 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.0002787379421221865, | |
| "loss": 2.5925, | |
| "theoretical_loss": 3.5727586038319155, | |
| "tokens_seen": 1255014400 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00027814686968034803, | |
| "loss": 2.5437, | |
| "theoretical_loss": 3.571909154576348, | |
| "tokens_seen": 1258291200 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00027755579723850955, | |
| "loss": 2.5461, | |
| "theoretical_loss": 3.5710625321096074, | |
| "tokens_seen": 1261568000 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00027696472479667107, | |
| "loss": 2.5288, | |
| "theoretical_loss": 3.570218719718989, | |
| "tokens_seen": 1264844800 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00027637365235483265, | |
| "loss": 2.5546, | |
| "theoretical_loss": 3.569377700833569, | |
| "tokens_seen": 1268121600 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00027578257991299417, | |
| "loss": 2.5354, | |
| "theoretical_loss": 3.568539459022639, | |
| "tokens_seen": 1271398400 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0002751915074711557, | |
| "loss": 2.5518, | |
| "theoretical_loss": 3.5677039779941584, | |
| "tokens_seen": 1274675200 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0002746004350293172, | |
| "loss": 2.5598, | |
| "theoretical_loss": 3.566871241593236, | |
| "tokens_seen": 1277952000 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00027400936258747873, | |
| "loss": 2.5445, | |
| "theoretical_loss": 3.5660412338006235, | |
| "tokens_seen": 1281228800 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00027341829014564025, | |
| "loss": 2.5946, | |
| "theoretical_loss": 3.565213938731236, | |
| "tokens_seen": 1284505600 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0002728272177038018, | |
| "loss": 2.5831, | |
| "theoretical_loss": 3.5643893406326868, | |
| "tokens_seen": 1287782400 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00027223614526196335, | |
| "loss": 2.5568, | |
| "theoretical_loss": 3.5635674238838466, | |
| "tokens_seen": 1291059200 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00027164507282012487, | |
| "loss": 2.5429, | |
| "theoretical_loss": 3.5627481729934196, | |
| "tokens_seen": 1294336000 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0002710540003782864, | |
| "loss": 2.5785, | |
| "theoretical_loss": 3.561931572598538, | |
| "tokens_seen": 1297612800 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0002704629279364479, | |
| "loss": 2.5493, | |
| "theoretical_loss": 3.5611176074633777, | |
| "tokens_seen": 1300889600 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00026987185549460943, | |
| "loss": 2.5602, | |
| "theoretical_loss": 3.5603062624777895, | |
| "tokens_seen": 1304166400 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00026928078305277095, | |
| "loss": 2.6058, | |
| "theoretical_loss": 3.559497522655951, | |
| "tokens_seen": 1307443200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.4840744728399488, | |
| "debugging/distinct-1-grams": 0.7704641680561449, | |
| "debugging/distinct-2-grams": 0.9548886056073045, | |
| "debugging/entropy-1-grams": 5.354809677980487, | |
| "debugging/entropy-2-grams": 6.077984561009924, | |
| "debugging/length": 581.0, | |
| "debugging/num_segments": 7, | |
| "epoch": 0.47, | |
| "objective/train/docs_used": 687328, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.7656729221343994, | |
| "objective/train/theoretical_loss": 3.5586913731350327, | |
| "objective/train/tokens_used": 1331180000, | |
| "theoretical_loss": 3.5586913731350327, | |
| "tokens_seen": 1310720000 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0002686897106109325, | |
| "loss": 2.6199, | |
| "theoretical_loss": 3.5586913731350327, | |
| "tokens_seen": 1310720000 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00026809863816909405, | |
| "loss": 2.6217, | |
| "theoretical_loss": 3.557887799173889, | |
| "tokens_seen": 1313996800 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00026750756572725557, | |
| "loss": 2.5979, | |
| "theoretical_loss": 3.557086786151754, | |
| "tokens_seen": 1317273600 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0002669164932854171, | |
| "loss": 2.5856, | |
| "theoretical_loss": 3.5562883195669697, | |
| "tokens_seen": 1320550400 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00026632542084357855, | |
| "loss": 2.5917, | |
| "theoretical_loss": 3.555492385035719, | |
| "tokens_seen": 1323827200 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0002657343484017401, | |
| "loss": 2.5804, | |
| "theoretical_loss": 3.5546989682907784, | |
| "tokens_seen": 1327104000 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026514327595990165, | |
| "loss": 2.5405, | |
| "theoretical_loss": 3.5539080551802895, | |
| "tokens_seen": 1330380800 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026455220351806317, | |
| "loss": 2.5578, | |
| "theoretical_loss": 3.553119631666546, | |
| "tokens_seen": 1333657600 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.0002639611310762247, | |
| "loss": 2.5811, | |
| "theoretical_loss": 3.5523336838247914, | |
| "tokens_seen": 1336934400 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.0002633700586343862, | |
| "loss": 2.5485, | |
| "theoretical_loss": 3.55155019784204, | |
| "tokens_seen": 1340211200 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026277898619254773, | |
| "loss": 2.5404, | |
| "theoretical_loss": 3.5507691600159053, | |
| "tokens_seen": 1343488000 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026218791375070925, | |
| "loss": 2.5348, | |
| "theoretical_loss": 3.5499905567534515, | |
| "tokens_seen": 1346764800 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026159684130887083, | |
| "loss": 2.5136, | |
| "theoretical_loss": 3.549214374570052, | |
| "tokens_seen": 1350041600 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026100576886703235, | |
| "loss": 2.5227, | |
| "theoretical_loss": 3.5484406000882665, | |
| "tokens_seen": 1353318400 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026041469642519387, | |
| "loss": 2.4971, | |
| "theoretical_loss": 3.5476692200367346, | |
| "tokens_seen": 1356595200 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.0002598236239833554, | |
| "loss": 2.5241, | |
| "theoretical_loss": 3.546900221249076, | |
| "tokens_seen": 1359872000 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.0002592325515415169, | |
| "loss": 2.5345, | |
| "theoretical_loss": 3.5461335906628157, | |
| "tokens_seen": 1363148800 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00025864147909967843, | |
| "loss": 2.5072, | |
| "theoretical_loss": 3.54536931531831, | |
| "tokens_seen": 1366425600 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00025805040665783995, | |
| "loss": 2.5089, | |
| "theoretical_loss": 3.5446073823576985, | |
| "tokens_seen": 1369702400 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00025745933421600153, | |
| "loss": 2.5158, | |
| "theoretical_loss": 3.543847779023859, | |
| "tokens_seen": 1372979200 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00025686826177416305, | |
| "loss": 2.5466, | |
| "theoretical_loss": 3.543090492659384, | |
| "tokens_seen": 1376256000 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00025627718933232457, | |
| "loss": 2.5079, | |
| "theoretical_loss": 3.542335510705562, | |
| "tokens_seen": 1379532800 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.0002556861168904861, | |
| "loss": 2.5099, | |
| "theoretical_loss": 3.541582820701378, | |
| "tokens_seen": 1382809600 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.0002550950444486476, | |
| "loss": 2.4877, | |
| "theoretical_loss": 3.5408324102825253, | |
| "tokens_seen": 1386086400 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.00025450397200680913, | |
| "loss": 2.4898, | |
| "theoretical_loss": 3.5400842671804265, | |
| "tokens_seen": 1389363200 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.0002539128995649707, | |
| "loss": 2.4993, | |
| "theoretical_loss": 3.5393383792212676, | |
| "tokens_seen": 1392640000 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.00025332182712313223, | |
| "loss": 2.4967, | |
| "theoretical_loss": 3.5385947343250486, | |
| "tokens_seen": 1395916800 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.00025273075468129375, | |
| "loss": 2.5021, | |
| "theoretical_loss": 3.53785332050464, | |
| "tokens_seen": 1399193600 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.00025213968223945527, | |
| "loss": 2.4976, | |
| "theoretical_loss": 3.537114125864853, | |
| "tokens_seen": 1402470400 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.0002515486097976168, | |
| "loss": 2.513, | |
| "theoretical_loss": 3.536377138601523, | |
| "tokens_seen": 1405747200 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.0002509575373557783, | |
| "loss": 2.5569, | |
| "theoretical_loss": 3.5356423470006018, | |
| "tokens_seen": 1409024000 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.0002503664649139399, | |
| "loss": 2.5286, | |
| "theoretical_loss": 3.5349097394372615, | |
| "tokens_seen": 1412300800 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.00024977539247210135, | |
| "loss": 2.517, | |
| "theoretical_loss": 3.5341793043750114, | |
| "tokens_seen": 1415577600 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.00024918432003026293, | |
| "loss": 2.544, | |
| "theoretical_loss": 3.533451030364823, | |
| "tokens_seen": 1418854400 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.00024859324758842445, | |
| "loss": 2.5262, | |
| "theoretical_loss": 3.5327249060442676, | |
| "tokens_seen": 1422131200 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.00024800217514658597, | |
| "loss": 2.5201, | |
| "theoretical_loss": 3.5320009201366616, | |
| "tokens_seen": 1425408000 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.0002474111027047475, | |
| "loss": 2.5654, | |
| "theoretical_loss": 3.531279061450226, | |
| "tokens_seen": 1428684800 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.000246820030262909, | |
| "loss": 2.5333, | |
| "theoretical_loss": 3.5305593188772533, | |
| "tokens_seen": 1431961600 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.00024622895782107053, | |
| "loss": 2.5331, | |
| "theoretical_loss": 3.5298416813932865, | |
| "tokens_seen": 1435238400 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.0002456378853792321, | |
| "loss": 2.5351, | |
| "theoretical_loss": 3.529126138056305, | |
| "tokens_seen": 1438515200 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.00024504681293739363, | |
| "loss": 2.5271, | |
| "theoretical_loss": 3.528412678005922, | |
| "tokens_seen": 1441792000 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 0.00024445574049555515, | |
| "loss": 2.5172, | |
| "theoretical_loss": 3.527701290462595, | |
| "tokens_seen": 1445068800 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 0.00024386466805371667, | |
| "loss": 2.5256, | |
| "theoretical_loss": 3.5269919647268377, | |
| "tokens_seen": 1448345600 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 0.0002432735956118782, | |
| "loss": 2.5083, | |
| "theoretical_loss": 3.526284690178448, | |
| "tokens_seen": 1451622400 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 0.0002426825231700397, | |
| "loss": 2.5048, | |
| "theoretical_loss": 3.5255794562757443, | |
| "tokens_seen": 1454899200 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 0.00024209145072820123, | |
| "loss": 2.4932, | |
| "theoretical_loss": 3.524876252554808, | |
| "tokens_seen": 1458176000 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 0.00024150037828636278, | |
| "loss": 2.4984, | |
| "theoretical_loss": 3.524175068628737, | |
| "tokens_seen": 1461452800 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 0.0002409093058445243, | |
| "loss": 2.51, | |
| "theoretical_loss": 3.5234758941869075, | |
| "tokens_seen": 1464729600 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 0.00024031823340268582, | |
| "loss": 2.5533, | |
| "theoretical_loss": 3.522778718994246, | |
| "tokens_seen": 1468006400 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.00023972716096084737, | |
| "loss": 2.5354, | |
| "theoretical_loss": 3.5220835328905062, | |
| "tokens_seen": 1471283200 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "objective/train/docs_used": 774403, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.511101245880127, | |
| "objective/train/theoretical_loss": 3.521390325789559, | |
| "objective/train/tokens_used": 1495020000, | |
| "theoretical_loss": 3.521390325789559, | |
| "tokens_seen": 1474560000 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.0002391360885190089, | |
| "loss": 2.5434, | |
| "theoretical_loss": 3.521390325789559, | |
| "tokens_seen": 1474560000 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.0002385450160771704, | |
| "loss": 2.5535, | |
| "theoretical_loss": 3.520699087678688, | |
| "tokens_seen": 1477836800 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.0002379775865330055, | |
| "loss": 2.5449, | |
| "theoretical_loss": 3.5200098086178926, | |
| "tokens_seen": 1481113600 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.000237386514091167, | |
| "loss": 2.5718, | |
| "theoretical_loss": 3.5193224787392015, | |
| "tokens_seen": 1484390400 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.00023679544164932853, | |
| "loss": 2.5271, | |
| "theoretical_loss": 3.518637088245993, | |
| "tokens_seen": 1487667200 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.00023620436920749008, | |
| "loss": 2.5737, | |
| "theoretical_loss": 3.5179536274123233, | |
| "tokens_seen": 1490944000 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.0002356132967656516, | |
| "loss": 2.5271, | |
| "theoretical_loss": 3.5172720865822633, | |
| "tokens_seen": 1494220800 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.00023502222432381312, | |
| "loss": 2.5327, | |
| "theoretical_loss": 3.5165924561692394, | |
| "tokens_seen": 1497497600 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 0.00023443115188197467, | |
| "loss": 2.4928, | |
| "theoretical_loss": 3.5159147266553887, | |
| "tokens_seen": 1500774400 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 0.0002338400794401362, | |
| "loss": 2.5022, | |
| "theoretical_loss": 3.515238888590916, | |
| "tokens_seen": 1504051200 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 0.0002332490069982977, | |
| "loss": 2.5223, | |
| "theoretical_loss": 3.51456493259346, | |
| "tokens_seen": 1507328000 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 0.00023265793455645926, | |
| "loss": 2.5179, | |
| "theoretical_loss": 3.5138928493474664, | |
| "tokens_seen": 1510604800 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 0.00023206686211462078, | |
| "loss": 2.4991, | |
| "theoretical_loss": 3.513222629603569, | |
| "tokens_seen": 1513881600 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 0.0002314757896727823, | |
| "loss": 2.5203, | |
| "theoretical_loss": 3.512554264177979, | |
| "tokens_seen": 1517158400 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 0.00023088471723094385, | |
| "loss": 2.507, | |
| "theoretical_loss": 3.511887743951874, | |
| "tokens_seen": 1520435200 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 0.00023029364478910537, | |
| "loss": 2.469, | |
| "theoretical_loss": 3.511223059870809, | |
| "tokens_seen": 1523712000 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.0002297025723472669, | |
| "loss": 2.4956, | |
| "theoretical_loss": 3.5105602029441143, | |
| "tokens_seen": 1526988800 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.00022911149990542844, | |
| "loss": 2.4958, | |
| "theoretical_loss": 3.5098991642443185, | |
| "tokens_seen": 1530265600 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.00022852042746358996, | |
| "loss": 2.5076, | |
| "theoretical_loss": 3.5092399349065655, | |
| "tokens_seen": 1533542400 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.00022792935502175148, | |
| "loss": 2.4798, | |
| "theoretical_loss": 3.5085825061280445, | |
| "tokens_seen": 1536819200 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.000227338282579913, | |
| "loss": 2.4804, | |
| "theoretical_loss": 3.507926869167425, | |
| "tokens_seen": 1540096000 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.00022674721013807455, | |
| "loss": 2.4992, | |
| "theoretical_loss": 3.507273015344296, | |
| "tokens_seen": 1543372800 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.00022615613769623604, | |
| "loss": 2.4689, | |
| "theoretical_loss": 3.5066209360386154, | |
| "tokens_seen": 1546649600 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.00022556506525439756, | |
| "loss": 2.4747, | |
| "theoretical_loss": 3.5059706226901595, | |
| "tokens_seen": 1549926400 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.0002249739928125591, | |
| "loss": 2.4259, | |
| "theoretical_loss": 3.505322066797988, | |
| "tokens_seen": 1553203200 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 0.00022438292037072063, | |
| "loss": 2.4646, | |
| "theoretical_loss": 3.504675259919904, | |
| "tokens_seen": 1556480000 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 0.00022379184792888215, | |
| "loss": 2.452, | |
| "theoretical_loss": 3.5040301936719302, | |
| "tokens_seen": 1559756800 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 0.0002232007754870437, | |
| "loss": 2.4496, | |
| "theoretical_loss": 3.5033868597277813, | |
| "tokens_seen": 1563033600 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 0.00022260970304520522, | |
| "loss": 2.4736, | |
| "theoretical_loss": 3.502745249818351, | |
| "tokens_seen": 1566310400 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 0.00022201863060336674, | |
| "loss": 2.4617, | |
| "theoretical_loss": 3.5021053557311985, | |
| "tokens_seen": 1569587200 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 0.0002214275581615283, | |
| "loss": 2.4671, | |
| "theoretical_loss": 3.501467169310044, | |
| "tokens_seen": 1572864000 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 0.0002208364857196898, | |
| "loss": 2.468, | |
| "theoretical_loss": 3.500830682454268, | |
| "tokens_seen": 1576140800 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 0.0002202572347266881, | |
| "loss": 2.507, | |
| "theoretical_loss": 3.5001958871184167, | |
| "tokens_seen": 1579417600 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.00021966616228484966, | |
| "loss": 2.5004, | |
| "theoretical_loss": 3.4995627753117127, | |
| "tokens_seen": 1582694400 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.00021907508984301118, | |
| "loss": 2.4757, | |
| "theoretical_loss": 3.498931339097571, | |
| "tokens_seen": 1585971200 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.0002184840174011727, | |
| "loss": 2.4741, | |
| "theoretical_loss": 3.4983015705931226, | |
| "tokens_seen": 1589248000 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.00021789294495933424, | |
| "loss": 2.4977, | |
| "theoretical_loss": 3.4976734619687377, | |
| "tokens_seen": 1592524800 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.00021730187251749577, | |
| "loss": 2.4703, | |
| "theoretical_loss": 3.497047005447558, | |
| "tokens_seen": 1595801600 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.00021671080007565726, | |
| "loss": 2.5305, | |
| "theoretical_loss": 3.4964221933050377, | |
| "tokens_seen": 1599078400 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.0002161197276338188, | |
| "loss": 2.5232, | |
| "theoretical_loss": 3.495799017868479, | |
| "tokens_seen": 1602355200 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.00021552865519198033, | |
| "loss": 2.5457, | |
| "theoretical_loss": 3.495177471516584, | |
| "tokens_seen": 1605632000 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.00021493758275014185, | |
| "loss": 2.5333, | |
| "theoretical_loss": 3.4945575466790024, | |
| "tokens_seen": 1608908800 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 0.00021434651030830337, | |
| "loss": 2.5204, | |
| "theoretical_loss": 3.4939392358358923, | |
| "tokens_seen": 1612185600 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 0.00021375543786646492, | |
| "loss": 2.5221, | |
| "theoretical_loss": 3.493322531517478, | |
| "tokens_seen": 1615462400 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 0.00021316436542462644, | |
| "loss": 2.5341, | |
| "theoretical_loss": 3.4927074263036157, | |
| "tokens_seen": 1618739200 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 0.00021257329298278796, | |
| "loss": 2.5277, | |
| "theoretical_loss": 3.492093912823367, | |
| "tokens_seen": 1622016000 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 0.0002119822205409495, | |
| "loss": 2.5054, | |
| "theoretical_loss": 3.4914819837545705, | |
| "tokens_seen": 1625292800 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 0.00021139114809911103, | |
| "loss": 2.5327, | |
| "theoretical_loss": 3.490871631823426, | |
| "tokens_seen": 1628569600 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 0.00021080007565727255, | |
| "loss": 2.4867, | |
| "theoretical_loss": 3.490262849804073, | |
| "tokens_seen": 1631846400 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 0.0002102090032154341, | |
| "loss": 2.5231, | |
| "theoretical_loss": 3.4896556305181834, | |
| "tokens_seen": 1635123200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.5511023142999684, | |
| "debugging/distinct-1-grams": 0.7979437388852283, | |
| "debugging/distinct-2-grams": 0.9830100075261367, | |
| "debugging/entropy-1-grams": 5.727417581977004, | |
| "debugging/entropy-2-grams": 6.430827220304544, | |
| "debugging/length": 552.8, | |
| "debugging/num_segments": 10, | |
| "epoch": 0.59, | |
| "objective/train/docs_used": 859276, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.7059619426727295, | |
| "objective/train/theoretical_loss": 3.4890499668345516, | |
| "objective/train/tokens_used": 1658860000, | |
| "theoretical_loss": 3.4890499668345516, | |
| "tokens_seen": 1638400000 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.00020961793077359562, | |
| "loss": 2.4859, | |
| "theoretical_loss": 3.4890499668345516, | |
| "tokens_seen": 1638400000 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.00020902685833175714, | |
| "loss": 2.5096, | |
| "theoretical_loss": 3.4884458516686947, | |
| "tokens_seen": 1641676800 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.00020843578588991869, | |
| "loss": 2.4816, | |
| "theoretical_loss": 3.487843277982451, | |
| "tokens_seen": 1644953600 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.0002078447134480802, | |
| "loss": 2.4955, | |
| "theoretical_loss": 3.4872422387835877, | |
| "tokens_seen": 1648230400 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.00020725364100624173, | |
| "loss": 2.4849, | |
| "theoretical_loss": 3.4866427271254077, | |
| "tokens_seen": 1651507200 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.00020666256856440328, | |
| "loss": 2.4517, | |
| "theoretical_loss": 3.4860447361063667, | |
| "tokens_seen": 1654784000 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.0002060714961225648, | |
| "loss": 2.4761, | |
| "theoretical_loss": 3.48544825886969, | |
| "tokens_seen": 1658060800 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.00020548042368072632, | |
| "loss": 2.4804, | |
| "theoretical_loss": 3.4848532886029924, | |
| "tokens_seen": 1661337600 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.00020488935123888786, | |
| "loss": 2.4687, | |
| "theoretical_loss": 3.4842598185379066, | |
| "tokens_seen": 1664614400 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "learning_rate": 0.00020429827879704939, | |
| "loss": 2.5146, | |
| "theoretical_loss": 3.4836678419497105, | |
| "tokens_seen": 1667891200 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "learning_rate": 0.0002037072063552109, | |
| "loss": 2.4929, | |
| "theoretical_loss": 3.483077352156964, | |
| "tokens_seen": 1671168000 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "learning_rate": 0.00020311613391337243, | |
| "loss": 2.4965, | |
| "theoretical_loss": 3.4824883425211413, | |
| "tokens_seen": 1674444800 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "learning_rate": 0.00020252506147153395, | |
| "loss": 2.4722, | |
| "theoretical_loss": 3.4819008064462764, | |
| "tokens_seen": 1677721600 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "learning_rate": 0.00020193398902969547, | |
| "loss": 2.4796, | |
| "theoretical_loss": 3.481314737378605, | |
| "tokens_seen": 1680998400 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "learning_rate": 0.000201342916587857, | |
| "loss": 2.4664, | |
| "theoretical_loss": 3.4807301288062145, | |
| "tokens_seen": 1684275200 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "learning_rate": 0.00020075184414601854, | |
| "loss": 2.4599, | |
| "theoretical_loss": 3.4801469742586955, | |
| "tokens_seen": 1687552000 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "learning_rate": 0.00020016077170418006, | |
| "loss": 2.4824, | |
| "theoretical_loss": 3.4795652673067954, | |
| "tokens_seen": 1690828800 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.00019956969926234158, | |
| "loss": 2.4943, | |
| "theoretical_loss": 3.478985001562082, | |
| "tokens_seen": 1694105600 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.00019897862682050313, | |
| "loss": 2.5213, | |
| "theoretical_loss": 3.4784061706766, | |
| "tokens_seen": 1697382400 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.00019838755437866465, | |
| "loss": 2.5173, | |
| "theoretical_loss": 3.477828768342543, | |
| "tokens_seen": 1700659200 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.00019779648193682617, | |
| "loss": 2.5176, | |
| "theoretical_loss": 3.477252788291917, | |
| "tokens_seen": 1703936000 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.00019720540949498772, | |
| "loss": 2.4807, | |
| "theoretical_loss": 3.4766782242962186, | |
| "tokens_seen": 1707212800 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.00019661433705314924, | |
| "loss": 2.4742, | |
| "theoretical_loss": 3.476105070166107, | |
| "tokens_seen": 1710489600 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.00019602326461131076, | |
| "loss": 2.4524, | |
| "theoretical_loss": 3.475533319751085, | |
| "tokens_seen": 1713766400 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.0001954321921694723, | |
| "loss": 2.4679, | |
| "theoretical_loss": 3.4749629669391813, | |
| "tokens_seen": 1717043200 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.00019484111972763383, | |
| "loss": 2.4656, | |
| "theoretical_loss": 3.474394005656636, | |
| "tokens_seen": 1720320000 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.00019425004728579535, | |
| "loss": 2.4875, | |
| "theoretical_loss": 3.473826429867589, | |
| "tokens_seen": 1723596800 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.0001936589748439569, | |
| "loss": 2.4938, | |
| "theoretical_loss": 3.4732602335737734, | |
| "tokens_seen": 1726873600 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.00019306790240211842, | |
| "loss": 2.4571, | |
| "theoretical_loss": 3.47269541081421, | |
| "tokens_seen": 1730150400 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.00019247682996027994, | |
| "loss": 2.476, | |
| "theoretical_loss": 3.4721319556649046, | |
| "tokens_seen": 1733427200 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.00019188575751844146, | |
| "loss": 2.4892, | |
| "theoretical_loss": 3.4715698622385487, | |
| "tokens_seen": 1736704000 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.000191294685076603, | |
| "loss": 2.502, | |
| "theoretical_loss": 3.471009124684226, | |
| "tokens_seen": 1739980800 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.00019070361263476453, | |
| "loss": 2.51, | |
| "theoretical_loss": 3.470449737187118, | |
| "tokens_seen": 1743257600 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.00019011254019292605, | |
| "loss": 2.4682, | |
| "theoretical_loss": 3.4698916939682123, | |
| "tokens_seen": 1746534400 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.0001895214677510876, | |
| "loss": 2.4698, | |
| "theoretical_loss": 3.4693349892840186, | |
| "tokens_seen": 1749811200 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 0.00018893039530924912, | |
| "loss": 2.5057, | |
| "theoretical_loss": 3.4687796174262826, | |
| "tokens_seen": 1753088000 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 0.0001883393228674106, | |
| "loss": 2.4379, | |
| "theoretical_loss": 3.468225572721704, | |
| "tokens_seen": 1756364800 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 0.00018774825042557216, | |
| "loss": 2.4452, | |
| "theoretical_loss": 3.4676728495316587, | |
| "tokens_seen": 1759641600 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 0.00018715717798373368, | |
| "loss": 2.433, | |
| "theoretical_loss": 3.467121442251922, | |
| "tokens_seen": 1762918400 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 0.00018657792699073198, | |
| "loss": 2.4977, | |
| "theoretical_loss": 3.4665713453123965, | |
| "tokens_seen": 1766195200 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 0.00018598685454889353, | |
| "loss": 2.4629, | |
| "theoretical_loss": 3.46602255317684, | |
| "tokens_seen": 1769472000 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 0.00018539578210705505, | |
| "loss": 2.4492, | |
| "theoretical_loss": 3.465475060342598, | |
| "tokens_seen": 1772748800 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 0.00018480470966521657, | |
| "loss": 2.4514, | |
| "theoretical_loss": 3.4649288613403386, | |
| "tokens_seen": 1776025600 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.00018421363722337811, | |
| "loss": 2.474, | |
| "theoretical_loss": 3.4643839507337892, | |
| "tokens_seen": 1779302400 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.00018362256478153964, | |
| "loss": 2.4632, | |
| "theoretical_loss": 3.4638403231194763, | |
| "tokens_seen": 1782579200 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.00018303149233970116, | |
| "loss": 2.4437, | |
| "theoretical_loss": 3.463297973126469, | |
| "tokens_seen": 1785856000 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.0001824404198978627, | |
| "loss": 2.4239, | |
| "theoretical_loss": 3.462756895416121, | |
| "tokens_seen": 1789132800 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.00018184934745602423, | |
| "loss": 2.4115, | |
| "theoretical_loss": 3.4622170846818205, | |
| "tokens_seen": 1792409600 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.00018125827501418575, | |
| "loss": 2.4332, | |
| "theoretical_loss": 3.4616785356487387, | |
| "tokens_seen": 1795686400 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.0001806672025723473, | |
| "loss": 2.4692, | |
| "theoretical_loss": 3.461141243073582, | |
| "tokens_seen": 1798963200 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "objective/train/docs_used": 945295, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.382375955581665, | |
| "objective/train/theoretical_loss": 3.4606052017443467, | |
| "objective/train/tokens_used": 1822700000, | |
| "theoretical_loss": 3.4606052017443467, | |
| "tokens_seen": 1802240000 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.00018007613013050881, | |
| "loss": 2.4503, | |
| "theoretical_loss": 3.4606052017443467, | |
| "tokens_seen": 1802240000 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.00017948505768867034, | |
| "loss": 2.4626, | |
| "theoretical_loss": 3.4600704064800745, | |
| "tokens_seen": 1805516800 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 0.00017889398524683183, | |
| "loss": 2.4555, | |
| "theoretical_loss": 3.459536852130613, | |
| "tokens_seen": 1808793600 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 0.00017830291280499338, | |
| "loss": 2.4643, | |
| "theoretical_loss": 3.4590045335763775, | |
| "tokens_seen": 1812070400 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 0.0001777118403631549, | |
| "loss": 2.4707, | |
| "theoretical_loss": 3.4584734457281128, | |
| "tokens_seen": 1815347200 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 0.00017712076792131642, | |
| "loss": 2.4604, | |
| "theoretical_loss": 3.457943583526662, | |
| "tokens_seen": 1818624000 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 0.00017652969547947797, | |
| "loss": 2.4922, | |
| "theoretical_loss": 3.45741494194273, | |
| "tokens_seen": 1821900800 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 0.0001759386230376395, | |
| "loss": 2.5316, | |
| "theoretical_loss": 3.4568875159766606, | |
| "tokens_seen": 1825177600 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 0.000175347550595801, | |
| "loss": 2.4871, | |
| "theoretical_loss": 3.4563613006582017, | |
| "tokens_seen": 1828454400 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 0.00017475647815396256, | |
| "loss": 2.4589, | |
| "theoretical_loss": 3.4558362910462863, | |
| "tokens_seen": 1831731200 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.00017416540571212408, | |
| "loss": 2.4763, | |
| "theoretical_loss": 3.4553124822288055, | |
| "tokens_seen": 1835008000 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.0001735743332702856, | |
| "loss": 2.5081, | |
| "theoretical_loss": 3.4547898693223873, | |
| "tokens_seen": 1838284800 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.00017298326082844715, | |
| "loss": 2.4759, | |
| "theoretical_loss": 3.4542684474721805, | |
| "tokens_seen": 1841561600 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.00017239218838660867, | |
| "loss": 2.4479, | |
| "theoretical_loss": 3.453748211851636, | |
| "tokens_seen": 1844838400 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.0001718011159447702, | |
| "loss": 2.4585, | |
| "theoretical_loss": 3.453229157662291, | |
| "tokens_seen": 1848115200 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.00017121004350293173, | |
| "loss": 2.4461, | |
| "theoretical_loss": 3.452711280133558, | |
| "tokens_seen": 1851392000 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.00017061897106109326, | |
| "loss": 2.4873, | |
| "theoretical_loss": 3.4521945745225118, | |
| "tokens_seen": 1854668800 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.00017002789861925478, | |
| "loss": 2.4803, | |
| "theoretical_loss": 3.451679036113682, | |
| "tokens_seen": 1857945600 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.00016943682617741632, | |
| "loss": 2.4435, | |
| "theoretical_loss": 3.451164660218848, | |
| "tokens_seen": 1861222400 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 0.00016884575373557785, | |
| "loss": 2.467, | |
| "theoretical_loss": 3.4506514421768273, | |
| "tokens_seen": 1864499200 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 0.00016825468129373937, | |
| "loss": 2.501, | |
| "theoretical_loss": 3.45013937735328, | |
| "tokens_seen": 1867776000 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 0.00016766360885190091, | |
| "loss": 2.5035, | |
| "theoretical_loss": 3.4496284611405024, | |
| "tokens_seen": 1871052800 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 0.00016707253641006243, | |
| "loss": 2.5022, | |
| "theoretical_loss": 3.4491186889572303, | |
| "tokens_seen": 1874329600 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 0.00016648146396822396, | |
| "loss": 2.4757, | |
| "theoretical_loss": 3.4486100562484396, | |
| "tokens_seen": 1877606400 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 0.00016589039152638548, | |
| "loss": 2.4724, | |
| "theoretical_loss": 3.448102558485152, | |
| "tokens_seen": 1880883200 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 0.00016529931908454702, | |
| "loss": 2.4937, | |
| "theoretical_loss": 3.447596191164239, | |
| "tokens_seen": 1884160000 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 0.00016470824664270854, | |
| "loss": 2.5345, | |
| "theoretical_loss": 3.447090949808233, | |
| "tokens_seen": 1887436800 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.00016411717420087004, | |
| "loss": 2.4963, | |
| "theoretical_loss": 3.446586829965132, | |
| "tokens_seen": 1890713600 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.00016352610175903159, | |
| "loss": 2.4926, | |
| "theoretical_loss": 3.4460838272082173, | |
| "tokens_seen": 1893990400 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.0001629350293171931, | |
| "loss": 2.4806, | |
| "theoretical_loss": 3.4455819371358594, | |
| "tokens_seen": 1897267200 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.00016234395687535463, | |
| "loss": 2.4956, | |
| "theoretical_loss": 3.445081155371338, | |
| "tokens_seen": 1900544000 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.00016175288443351618, | |
| "loss": 2.5078, | |
| "theoretical_loss": 3.4445814775626564, | |
| "tokens_seen": 1903820800 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.0001611618119916777, | |
| "loss": 2.4699, | |
| "theoretical_loss": 3.4440828993823605, | |
| "tokens_seen": 1907097600 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.00016057073954983922, | |
| "loss": 2.4776, | |
| "theoretical_loss": 3.4435854165273567, | |
| "tokens_seen": 1910374400 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.00015997966710800077, | |
| "loss": 2.4853, | |
| "theoretical_loss": 3.443089024718735, | |
| "tokens_seen": 1913651200 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.00015938859466616229, | |
| "loss": 2.4962, | |
| "theoretical_loss": 3.4425937197015912, | |
| "tokens_seen": 1916928000 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 0.0001587975222243238, | |
| "loss": 2.4565, | |
| "theoretical_loss": 3.4420994972448513, | |
| "tokens_seen": 1920204800 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 0.00015820644978248535, | |
| "loss": 2.458, | |
| "theoretical_loss": 3.4416063531410988, | |
| "tokens_seen": 1923481600 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 0.00015761537734064688, | |
| "loss": 2.4562, | |
| "theoretical_loss": 3.4411142832064003, | |
| "tokens_seen": 1926758400 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 0.0001570243048988084, | |
| "loss": 2.4597, | |
| "theoretical_loss": 3.4406232832801376, | |
| "tokens_seen": 1930035200 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 0.00015643323245696994, | |
| "loss": 2.4618, | |
| "theoretical_loss": 3.440133349224835, | |
| "tokens_seen": 1933312000 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 0.00015584216001513147, | |
| "loss": 2.4471, | |
| "theoretical_loss": 3.439644476925995, | |
| "tokens_seen": 1936588800 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 0.00015525108757329299, | |
| "loss": 2.4407, | |
| "theoretical_loss": 3.4391566622919303, | |
| "tokens_seen": 1939865600 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 0.0001546600151314545, | |
| "loss": 2.4691, | |
| "theoretical_loss": 3.438669901253598, | |
| "tokens_seen": 1943142400 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.00015406894268961605, | |
| "loss": 2.47, | |
| "theoretical_loss": 3.4381841897644385, | |
| "tokens_seen": 1946419200 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.00015347787024777758, | |
| "loss": 2.4146, | |
| "theoretical_loss": 3.4376995238002137, | |
| "tokens_seen": 1949696000 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.00015289861925477585, | |
| "loss": 2.4399, | |
| "theoretical_loss": 3.4372158993588444, | |
| "tokens_seen": 1952972800 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.0001523075468129374, | |
| "loss": 2.4267, | |
| "theoretical_loss": 3.436733312460253, | |
| "tokens_seen": 1956249600 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.00015171647437109892, | |
| "loss": 2.4118, | |
| "theoretical_loss": 3.436251759146206, | |
| "tokens_seen": 1959526400 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.00015112540192926044, | |
| "loss": 2.4501, | |
| "theoretical_loss": 3.435771235480157, | |
| "tokens_seen": 1962803200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.6850276452122688, | |
| "debugging/distinct-1-grams": 0.7993635523186263, | |
| "debugging/distinct-2-grams": 0.9598594090668269, | |
| "debugging/entropy-1-grams": 6.288061727484086, | |
| "debugging/entropy-2-grams": 7.265518235035818, | |
| "debugging/length": 620.4347826086956, | |
| "debugging/num_segments": 23, | |
| "epoch": 0.7, | |
| "objective/train/docs_used": 1029397, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.542283296585083, | |
| "objective/train/theoretical_loss": 3.435291737547092, | |
| "objective/train/tokens_used": 1986540000, | |
| "theoretical_loss": 3.435291737547092, | |
| "tokens_seen": 1966080000 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.00015053432948742198, | |
| "loss": 2.473, | |
| "theoretical_loss": 3.435291737547092, | |
| "tokens_seen": 1966080000 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.0001499432570455835, | |
| "loss": 2.4627, | |
| "theoretical_loss": 3.434813261453378, | |
| "tokens_seen": 1969356800 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.00014935218460374503, | |
| "loss": 2.4539, | |
| "theoretical_loss": 3.434335803326607, | |
| "tokens_seen": 1972633600 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "learning_rate": 0.00014876111216190657, | |
| "loss": 2.4198, | |
| "theoretical_loss": 3.433859359315448, | |
| "tokens_seen": 1975910400 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "learning_rate": 0.0001481700397200681, | |
| "loss": 2.4494, | |
| "theoretical_loss": 3.433383925589498, | |
| "tokens_seen": 1979187200 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "learning_rate": 0.00014757896727822962, | |
| "loss": 2.4186, | |
| "theoretical_loss": 3.432909498339132, | |
| "tokens_seen": 1982464000 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "learning_rate": 0.00014698789483639116, | |
| "loss": 2.4585, | |
| "theoretical_loss": 3.432436073775358, | |
| "tokens_seen": 1985740800 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "learning_rate": 0.00014639682239455268, | |
| "loss": 2.4603, | |
| "theoretical_loss": 3.4319636481296687, | |
| "tokens_seen": 1989017600 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "learning_rate": 0.0001458057499527142, | |
| "loss": 2.455, | |
| "theoretical_loss": 3.4314922176539002, | |
| "tokens_seen": 1992294400 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "learning_rate": 0.00014521467751087575, | |
| "loss": 2.5112, | |
| "theoretical_loss": 3.431021778620088, | |
| "tokens_seen": 1995571200 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "learning_rate": 0.00014462360506903727, | |
| "loss": 2.5101, | |
| "theoretical_loss": 3.4305523273203233, | |
| "tokens_seen": 1998848000 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.0001440325326271988, | |
| "loss": 2.5214, | |
| "theoretical_loss": 3.430083860066617, | |
| "tokens_seen": 2002124800 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.00014344146018536034, | |
| "loss": 2.4883, | |
| "theoretical_loss": 3.429616373190753, | |
| "tokens_seen": 2005401600 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.00014285038774352186, | |
| "loss": 2.537, | |
| "theoretical_loss": 3.4291498630441586, | |
| "tokens_seen": 2008678400 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.00014225931530168338, | |
| "loss": 2.4942, | |
| "theoretical_loss": 3.4286843259977613, | |
| "tokens_seen": 2011955200 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.0001416682428598449, | |
| "loss": 2.4595, | |
| "theoretical_loss": 3.4282197584418546, | |
| "tokens_seen": 2015232000 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.00014107717041800645, | |
| "loss": 2.4701, | |
| "theoretical_loss": 3.427756156785965, | |
| "tokens_seen": 2018508800 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.00014048609797616795, | |
| "loss": 2.4415, | |
| "theoretical_loss": 3.427293517458717, | |
| "tokens_seen": 2021785600 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.00013989502553432947, | |
| "loss": 2.4121, | |
| "theoretical_loss": 3.4268318369076995, | |
| "tokens_seen": 2025062400 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.00013930395309249102, | |
| "loss": 2.4075, | |
| "theoretical_loss": 3.4263711115993383, | |
| "tokens_seen": 2028339200 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.00013871288065065254, | |
| "loss": 2.4143, | |
| "theoretical_loss": 3.425911338018763, | |
| "tokens_seen": 2031616000 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.00013812180820881406, | |
| "loss": 2.4465, | |
| "theoretical_loss": 3.425452512669678, | |
| "tokens_seen": 2034892800 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.0001375307357669756, | |
| "loss": 2.4292, | |
| "theoretical_loss": 3.4249946320742364, | |
| "tokens_seen": 2038169600 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.00013693966332513713, | |
| "loss": 2.4195, | |
| "theoretical_loss": 3.4245376927729128, | |
| "tokens_seen": 2041446400 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.00013634859088329865, | |
| "loss": 2.4473, | |
| "theoretical_loss": 3.424081691324375, | |
| "tokens_seen": 2044723200 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.0001357575184414602, | |
| "loss": 2.4285, | |
| "theoretical_loss": 3.4236266243053626, | |
| "tokens_seen": 2048000000 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.00013516644599962171, | |
| "loss": 2.4598, | |
| "theoretical_loss": 3.4231724883105628, | |
| "tokens_seen": 2051276800 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.00013457537355778324, | |
| "loss": 2.421, | |
| "theoretical_loss": 3.422719279952486, | |
| "tokens_seen": 2054553600 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.00013398430111594478, | |
| "loss": 2.3807, | |
| "theoretical_loss": 3.422266995861345, | |
| "tokens_seen": 2057830400 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 0.0001333932286741063, | |
| "loss": 2.3945, | |
| "theoretical_loss": 3.421815632684937, | |
| "tokens_seen": 2061107200 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 0.00013280215623226783, | |
| "loss": 2.3982, | |
| "theoretical_loss": 3.4213651870885196, | |
| "tokens_seen": 2064384000 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 0.00013221108379042937, | |
| "loss": 2.4228, | |
| "theoretical_loss": 3.4209156557546954, | |
| "tokens_seen": 2067660800 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 0.0001316200113485909, | |
| "loss": 2.4581, | |
| "theoretical_loss": 3.420467035383294, | |
| "tokens_seen": 2070937600 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 0.00013102893890675241, | |
| "loss": 2.4272, | |
| "theoretical_loss": 3.4200193226912554, | |
| "tokens_seen": 2074214400 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 0.00013043786646491394, | |
| "loss": 2.4619, | |
| "theoretical_loss": 3.419572514412513, | |
| "tokens_seen": 2077491200 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 0.00012984679402307548, | |
| "loss": 2.4987, | |
| "theoretical_loss": 3.4191266072978816, | |
| "tokens_seen": 2080768000 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 0.000129255721581237, | |
| "loss": 2.4782, | |
| "theoretical_loss": 3.418681598114942, | |
| "tokens_seen": 2084044800 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.00012866464913939852, | |
| "loss": 2.4783, | |
| "theoretical_loss": 3.4182374836479266, | |
| "tokens_seen": 2087321600 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.00012807357669756007, | |
| "loss": 2.4981, | |
| "theoretical_loss": 3.4177942606976126, | |
| "tokens_seen": 2090598400 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.0001274825042557216, | |
| "loss": 2.4907, | |
| "theoretical_loss": 3.4173519260812064, | |
| "tokens_seen": 2093875200 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.00012689143181388311, | |
| "loss": 2.4911, | |
| "theoretical_loss": 3.4169104766322356, | |
| "tokens_seen": 2097152000 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.00012630035937204464, | |
| "loss": 2.4687, | |
| "theoretical_loss": 3.4164699092004405, | |
| "tokens_seen": 2100428800 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.00012570928693020616, | |
| "loss": 2.4532, | |
| "theoretical_loss": 3.416030220651664, | |
| "tokens_seen": 2103705600 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.00012511821448836768, | |
| "loss": 2.4404, | |
| "theoretical_loss": 3.4155914078677467, | |
| "tokens_seen": 2106982400 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.00012452714204652922, | |
| "loss": 2.4475, | |
| "theoretical_loss": 3.4151534677464195, | |
| "tokens_seen": 2110259200 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.00012393606960469075, | |
| "loss": 2.4738, | |
| "theoretical_loss": 3.414716397201196, | |
| "tokens_seen": 2113536000 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "learning_rate": 0.0001233449971628523, | |
| "loss": 2.4582, | |
| "theoretical_loss": 3.4142801931612725, | |
| "tokens_seen": 2116812800 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "learning_rate": 0.00012275392472101381, | |
| "loss": 2.4244, | |
| "theoretical_loss": 3.41384485257142, | |
| "tokens_seen": 2120089600 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "learning_rate": 0.00012216285227917533, | |
| "loss": 2.4584, | |
| "theoretical_loss": 3.4134103723918834, | |
| "tokens_seen": 2123366400 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "learning_rate": 0.00012157177983733686, | |
| "loss": 2.4488, | |
| "theoretical_loss": 3.412976749598278, | |
| "tokens_seen": 2126643200 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "objective/train/docs_used": 1116328, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.2849831581115723, | |
| "objective/train/theoretical_loss": 3.412543981181491, | |
| "objective/train/tokens_used": 2150380000, | |
| "theoretical_loss": 3.412543981181491, | |
| "tokens_seen": 2129920000 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "learning_rate": 0.00012098070739549839, | |
| "loss": 2.4509, | |
| "theoretical_loss": 3.412543981181491, | |
| "tokens_seen": 2129920000 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "learning_rate": 0.00012038963495365992, | |
| "loss": 2.4389, | |
| "theoretical_loss": 3.412112064147577, | |
| "tokens_seen": 2133196800 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "learning_rate": 0.00011979856251182145, | |
| "loss": 2.4258, | |
| "theoretical_loss": 3.4116809955176617, | |
| "tokens_seen": 2136473600 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "learning_rate": 0.00011920749006998298, | |
| "loss": 2.4256, | |
| "theoretical_loss": 3.4112507723278416, | |
| "tokens_seen": 2139750400 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.00011861641762814451, | |
| "loss": 2.4627, | |
| "theoretical_loss": 3.410821391629086, | |
| "tokens_seen": 2143027200 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.00011802534518630603, | |
| "loss": 2.4488, | |
| "theoretical_loss": 3.4103928504871406, | |
| "tokens_seen": 2146304000 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.00011743427274446757, | |
| "loss": 2.4538, | |
| "theoretical_loss": 3.4099651459824294, | |
| "tokens_seen": 2149580800 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.0001168432003026291, | |
| "loss": 2.4259, | |
| "theoretical_loss": 3.409538275209961, | |
| "tokens_seen": 2152857600 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.00011625212786079062, | |
| "loss": 2.4701, | |
| "theoretical_loss": 3.409112235279233, | |
| "tokens_seen": 2156134400 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.00011566105541895216, | |
| "loss": 2.5087, | |
| "theoretical_loss": 3.4086870233141355, | |
| "tokens_seen": 2159411200 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.00011506998297711367, | |
| "loss": 2.5059, | |
| "theoretical_loss": 3.4082626364528625, | |
| "tokens_seen": 2162688000 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.0001144789105352752, | |
| "loss": 2.5062, | |
| "theoretical_loss": 3.4078390718478153, | |
| "tokens_seen": 2165964800 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.00011388783809343673, | |
| "loss": 2.4786, | |
| "theoretical_loss": 3.4074163266655115, | |
| "tokens_seen": 2169241600 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "learning_rate": 0.00011329676565159826, | |
| "loss": 2.4781, | |
| "theoretical_loss": 3.4069943980864945, | |
| "tokens_seen": 2172518400 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "learning_rate": 0.00011270569320975979, | |
| "loss": 2.4799, | |
| "theoretical_loss": 3.4065732833052436, | |
| "tokens_seen": 2175795200 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "learning_rate": 0.00011211462076792132, | |
| "loss": 2.48, | |
| "theoretical_loss": 3.406152979530081, | |
| "tokens_seen": 2179072000 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "learning_rate": 0.00011152354832608284, | |
| "loss": 2.4456, | |
| "theoretical_loss": 3.405733483983088, | |
| "tokens_seen": 2182348800 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "learning_rate": 0.00011093247588424438, | |
| "loss": 2.4384, | |
| "theoretical_loss": 3.4053147939000117, | |
| "tokens_seen": 2185625600 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "learning_rate": 0.00011034140344240591, | |
| "loss": 2.4279, | |
| "theoretical_loss": 3.4048969065301806, | |
| "tokens_seen": 2188902400 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "learning_rate": 0.00010975033100056743, | |
| "loss": 2.4031, | |
| "theoretical_loss": 3.4044798191364163, | |
| "tokens_seen": 2192179200 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "learning_rate": 0.00010915925855872897, | |
| "loss": 2.3709, | |
| "theoretical_loss": 3.4040635289949472, | |
| "tokens_seen": 2195456000 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.00010856818611689049, | |
| "loss": 2.4149, | |
| "theoretical_loss": 3.403648033395324, | |
| "tokens_seen": 2198732800 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.00010797711367505201, | |
| "loss": 2.408, | |
| "theoretical_loss": 3.403233329640333, | |
| "tokens_seen": 2202009600 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.00010738604123321354, | |
| "loss": 2.4003, | |
| "theoretical_loss": 3.4028194150459137, | |
| "tokens_seen": 2205286400 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.00010679496879137507, | |
| "loss": 2.393, | |
| "theoretical_loss": 3.4024062869410745, | |
| "tokens_seen": 2208563200 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.0001062038963495366, | |
| "loss": 2.4119, | |
| "theoretical_loss": 3.4019939426678096, | |
| "tokens_seen": 2211840000 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.00010561282390769813, | |
| "loss": 2.3949, | |
| "theoretical_loss": 3.401582379581017, | |
| "tokens_seen": 2215116800 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.00010502175146585965, | |
| "loss": 2.3869, | |
| "theoretical_loss": 3.401171595048417, | |
| "tokens_seen": 2218393600 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.00010443067902402119, | |
| "loss": 2.3857, | |
| "theoretical_loss": 3.40076158645047, | |
| "tokens_seen": 2221670400 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.00010383960658218271, | |
| "loss": 2.3841, | |
| "theoretical_loss": 3.400352351180298, | |
| "tokens_seen": 2224947200 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 0.00010324853414034424, | |
| "loss": 2.423, | |
| "theoretical_loss": 3.3999438866436047, | |
| "tokens_seen": 2228224000 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 0.00010265746169850578, | |
| "loss": 2.4174, | |
| "theoretical_loss": 3.399536190258594, | |
| "tokens_seen": 2231500800 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 0.0001020663892566673, | |
| "loss": 2.4116, | |
| "theoretical_loss": 3.399129259455895, | |
| "tokens_seen": 2234777600 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 0.00010147531681482883, | |
| "loss": 2.4217, | |
| "theoretical_loss": 3.3987230916784803, | |
| "tokens_seen": 2238054400 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 0.00010089606582182713, | |
| "loss": 2.4394, | |
| "theoretical_loss": 3.398317684381593, | |
| "tokens_seen": 2241331200 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 0.00010030499337998865, | |
| "loss": 2.4456, | |
| "theoretical_loss": 3.397913035032666, | |
| "tokens_seen": 2244608000 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 9.971392093815019e-05, | |
| "loss": 2.428, | |
| "theoretical_loss": 3.3975091411112484, | |
| "tokens_seen": 2247884800 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 9.912284849631172e-05, | |
| "loss": 2.4186, | |
| "theoretical_loss": 3.397106000108929, | |
| "tokens_seen": 2251161600 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 9.853177605447323e-05, | |
| "loss": 2.4073, | |
| "theoretical_loss": 3.3967036095292613, | |
| "tokens_seen": 2254438400 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 9.794070361263476e-05, | |
| "loss": 2.4566, | |
| "theoretical_loss": 3.3963019668876897, | |
| "tokens_seen": 2257715200 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 9.73496311707963e-05, | |
| "loss": 2.4387, | |
| "theoretical_loss": 3.395901069711476, | |
| "tokens_seen": 2260992000 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 9.675855872895782e-05, | |
| "loss": 2.4343, | |
| "theoretical_loss": 3.3955009155396247, | |
| "tokens_seen": 2264268800 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 9.616748628711935e-05, | |
| "loss": 2.4714, | |
| "theoretical_loss": 3.3951015019228117, | |
| "tokens_seen": 2267545600 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 9.557641384528087e-05, | |
| "loss": 2.4124, | |
| "theoretical_loss": 3.3947028264233112, | |
| "tokens_seen": 2270822400 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 9.498534140344241e-05, | |
| "loss": 2.4503, | |
| "theoretical_loss": 3.3943048866149255, | |
| "tokens_seen": 2274099200 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 9.439426896160394e-05, | |
| "loss": 2.4665, | |
| "theoretical_loss": 3.3939076800829118, | |
| "tokens_seen": 2277376000 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 9.380319651976546e-05, | |
| "loss": 2.4539, | |
| "theoretical_loss": 3.393511204423914, | |
| "tokens_seen": 2280652800 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 9.3212124077927e-05, | |
| "loss": 2.4307, | |
| "theoretical_loss": 3.3931154572458917, | |
| "tokens_seen": 2283929600 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 9.262105163608853e-05, | |
| "loss": 2.4329, | |
| "theoretical_loss": 3.3927204361680503, | |
| "tokens_seen": 2287206400 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 9.202997919425005e-05, | |
| "loss": 2.4471, | |
| "theoretical_loss": 3.3923261388207733, | |
| "tokens_seen": 2290483200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.5163479427815987, | |
| "debugging/distinct-1-grams": 0.8142531206442917, | |
| "debugging/distinct-2-grams": 0.9809195230897494, | |
| "debugging/entropy-1-grams": 5.498653003025727, | |
| "debugging/entropy-2-grams": 6.0913827978956725, | |
| "debugging/length": 627.6666666666666, | |
| "debugging/num_segments": 6, | |
| "epoch": 0.82, | |
| "objective/train/docs_used": 1203705, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.3715481758117676, | |
| "objective/train/theoretical_loss": 3.3919325628455534, | |
| "objective/train/tokens_used": 2314220000, | |
| "theoretical_loss": 3.3919325628455534, | |
| "tokens_seen": 2293760000 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 9.143890675241157e-05, | |
| "loss": 2.3942, | |
| "theoretical_loss": 3.3919325628455534, | |
| "tokens_seen": 2293760000 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 9.08478343105731e-05, | |
| "loss": 2.4027, | |
| "theoretical_loss": 3.3915397058949233, | |
| "tokens_seen": 2297036800 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 9.025676186873463e-05, | |
| "loss": 2.4028, | |
| "theoretical_loss": 3.3911475656323904, | |
| "tokens_seen": 2300313600 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 8.966568942689616e-05, | |
| "loss": 2.4401, | |
| "theoretical_loss": 3.3907561397323693, | |
| "tokens_seen": 2303590400 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 8.907461698505768e-05, | |
| "loss": 2.4061, | |
| "theoretical_loss": 3.3903654258801152, | |
| "tokens_seen": 2306867200 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 8.848354454321922e-05, | |
| "loss": 2.4496, | |
| "theoretical_loss": 3.389975421771658, | |
| "tokens_seen": 2310144000 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 8.789247210138075e-05, | |
| "loss": 2.4246, | |
| "theoretical_loss": 3.389586125113736, | |
| "tokens_seen": 2313420800 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 8.730139965954227e-05, | |
| "loss": 2.4304, | |
| "theoretical_loss": 3.389197533623734, | |
| "tokens_seen": 2316697600 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 8.671032721770381e-05, | |
| "loss": 2.4394, | |
| "theoretical_loss": 3.3888096450296152, | |
| "tokens_seen": 2319974400 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 8.611925477586534e-05, | |
| "loss": 2.435, | |
| "theoretical_loss": 3.3884224570698613, | |
| "tokens_seen": 2323251200 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 8.552818233402686e-05, | |
| "loss": 2.3935, | |
| "theoretical_loss": 3.388035967493405, | |
| "tokens_seen": 2326528000 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 8.49371098921884e-05, | |
| "loss": 2.4022, | |
| "theoretical_loss": 3.387650174059569, | |
| "tokens_seen": 2329804800 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 8.43460374503499e-05, | |
| "loss": 2.4269, | |
| "theoretical_loss": 3.3872650745380044, | |
| "tokens_seen": 2333081600 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 8.375496500851144e-05, | |
| "loss": 2.4196, | |
| "theoretical_loss": 3.3868806667086275, | |
| "tokens_seen": 2336358400 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 8.316389256667297e-05, | |
| "loss": 2.4344, | |
| "theoretical_loss": 3.3864969483615575, | |
| "tokens_seen": 2339635200 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 8.25728201248345e-05, | |
| "loss": 2.4353, | |
| "theoretical_loss": 3.3861139172970582, | |
| "tokens_seen": 2342912000 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 8.198174768299603e-05, | |
| "loss": 2.4425, | |
| "theoretical_loss": 3.385731571325474, | |
| "tokens_seen": 2346188800 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 8.139067524115756e-05, | |
| "loss": 2.4314, | |
| "theoretical_loss": 3.385349908267171, | |
| "tokens_seen": 2349465600 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 8.079960279931908e-05, | |
| "loss": 2.417, | |
| "theoretical_loss": 3.3849689259524784, | |
| "tokens_seen": 2352742400 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 8.020853035748062e-05, | |
| "loss": 2.4134, | |
| "theoretical_loss": 3.3845886222216275, | |
| "tokens_seen": 2356019200 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 7.961745791564215e-05, | |
| "loss": 2.4248, | |
| "theoretical_loss": 3.384208994924694, | |
| "tokens_seen": 2359296000 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 7.902638547380367e-05, | |
| "loss": 2.4325, | |
| "theoretical_loss": 3.3838300419215384, | |
| "tokens_seen": 2362572800 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 7.843531303196521e-05, | |
| "loss": 2.4346, | |
| "theoretical_loss": 3.383451761081749, | |
| "tokens_seen": 2365849600 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "learning_rate": 7.784424059012673e-05, | |
| "loss": 2.4447, | |
| "theoretical_loss": 3.383074150284584, | |
| "tokens_seen": 2369126400 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "learning_rate": 7.725316814828825e-05, | |
| "loss": 2.4185, | |
| "theoretical_loss": 3.3826972074189134, | |
| "tokens_seen": 2372403200 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "learning_rate": 7.666209570644978e-05, | |
| "loss": 2.4274, | |
| "theoretical_loss": 3.3823209303831634, | |
| "tokens_seen": 2375680000 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "learning_rate": 7.60710232646113e-05, | |
| "loss": 2.47, | |
| "theoretical_loss": 3.3819453170852602, | |
| "tokens_seen": 2378956800 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "learning_rate": 7.547995082277284e-05, | |
| "loss": 2.5363, | |
| "theoretical_loss": 3.381570365442573, | |
| "tokens_seen": 2382233600 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "learning_rate": 7.488887838093437e-05, | |
| "loss": 2.607, | |
| "theoretical_loss": 3.3811960733818585, | |
| "tokens_seen": 2385510400 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "learning_rate": 7.42978059390959e-05, | |
| "loss": 2.644, | |
| "theoretical_loss": 3.380822438839207, | |
| "tokens_seen": 2388787200 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "learning_rate": 7.370673349725743e-05, | |
| "loss": 2.6663, | |
| "theoretical_loss": 3.380449459759986, | |
| "tokens_seen": 2392064000 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 7.311566105541895e-05, | |
| "loss": 2.6241, | |
| "theoretical_loss": 3.380077134098787, | |
| "tokens_seen": 2395340800 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 7.252458861358048e-05, | |
| "loss": 2.606, | |
| "theoretical_loss": 3.3797054598193723, | |
| "tokens_seen": 2398617600 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 7.193351617174202e-05, | |
| "loss": 2.5699, | |
| "theoretical_loss": 3.379334434894619, | |
| "tokens_seen": 2401894400 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 7.134244372990354e-05, | |
| "loss": 2.5409, | |
| "theoretical_loss": 3.378964057306467, | |
| "tokens_seen": 2405171200 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 7.075137128806507e-05, | |
| "loss": 2.5161, | |
| "theoretical_loss": 3.378594325045867, | |
| "tokens_seen": 2408448000 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 7.01602988462266e-05, | |
| "loss": 2.4692, | |
| "theoretical_loss": 3.378225236112729, | |
| "tokens_seen": 2411724800 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 6.956922640438811e-05, | |
| "loss": 2.5072, | |
| "theoretical_loss": 3.377856788515865, | |
| "tokens_seen": 2415001600 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 6.897815396254965e-05, | |
| "loss": 2.4695, | |
| "theoretical_loss": 3.3774889802729446, | |
| "tokens_seen": 2418278400 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 6.838708152071118e-05, | |
| "loss": 2.4362, | |
| "theoretical_loss": 3.377121809410438, | |
| "tokens_seen": 2421555200 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "learning_rate": 6.77960090788727e-05, | |
| "loss": 2.4549, | |
| "theoretical_loss": 3.3767552739635684, | |
| "tokens_seen": 2424832000 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "learning_rate": 6.720493663703424e-05, | |
| "loss": 2.4081, | |
| "theoretical_loss": 3.37638937197626, | |
| "tokens_seen": 2428108800 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "learning_rate": 6.661386419519576e-05, | |
| "loss": 2.3973, | |
| "theoretical_loss": 3.3760241015010872, | |
| "tokens_seen": 2431385600 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "learning_rate": 6.602279175335729e-05, | |
| "loss": 2.41, | |
| "theoretical_loss": 3.375659460599226, | |
| "tokens_seen": 2434662400 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "learning_rate": 6.543171931151883e-05, | |
| "loss": 2.4287, | |
| "theoretical_loss": 3.375295447340404, | |
| "tokens_seen": 2437939200 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "learning_rate": 6.484064686968035e-05, | |
| "loss": 2.443, | |
| "theoretical_loss": 3.3749320598028527, | |
| "tokens_seen": 2441216000 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "learning_rate": 6.424957442784188e-05, | |
| "loss": 2.4259, | |
| "theoretical_loss": 3.3745692960732545, | |
| "tokens_seen": 2444492800 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "learning_rate": 6.365850198600342e-05, | |
| "loss": 2.411, | |
| "theoretical_loss": 3.374207154246699, | |
| "tokens_seen": 2447769600 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 6.306742954416494e-05, | |
| "loss": 2.3904, | |
| "theoretical_loss": 3.373845632426633, | |
| "tokens_seen": 2451046400 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 6.247635710232646e-05, | |
| "loss": 2.4054, | |
| "theoretical_loss": 3.373484728724812, | |
| "tokens_seen": 2454323200 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "objective/train/docs_used": 1284127, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.444608688354492, | |
| "objective/train/theoretical_loss": 3.3731244412612527, | |
| "objective/train/tokens_used": 2478060000, | |
| "theoretical_loss": 3.3731244412612527, | |
| "tokens_seen": 2457600000 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 6.188528466048799e-05, | |
| "loss": 2.4271, | |
| "theoretical_loss": 3.3731244412612527, | |
| "tokens_seen": 2457600000 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 6.129421221864951e-05, | |
| "loss": 2.4081, | |
| "theoretical_loss": 3.3727647681641884, | |
| "tokens_seen": 2460876800 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 6.070313977681105e-05, | |
| "loss": 2.4155, | |
| "theoretical_loss": 3.3724057075700182, | |
| "tokens_seen": 2464153600 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 6.0112067334972575e-05, | |
| "loss": 2.433, | |
| "theoretical_loss": 3.372047257623265, | |
| "tokens_seen": 2467430400 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 5.95209948931341e-05, | |
| "loss": 2.4307, | |
| "theoretical_loss": 3.3716894164765256, | |
| "tokens_seen": 2470707200 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 5.892992245129563e-05, | |
| "loss": 2.3821, | |
| "theoretical_loss": 3.371332182290427, | |
| "tokens_seen": 2473984000 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 5.8338850009457165e-05, | |
| "loss": 2.445, | |
| "theoretical_loss": 3.37097555323358, | |
| "tokens_seen": 2477260800 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "learning_rate": 5.774777756761869e-05, | |
| "loss": 2.4663, | |
| "theoretical_loss": 3.370619527482535, | |
| "tokens_seen": 2480537600 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "learning_rate": 5.715670512578021e-05, | |
| "loss": 2.5403, | |
| "theoretical_loss": 3.370264103221736, | |
| "tokens_seen": 2483814400 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "learning_rate": 5.656563268394174e-05, | |
| "loss": 2.6042, | |
| "theoretical_loss": 3.369909278643477, | |
| "tokens_seen": 2487091200 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "learning_rate": 5.5974560242103275e-05, | |
| "loss": 2.5505, | |
| "theoretical_loss": 3.369555051947857, | |
| "tokens_seen": 2490368000 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "learning_rate": 5.53834878002648e-05, | |
| "loss": 2.5479, | |
| "theoretical_loss": 3.369201421342737, | |
| "tokens_seen": 2493644800 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "learning_rate": 5.479241535842633e-05, | |
| "loss": 2.5129, | |
| "theoretical_loss": 3.3688483850436945, | |
| "tokens_seen": 2496921600 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "learning_rate": 5.420134291658786e-05, | |
| "loss": 2.4902, | |
| "theoretical_loss": 3.368495941273984, | |
| "tokens_seen": 2500198400 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "learning_rate": 5.3610270474749385e-05, | |
| "loss": 2.4599, | |
| "theoretical_loss": 3.3681440882644886, | |
| "tokens_seen": 2503475200 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 5.301919803291091e-05, | |
| "loss": 2.4707, | |
| "theoretical_loss": 3.3677928242536828, | |
| "tokens_seen": 2506752000 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 5.243994703990921e-05, | |
| "loss": 2.4956, | |
| "theoretical_loss": 3.3674421474875853, | |
| "tokens_seen": 2510028800 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 5.1848874598070746e-05, | |
| "loss": 2.471, | |
| "theoretical_loss": 3.367092056219721, | |
| "tokens_seen": 2513305600 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 5.125780215623227e-05, | |
| "loss": 2.4052, | |
| "theoretical_loss": 3.3667425487110756, | |
| "tokens_seen": 2516582400 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 5.0666729714393795e-05, | |
| "loss": 2.4147, | |
| "theoretical_loss": 3.366393623230057, | |
| "tokens_seen": 2519859200 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 5.007565727255532e-05, | |
| "loss": 2.4369, | |
| "theoretical_loss": 3.366045278052451, | |
| "tokens_seen": 2523136000 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 4.9484584830716857e-05, | |
| "loss": 2.4195, | |
| "theoretical_loss": 3.365697511461384, | |
| "tokens_seen": 2526412800 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 4.8893512388878384e-05, | |
| "loss": 2.474, | |
| "theoretical_loss": 3.365350321747279, | |
| "tokens_seen": 2529689600 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 4.830243994703991e-05, | |
| "loss": 2.4249, | |
| "theoretical_loss": 3.365003707207816, | |
| "tokens_seen": 2532966400 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "learning_rate": 4.771136750520144e-05, | |
| "loss": 2.461, | |
| "theoretical_loss": 3.3646576661478944, | |
| "tokens_seen": 2536243200 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "learning_rate": 4.712029506336297e-05, | |
| "loss": 2.4328, | |
| "theoretical_loss": 3.3643121968795895, | |
| "tokens_seen": 2539520000 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "learning_rate": 4.6529222621524494e-05, | |
| "loss": 2.4464, | |
| "theoretical_loss": 3.3639672977221142, | |
| "tokens_seen": 2542796800 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "learning_rate": 4.593815017968602e-05, | |
| "loss": 2.4926, | |
| "theoretical_loss": 3.3636229670017808, | |
| "tokens_seen": 2546073600 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "learning_rate": 4.5347077737847556e-05, | |
| "loss": 2.4872, | |
| "theoretical_loss": 3.3632792030519605, | |
| "tokens_seen": 2549350400 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "learning_rate": 4.4756005296009084e-05, | |
| "loss": 2.4776, | |
| "theoretical_loss": 3.362936004213045, | |
| "tokens_seen": 2552627200 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "learning_rate": 4.4164932854170605e-05, | |
| "loss": 2.4603, | |
| "theoretical_loss": 3.362593368832409, | |
| "tokens_seen": 2555904000 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "learning_rate": 4.357386041233213e-05, | |
| "loss": 2.4604, | |
| "theoretical_loss": 3.36225129526437, | |
| "tokens_seen": 2559180800 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 4.2982787970493667e-05, | |
| "loss": 2.4429, | |
| "theoretical_loss": 3.3619097818701515, | |
| "tokens_seen": 2562457600 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 4.2391715528655194e-05, | |
| "loss": 2.4594, | |
| "theoretical_loss": 3.3615688270178454, | |
| "tokens_seen": 2565734400 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 4.180064308681672e-05, | |
| "loss": 2.4138, | |
| "theoretical_loss": 3.3612284290823737, | |
| "tokens_seen": 2569011200 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 4.120957064497825e-05, | |
| "loss": 2.4357, | |
| "theoretical_loss": 3.3608885864454523, | |
| "tokens_seen": 2572288000 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 4.061849820313978e-05, | |
| "loss": 2.4796, | |
| "theoretical_loss": 3.360549297495553, | |
| "tokens_seen": 2575564800 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 4.0027425761301304e-05, | |
| "loss": 2.4611, | |
| "theoretical_loss": 3.3602105606278663, | |
| "tokens_seen": 2578841600 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 3.943635331946283e-05, | |
| "loss": 2.4559, | |
| "theoretical_loss": 3.359872374244266, | |
| "tokens_seen": 2582118400 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 3.8845280877624366e-05, | |
| "loss": 2.4486, | |
| "theoretical_loss": 3.3595347367532744, | |
| "tokens_seen": 2585395200 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 3.8254208435785894e-05, | |
| "loss": 2.4564, | |
| "theoretical_loss": 3.359197646570022, | |
| "tokens_seen": 2588672000 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 3.766313599394742e-05, | |
| "loss": 2.4296, | |
| "theoretical_loss": 3.3588611021162156, | |
| "tokens_seen": 2591948800 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 3.707206355210894e-05, | |
| "loss": 2.4508, | |
| "theoretical_loss": 3.3585251018201006, | |
| "tokens_seen": 2595225600 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 3.6480991110270477e-05, | |
| "loss": 2.4329, | |
| "theoretical_loss": 3.358189644116427, | |
| "tokens_seen": 2598502400 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 3.5889918668432004e-05, | |
| "loss": 2.418, | |
| "theoretical_loss": 3.3578547274464134, | |
| "tokens_seen": 2601779200 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 3.529884622659353e-05, | |
| "loss": 2.3832, | |
| "theoretical_loss": 3.357520350257713, | |
| "tokens_seen": 2605056000 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 3.470777378475506e-05, | |
| "loss": 2.4153, | |
| "theoretical_loss": 3.357186511004378, | |
| "tokens_seen": 2608332800 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 3.4116701342916594e-05, | |
| "loss": 2.4282, | |
| "theoretical_loss": 3.356853208146826, | |
| "tokens_seen": 2611609600 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 3.3525628901078114e-05, | |
| "loss": 2.416, | |
| "theoretical_loss": 3.356520440151805, | |
| "tokens_seen": 2614886400 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 3.293455645923964e-05, | |
| "loss": 2.4274, | |
| "theoretical_loss": 3.3561882054923613, | |
| "tokens_seen": 2618163200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.5785434453448561, | |
| "debugging/distinct-1-grams": 0.7696164737891201, | |
| "debugging/distinct-2-grams": 0.9658942396559373, | |
| "debugging/entropy-1-grams": 5.881700125048937, | |
| "debugging/entropy-2-grams": 6.752966734110217, | |
| "debugging/length": 605.5833333333334, | |
| "debugging/num_segments": 12, | |
| "epoch": 0.94, | |
| "objective/train/docs_used": 1365290, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.506117820739746, | |
| "objective/train/theoretical_loss": 3.3558565026478027, | |
| "objective/train/tokens_used": 2641900000, | |
| "theoretical_loss": 3.3558565026478027, | |
| "tokens_seen": 2621440000 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 3.234348401740117e-05, | |
| "loss": 2.4127, | |
| "theoretical_loss": 3.3558565026478027, | |
| "tokens_seen": 2621440000 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 3.1752411575562704e-05, | |
| "loss": 2.4226, | |
| "theoretical_loss": 3.3555253301036685, | |
| "tokens_seen": 2624716800 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 3.116133913372423e-05, | |
| "loss": 2.4392, | |
| "theoretical_loss": 3.3551946863516937, | |
| "tokens_seen": 2627993600 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 3.057026669188576e-05, | |
| "loss": 2.4011, | |
| "theoretical_loss": 3.3548645698897777, | |
| "tokens_seen": 2631270400 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 2.9979194250047287e-05, | |
| "loss": 2.3666, | |
| "theoretical_loss": 3.3545349792219494, | |
| "tokens_seen": 2634547200 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 2.9388121808208818e-05, | |
| "loss": 2.4027, | |
| "theoretical_loss": 3.354205912858337, | |
| "tokens_seen": 2637824000 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 2.8797049366370342e-05, | |
| "loss": 2.3829, | |
| "theoretical_loss": 3.353877369315133, | |
| "tokens_seen": 2641100800 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 2.8205976924531873e-05, | |
| "loss": 2.4084, | |
| "theoretical_loss": 3.353549347114565, | |
| "tokens_seen": 2644377600 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 2.76149044826934e-05, | |
| "loss": 2.4441, | |
| "theoretical_loss": 3.3532218447848616, | |
| "tokens_seen": 2647654400 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 2.7023832040854928e-05, | |
| "loss": 2.4139, | |
| "theoretical_loss": 3.352894860860221, | |
| "tokens_seen": 2650931200 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 2.6444581047853227e-05, | |
| "loss": 2.397, | |
| "theoretical_loss": 3.352568393880779, | |
| "tokens_seen": 2654208000 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 2.5853508606014754e-05, | |
| "loss": 2.4141, | |
| "theoretical_loss": 3.3522424423925794, | |
| "tokens_seen": 2657484800 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 2.5262436164176282e-05, | |
| "loss": 2.4163, | |
| "theoretical_loss": 3.3519170049475413, | |
| "tokens_seen": 2660761600 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 2.467136372233781e-05, | |
| "loss": 2.4343, | |
| "theoretical_loss": 3.351592080103429, | |
| "tokens_seen": 2664038400 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 2.408029128049934e-05, | |
| "loss": 2.398, | |
| "theoretical_loss": 3.3512676664238192, | |
| "tokens_seen": 2667315200 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 2.3489218838660865e-05, | |
| "loss": 2.3978, | |
| "theoretical_loss": 3.350943762478075, | |
| "tokens_seen": 2670592000 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 2.2898146396822396e-05, | |
| "loss": 2.4306, | |
| "theoretical_loss": 3.3506203668413095, | |
| "tokens_seen": 2673868800 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 2.2307073954983923e-05, | |
| "loss": 2.4148, | |
| "theoretical_loss": 3.3502974780943613, | |
| "tokens_seen": 2677145600 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 2.171600151314545e-05, | |
| "loss": 2.4012, | |
| "theoretical_loss": 3.34997509482376, | |
| "tokens_seen": 2680422400 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 2.112492907130698e-05, | |
| "loss": 2.3919, | |
| "theoretical_loss": 3.3496532156217005, | |
| "tokens_seen": 2683699200 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 2.053385662946851e-05, | |
| "loss": 2.3497, | |
| "theoretical_loss": 3.34933183908601, | |
| "tokens_seen": 2686976000 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 1.9942784187630033e-05, | |
| "loss": 2.3412, | |
| "theoretical_loss": 3.3490109638201195, | |
| "tokens_seen": 2690252800 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 1.9351711745791564e-05, | |
| "loss": 2.3734, | |
| "theoretical_loss": 3.348690588433038, | |
| "tokens_seen": 2693529600 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 1.8760639303953092e-05, | |
| "loss": 2.3456, | |
| "theoretical_loss": 3.3483707115393164, | |
| "tokens_seen": 2696806400 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 1.816956686211462e-05, | |
| "loss": 2.3727, | |
| "theoretical_loss": 3.3480513317590264, | |
| "tokens_seen": 2700083200 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 1.757849442027615e-05, | |
| "loss": 2.3929, | |
| "theoretical_loss": 3.347732447717727, | |
| "tokens_seen": 2703360000 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 1.6987421978437678e-05, | |
| "loss": 2.4225, | |
| "theoretical_loss": 3.347414058046437, | |
| "tokens_seen": 2706636800 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 1.639634953659921e-05, | |
| "loss": 2.4244, | |
| "theoretical_loss": 3.3470961613816086, | |
| "tokens_seen": 2709913600 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 1.5805277094760733e-05, | |
| "loss": 2.3979, | |
| "theoretical_loss": 3.3467787563650955, | |
| "tokens_seen": 2713190400 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 1.5214204652922262e-05, | |
| "loss": 2.3819, | |
| "theoretical_loss": 3.34646184164413, | |
| "tokens_seen": 2716467200 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 1.462313221108379e-05, | |
| "loss": 2.386, | |
| "theoretical_loss": 3.346145415871291, | |
| "tokens_seen": 2719744000 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 1.4032059769245318e-05, | |
| "loss": 2.3906, | |
| "theoretical_loss": 3.345829477704478, | |
| "tokens_seen": 2723020800 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 1.3440987327406847e-05, | |
| "loss": 2.2801, | |
| "theoretical_loss": 3.345514025806884, | |
| "tokens_seen": 2726297600 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 1.2849914885568376e-05, | |
| "loss": 2.3417, | |
| "theoretical_loss": 3.3451990588469687, | |
| "tokens_seen": 2729574400 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 1.2258842443729904e-05, | |
| "loss": 2.353, | |
| "theoretical_loss": 3.3448845754984307, | |
| "tokens_seen": 2732851200 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 1.1667770001891433e-05, | |
| "loss": 2.3525, | |
| "theoretical_loss": 3.344570574440179, | |
| "tokens_seen": 2736128000 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 1.107669756005296e-05, | |
| "loss": 2.3459, | |
| "theoretical_loss": 3.3442570543563104, | |
| "tokens_seen": 2739404800 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 1.0485625118214488e-05, | |
| "loss": 2.3562, | |
| "theoretical_loss": 3.34394401393608, | |
| "tokens_seen": 2742681600 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 9.894552676376017e-06, | |
| "loss": 2.3324, | |
| "theoretical_loss": 3.343631451873874, | |
| "tokens_seen": 2745958400 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 9.303480234537545e-06, | |
| "loss": 2.3728, | |
| "theoretical_loss": 3.343319366869187, | |
| "tokens_seen": 2749235200 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 8.712407792699074e-06, | |
| "loss": 2.3421, | |
| "theoretical_loss": 3.343007757626593, | |
| "tokens_seen": 2752512000 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 8.121335350860602e-06, | |
| "loss": 2.3363, | |
| "theoretical_loss": 3.34269662285572, | |
| "tokens_seen": 2755788800 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 7.530262909022129e-06, | |
| "loss": 2.3763, | |
| "theoretical_loss": 3.342385961271226, | |
| "tokens_seen": 2759065600 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 6.939190467183658e-06, | |
| "loss": 2.3382, | |
| "theoretical_loss": 3.3420757715927705, | |
| "tokens_seen": 2762342400 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 6.348118025345187e-06, | |
| "loss": 2.3999, | |
| "theoretical_loss": 3.341766052544993, | |
| "tokens_seen": 2765619200 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 5.757045583506715e-06, | |
| "loss": 2.3634, | |
| "theoretical_loss": 3.341456802857484, | |
| "tokens_seen": 2768896000 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 5.165973141668243e-06, | |
| "loss": 2.3467, | |
| "theoretical_loss": 3.341148021264762, | |
| "tokens_seen": 2772172800 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 4.574900699829771e-06, | |
| "loss": 2.3736, | |
| "theoretical_loss": 3.3408397065062485, | |
| "tokens_seen": 2775449600 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 3.9838282579913e-06, | |
| "loss": 2.3877, | |
| "theoretical_loss": 3.3405318573262424, | |
| "tokens_seen": 2778726400 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 3.392755816152828e-06, | |
| "loss": 2.3986, | |
| "theoretical_loss": 3.3402244724738965, | |
| "tokens_seen": 2782003200 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "objective/train/docs_used": 1451834, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.562957286834717, | |
| "objective/train/theoretical_loss": 3.3399175507031913, | |
| "objective/train/tokens_used": 2805740000, | |
| "theoretical_loss": 3.3399175507031913, | |
| "tokens_seen": 2785280000 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 2.801683374314356e-06, | |
| "loss": 2.3794, | |
| "theoretical_loss": 3.3399175507031913, | |
| "tokens_seen": 2785280000 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "learning_rate": 2.210610932475884e-06, | |
| "loss": 2.3641, | |
| "theoretical_loss": 3.3396110907729133, | |
| "tokens_seen": 2788556800 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "learning_rate": 1.6195384906374124e-06, | |
| "loss": 2.3827, | |
| "theoretical_loss": 3.3393050914466285, | |
| "tokens_seen": 2791833600 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "learning_rate": 1.0284660487989408e-06, | |
| "loss": 2.3469, | |
| "theoretical_loss": 3.33899955149266, | |
| "tokens_seen": 2795110400 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "learning_rate": 4.373936069604691e-07, | |
| "loss": 2.3693, | |
| "theoretical_loss": 3.338694469684062, | |
| "tokens_seen": 2798387200 | |
| } | |
| ], | |
| "max_steps": 42724, | |
| "num_train_epochs": 9223372036854775807, | |
| "total_flos": 1.428920419221504e+18, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |