| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.0, | |
| "global_step": 42724, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 1.1682242990654204e-06, | |
| "loss": 10.8138, | |
| "theoretical_loss": 20.81281176760504, | |
| "tokens_seen": 65536 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 5.841121495327103e-05, | |
| "loss": 9.0175, | |
| "theoretical_loss": 8.563476630668958, | |
| "tokens_seen": 3276800 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00011682242990654206, | |
| "loss": 6.8036, | |
| "theoretical_loss": 7.477752684105921, | |
| "tokens_seen": 6553600 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00017523364485981307, | |
| "loss": 5.897, | |
| "theoretical_loss": 6.9337484549527915, | |
| "tokens_seen": 9830400 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00023364485981308412, | |
| "loss": 5.5149, | |
| "theoretical_loss": 6.5835601944843045, | |
| "tokens_seen": 13107200 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00029205607476635517, | |
| "loss": 5.254, | |
| "theoretical_loss": 6.3307075311739744, | |
| "tokens_seen": 16384000 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00035046728971962614, | |
| "loss": 5.0768, | |
| "theoretical_loss": 6.135523197998216, | |
| "tokens_seen": 19660800 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004088785046728972, | |
| "loss": 4.9043, | |
| "theoretical_loss": 5.978095549927499, | |
| "tokens_seen": 22937600 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00046728971962616824, | |
| "loss": 4.8267, | |
| "theoretical_loss": 5.847111292323815, | |
| "tokens_seen": 26214400 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000499739928125591, | |
| "loss": 4.7172, | |
| "theoretical_loss": 5.735570781940016, | |
| "tokens_seen": 29491200 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004991488556837526, | |
| "loss": 4.5747, | |
| "theoretical_loss": 5.638864110129244, | |
| "tokens_seen": 32768000 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004985577832419141, | |
| "loss": 4.44, | |
| "theoretical_loss": 5.553806347902798, | |
| "tokens_seen": 36044800 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004979667108000757, | |
| "loss": 4.3527, | |
| "theoretical_loss": 5.478112046614329, | |
| "tokens_seen": 39321600 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004973756383582371, | |
| "loss": 4.2825, | |
| "theoretical_loss": 5.410089925637252, | |
| "tokens_seen": 42598400 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004967845659163987, | |
| "loss": 4.1954, | |
| "theoretical_loss": 5.348456049793725, | |
| "tokens_seen": 45875200 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004961934934745603, | |
| "loss": 4.1451, | |
| "theoretical_loss": 5.292214532995457, | |
| "tokens_seen": 49152000 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004956024210327218, | |
| "loss": 4.0511, | |
| "theoretical_loss": 5.240578591827869, | |
| "tokens_seen": 52428800 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004950113485908833, | |
| "loss": 3.9805, | |
| "theoretical_loss": 5.192916690583679, | |
| "tokens_seen": 55705600 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004944202761490448, | |
| "loss": 3.9453, | |
| "theoretical_loss": 5.148714829414331, | |
| "tokens_seen": 58982400 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004938292037072064, | |
| "loss": 3.8613, | |
| "theoretical_loss": 5.107549528462992, | |
| "tokens_seen": 62259200 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004932381312653678, | |
| "loss": 3.7269, | |
| "theoretical_loss": 5.069068083201136, | |
| "tokens_seen": 65536000 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004926470588235294, | |
| "loss": 3.6728, | |
| "theoretical_loss": 5.032973875895897, | |
| "tokens_seen": 68812800 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.000492055986381691, | |
| "loss": 3.5894, | |
| "theoretical_loss": 4.999015274282555, | |
| "tokens_seen": 72089600 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004914649139398525, | |
| "loss": 3.5112, | |
| "theoretical_loss": 4.966977121409853, | |
| "tokens_seen": 75366400 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.000490873841498014, | |
| "loss": 3.4986, | |
| "theoretical_loss": 4.936674127683027, | |
| "tokens_seen": 78643200 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004902827690561755, | |
| "loss": 3.4625, | |
| "theoretical_loss": 4.907945679887972, | |
| "tokens_seen": 81920000 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004896916966143371, | |
| "loss": 3.4264, | |
| "theoretical_loss": 4.8806517198708175, | |
| "tokens_seen": 85196800 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004891006241724985, | |
| "loss": 3.4137, | |
| "theoretical_loss": 4.85466944053967, | |
| "tokens_seen": 88473600 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004885095517306601, | |
| "loss": 3.3837, | |
| "theoretical_loss": 4.829890613366704, | |
| "tokens_seen": 91750400 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004879184792888217, | |
| "loss": 3.3608, | |
| "theoretical_loss": 4.806219408835812, | |
| "tokens_seen": 95027200 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004873274068469832, | |
| "loss": 3.3146, | |
| "theoretical_loss": 4.783570605334148, | |
| "tokens_seen": 98304000 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.00048673633440514467, | |
| "loss": 3.2702, | |
| "theoretical_loss": 4.761868106830299, | |
| "tokens_seen": 101580800 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004861452619633062, | |
| "loss": 3.3171, | |
| "theoretical_loss": 4.741043708020364, | |
| "tokens_seen": 104857600 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004855541895214677, | |
| "loss": 3.2743, | |
| "theoretical_loss": 4.721036059306941, | |
| "tokens_seen": 108134400 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004849631170796293, | |
| "loss": 3.2176, | |
| "theoretical_loss": 4.701789794289756, | |
| "tokens_seen": 111411200 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004843720446377908, | |
| "loss": 3.2875, | |
| "theoretical_loss": 4.68325479029382, | |
| "tokens_seen": 114688000 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.00048378097219595233, | |
| "loss": 3.2317, | |
| "theoretical_loss": 4.6653855384841725, | |
| "tokens_seen": 117964800 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.00048318989975411385, | |
| "loss": 3.2294, | |
| "theoretical_loss": 4.6481406047776295, | |
| "tokens_seen": 121241600 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.00048259882731227537, | |
| "loss": 3.2435, | |
| "theoretical_loss": 4.631482166397534, | |
| "tokens_seen": 124518400 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0004820077548704369, | |
| "loss": 3.1946, | |
| "theoretical_loss": 4.615375611773608, | |
| "tokens_seen": 127795200 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00048141668242859847, | |
| "loss": 3.2154, | |
| "theoretical_loss": 4.5997891937483955, | |
| "tokens_seen": 131072000 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00048082560998676, | |
| "loss": 3.1983, | |
| "theoretical_loss": 4.584693727850565, | |
| "tokens_seen": 134348800 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0004802345375449215, | |
| "loss": 3.2056, | |
| "theoretical_loss": 4.570062328836407, | |
| "tokens_seen": 137625600 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00047964346510308303, | |
| "loss": 3.1738, | |
| "theoretical_loss": 4.5558701798619285, | |
| "tokens_seen": 140902400 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00047905239266124455, | |
| "loss": 3.1418, | |
| "theoretical_loss": 4.542094329588689, | |
| "tokens_seen": 144179200 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00047846132021940607, | |
| "loss": 3.1549, | |
| "theoretical_loss": 4.528713513292708, | |
| "tokens_seen": 147456000 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0004778702477775676, | |
| "loss": 3.1277, | |
| "theoretical_loss": 4.515707994672887, | |
| "tokens_seen": 150732800 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047727917533572917, | |
| "loss": 3.1174, | |
| "theoretical_loss": 4.503059425571229, | |
| "tokens_seen": 154009600 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0004766881028938907, | |
| "loss": 3.1369, | |
| "theoretical_loss": 4.490750721243157, | |
| "tokens_seen": 157286400 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0004760970304520522, | |
| "loss": 3.0965, | |
| "theoretical_loss": 4.478765949169858, | |
| "tokens_seen": 160563200 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "objective/train/docs_used": 96671, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 3.070615530014038, | |
| "objective/train/theoretical_loss": 4.46709022969911, | |
| "objective/train/tokens_used": 184300000, | |
| "theoretical_loss": 4.46709022969911, | |
| "tokens_seen": 163840000 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047550595801021373, | |
| "loss": 3.1245, | |
| "theoretical_loss": 4.46709022969911, | |
| "tokens_seen": 163840000 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047491488556837525, | |
| "loss": 3.1083, | |
| "theoretical_loss": 4.455709647047437, | |
| "tokens_seen": 167116800 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047432381312653677, | |
| "loss": 3.0853, | |
| "theoretical_loss": 4.444611169403321, | |
| "tokens_seen": 170393600 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047373274068469835, | |
| "loss": 3.0679, | |
| "theoretical_loss": 4.4337825770455375, | |
| "tokens_seen": 173670400 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047314166824285987, | |
| "loss": 3.0801, | |
| "theoretical_loss": 4.423212397538051, | |
| "tokens_seen": 176947200 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0004725505958010214, | |
| "loss": 3.0736, | |
| "theoretical_loss": 4.4128898471880325, | |
| "tokens_seen": 180224000 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0004719595233591829, | |
| "loss": 3.0484, | |
| "theoretical_loss": 4.40280477805997, | |
| "tokens_seen": 183500800 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00047136845091734443, | |
| "loss": 3.0285, | |
| "theoretical_loss": 4.392947629929752, | |
| "tokens_seen": 186777600 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00047077737847550595, | |
| "loss": 3.034, | |
| "theoretical_loss": 4.383309386640423, | |
| "tokens_seen": 190054400 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0004701863060336675, | |
| "loss": 3.0079, | |
| "theoretical_loss": 4.373881536388167, | |
| "tokens_seen": 193331200 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00046959523359182905, | |
| "loss": 3.0063, | |
| "theoretical_loss": 4.364656035524595, | |
| "tokens_seen": 196608000 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00046900416114999057, | |
| "loss": 3.0029, | |
| "theoretical_loss": 4.355625275511174, | |
| "tokens_seen": 199884800 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0004684130887081521, | |
| "loss": 3.0104, | |
| "theoretical_loss": 4.346782052704563, | |
| "tokens_seen": 203161600 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0004678220162663136, | |
| "loss": 3.0287, | |
| "theoretical_loss": 4.338119540689052, | |
| "tokens_seen": 206438400 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00046723094382447513, | |
| "loss": 3.0217, | |
| "theoretical_loss": 4.329631264904703, | |
| "tokens_seen": 209715200 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.00046663987138263665, | |
| "loss": 2.9704, | |
| "theoretical_loss": 4.321311079348144, | |
| "tokens_seen": 212992000 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0004660487989407982, | |
| "loss": 2.9808, | |
| "theoretical_loss": 4.313153145147683, | |
| "tokens_seen": 216268800 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.00046545772649895975, | |
| "loss": 2.9749, | |
| "theoretical_loss": 4.305151910836119, | |
| "tokens_seen": 219545600 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.00046486665405712127, | |
| "loss": 3.0127, | |
| "theoretical_loss": 4.2973020941635784, | |
| "tokens_seen": 222822400 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0004642755816152828, | |
| "loss": 2.9855, | |
| "theoretical_loss": 4.28959866530949, | |
| "tokens_seen": 226099200 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0004636845091734443, | |
| "loss": 2.9321, | |
| "theoretical_loss": 4.282036831367506, | |
| "tokens_seen": 229376000 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.00046309343673160583, | |
| "loss": 2.93, | |
| "theoretical_loss": 4.274612021990189, | |
| "tokens_seen": 232652800 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0004625023642897674, | |
| "loss": 2.9449, | |
| "theoretical_loss": 4.267319876091788, | |
| "tokens_seen": 235929600 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004619112918479289, | |
| "loss": 2.9467, | |
| "theoretical_loss": 4.260156229517635, | |
| "tokens_seen": 239206400 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.00046132021940609044, | |
| "loss": 2.9644, | |
| "theoretical_loss": 4.253117103597704, | |
| "tokens_seen": 242483200 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.00046072914696425197, | |
| "loss": 3.0273, | |
| "theoretical_loss": 4.246198694509945, | |
| "tokens_seen": 245760000 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004601380745224135, | |
| "loss": 2.9966, | |
| "theoretical_loss": 4.239397363386152, | |
| "tokens_seen": 249036800 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.000459547002080575, | |
| "loss": 2.9736, | |
| "theoretical_loss": 4.232709627099522, | |
| "tokens_seen": 252313600 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004589559296387366, | |
| "loss": 2.9356, | |
| "theoretical_loss": 4.226132149678757, | |
| "tokens_seen": 255590400 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004583648571968981, | |
| "loss": 2.9407, | |
| "theoretical_loss": 4.219661734298666, | |
| "tokens_seen": 258867200 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004577737847550596, | |
| "loss": 2.933, | |
| "theoretical_loss": 4.213295315801815, | |
| "tokens_seen": 262144000 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004571827123132211, | |
| "loss": 2.9047, | |
| "theoretical_loss": 4.207029953709861, | |
| "tokens_seen": 265420800 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0004565916398713826, | |
| "loss": 2.8924, | |
| "theoretical_loss": 4.200862825686893, | |
| "tokens_seen": 268697600 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.00045600056742954413, | |
| "loss": 2.9114, | |
| "theoretical_loss": 4.19479122142044, | |
| "tokens_seen": 271974400 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0004554094949877057, | |
| "loss": 2.8889, | |
| "theoretical_loss": 4.188812536888775, | |
| "tokens_seen": 275251200 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.00045481842254586723, | |
| "loss": 2.9125, | |
| "theoretical_loss": 4.182924268985855, | |
| "tokens_seen": 278528000 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.00045422735010402875, | |
| "loss": 2.8775, | |
| "theoretical_loss": 4.177124010477671, | |
| "tokens_seen": 281804800 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.00045363627766219027, | |
| "loss": 2.8512, | |
| "theoretical_loss": 4.171409445265983, | |
| "tokens_seen": 285081600 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0004530570266691886, | |
| "loss": 3.0123, | |
| "theoretical_loss": 4.165778343937409, | |
| "tokens_seen": 288358400 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0004524659542273501, | |
| "loss": 2.8908, | |
| "theoretical_loss": 4.160228559577659, | |
| "tokens_seen": 291635200 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.00045187488178551164, | |
| "loss": 2.8512, | |
| "theoretical_loss": 4.15475802383233, | |
| "tokens_seen": 294912000 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0004512838093436732, | |
| "loss": 2.8741, | |
| "theoretical_loss": 4.149364743197177, | |
| "tokens_seen": 298188800 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.00045069273690183473, | |
| "loss": 2.8669, | |
| "theoretical_loss": 4.14404679552214, | |
| "tokens_seen": 301465600 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.00045010166445999625, | |
| "loss": 2.8488, | |
| "theoretical_loss": 4.138802326714632, | |
| "tokens_seen": 304742400 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0004495105920181578, | |
| "loss": 2.8781, | |
| "theoretical_loss": 4.133629547628726, | |
| "tokens_seen": 308019200 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0004489195195763193, | |
| "loss": 2.893, | |
| "theoretical_loss": 4.128526731127894, | |
| "tokens_seen": 311296000 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.00044832844713448076, | |
| "loss": 2.8765, | |
| "theoretical_loss": 4.123492209309923, | |
| "tokens_seen": 314572800 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.00044773737469264234, | |
| "loss": 2.8696, | |
| "theoretical_loss": 4.118524370883447, | |
| "tokens_seen": 317849600 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.00044714630225080386, | |
| "loss": 2.8335, | |
| "theoretical_loss": 4.113621658686355, | |
| "tokens_seen": 321126400 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0004465552298089654, | |
| "loss": 2.8533, | |
| "theoretical_loss": 4.108782567337039, | |
| "tokens_seen": 324403200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.6775872410093459, | |
| "debugging/distinct-1-grams": 0.776911879119778, | |
| "debugging/distinct-2-grams": 0.947649399368004, | |
| "debugging/entropy-1-grams": 6.145310820258306, | |
| "debugging/entropy-2-grams": 7.078256509076434, | |
| "debugging/length": 648.1666666666666, | |
| "debugging/num_segments": 18, | |
| "epoch": 0.12, | |
| "objective/train/docs_used": 182488, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.63395357131958, | |
| "objective/train/theoretical_loss": 4.104005641010112, | |
| "objective/train/tokens_used": 348140000, | |
| "theoretical_loss": 4.104005641010112, | |
| "tokens_seen": 327680000 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0004459641573671269, | |
| "loss": 2.8468, | |
| "theoretical_loss": 4.104005641010112, | |
| "tokens_seen": 327680000 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0004453730849252884, | |
| "loss": 2.8279, | |
| "theoretical_loss": 4.099289471328812, | |
| "tokens_seen": 330956800 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.00044478201248344994, | |
| "loss": 2.8103, | |
| "theoretical_loss": 4.094632695366921, | |
| "tokens_seen": 334233600 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0004441909400416115, | |
| "loss": 2.7814, | |
| "theoretical_loss": 4.090033993753448, | |
| "tokens_seen": 337510400 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.00044359986759977304, | |
| "loss": 2.7574, | |
| "theoretical_loss": 4.085492088873883, | |
| "tokens_seen": 340787200 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.00044300879515793456, | |
| "loss": 2.7824, | |
| "theoretical_loss": 4.081005743162224, | |
| "tokens_seen": 344064000 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0004424177227160961, | |
| "loss": 2.8428, | |
| "theoretical_loss": 4.076573757478361, | |
| "tokens_seen": 347340800 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0004418266502742576, | |
| "loss": 2.8106, | |
| "theoretical_loss": 4.072194969565807, | |
| "tokens_seen": 350617600 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0004412355778324191, | |
| "loss": 2.8311, | |
| "theoretical_loss": 4.067868252585089, | |
| "tokens_seen": 353894400 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00044064450539058064, | |
| "loss": 2.807, | |
| "theoretical_loss": 4.063592513718411, | |
| "tokens_seen": 357171200 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0004400534329487422, | |
| "loss": 2.8109, | |
| "theoretical_loss": 4.059366692841521, | |
| "tokens_seen": 360448000 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00043946236050690374, | |
| "loss": 2.8095, | |
| "theoretical_loss": 4.055189761258959, | |
| "tokens_seen": 363724800 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00043887128806506526, | |
| "loss": 2.7752, | |
| "theoretical_loss": 4.051060720499127, | |
| "tokens_seen": 367001600 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0004382802156232268, | |
| "loss": 2.7986, | |
| "theoretical_loss": 4.046978601165831, | |
| "tokens_seen": 370278400 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0004376891431813883, | |
| "loss": 2.7653, | |
| "theoretical_loss": 4.042942461843204, | |
| "tokens_seen": 373555200 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0004370980707395498, | |
| "loss": 2.7984, | |
| "theoretical_loss": 4.038951388051044, | |
| "tokens_seen": 376832000 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0004365069982977114, | |
| "loss": 2.8074, | |
| "theoretical_loss": 4.035004491247873, | |
| "tokens_seen": 380108800 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0004359159258558729, | |
| "loss": 2.7788, | |
| "theoretical_loss": 4.031100907879109, | |
| "tokens_seen": 383385600 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.00043532485341403444, | |
| "loss": 2.8255, | |
| "theoretical_loss": 4.02723979846797, | |
| "tokens_seen": 386662400 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.00043473378097219596, | |
| "loss": 2.7839, | |
| "theoretical_loss": 4.023420346746835, | |
| "tokens_seen": 389939200 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0004341427085303575, | |
| "loss": 2.7849, | |
| "theoretical_loss": 4.019641758826938, | |
| "tokens_seen": 393216000 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.000433551636088519, | |
| "loss": 2.7756, | |
| "theoretical_loss": 4.015903262404413, | |
| "tokens_seen": 396492800 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0004329605636466806, | |
| "loss": 2.7589, | |
| "theoretical_loss": 4.012204106000786, | |
| "tokens_seen": 399769600 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0004323694912048421, | |
| "loss": 2.7688, | |
| "theoretical_loss": 4.008543558236181, | |
| "tokens_seen": 403046400 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004317784187630036, | |
| "loss": 2.7891, | |
| "theoretical_loss": 4.004920907133565, | |
| "tokens_seen": 406323200 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.00043118734632116514, | |
| "loss": 2.7671, | |
| "theoretical_loss": 4.001335459452449, | |
| "tokens_seen": 409600000 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.00043059627387932666, | |
| "loss": 2.8043, | |
| "theoretical_loss": 3.997786540050617, | |
| "tokens_seen": 412876800 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004300052014374882, | |
| "loss": 2.7799, | |
| "theoretical_loss": 3.9942734912724456, | |
| "tokens_seen": 416153600 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004294141289956497, | |
| "loss": 2.724, | |
| "theoretical_loss": 3.9907956723625375, | |
| "tokens_seen": 419430400 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004288230565538113, | |
| "loss": 2.7849, | |
| "theoretical_loss": 3.9873524589034224, | |
| "tokens_seen": 422707200 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004282319841119728, | |
| "loss": 2.7838, | |
| "theoretical_loss": 3.9839432422761556, | |
| "tokens_seen": 425984000 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004276409116701343, | |
| "loss": 2.8161, | |
| "theoretical_loss": 3.980567429142721, | |
| "tokens_seen": 429260800 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.00042704983922829584, | |
| "loss": 2.7718, | |
| "theoretical_loss": 3.977224440949197, | |
| "tokens_seen": 432537600 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042645876678645736, | |
| "loss": 2.7692, | |
| "theoretical_loss": 3.9739137134486917, | |
| "tokens_seen": 435814400 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0004258676943446189, | |
| "loss": 2.8124, | |
| "theoretical_loss": 3.9706346962431396, | |
| "tokens_seen": 439091200 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042527662190278045, | |
| "loss": 2.8134, | |
| "theoretical_loss": 3.9673868523430564, | |
| "tokens_seen": 442368000 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042468554946094197, | |
| "loss": 2.8103, | |
| "theoretical_loss": 3.9641696577444376, | |
| "tokens_seen": 445644800 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0004240944770191035, | |
| "loss": 2.7859, | |
| "theoretical_loss": 3.9609826010220033, | |
| "tokens_seen": 448921600 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.000423503404577265, | |
| "loss": 2.8046, | |
| "theoretical_loss": 3.9578251829380506, | |
| "tokens_seen": 452198400 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042291233213542654, | |
| "loss": 2.7949, | |
| "theoretical_loss": 3.954696916066199, | |
| "tokens_seen": 455475200 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042232125969358806, | |
| "loss": 2.7974, | |
| "theoretical_loss": 3.9515973244293643, | |
| "tokens_seen": 458752000 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.00042173018725174963, | |
| "loss": 2.7685, | |
| "theoretical_loss": 3.948525943151326, | |
| "tokens_seen": 462028800 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.00042113911480991115, | |
| "loss": 2.7622, | |
| "theoretical_loss": 3.9454823181212815, | |
| "tokens_seen": 465305600 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.00042054804236807267, | |
| "loss": 2.7868, | |
| "theoretical_loss": 3.9424660056708167, | |
| "tokens_seen": 468582400 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0004199569699262342, | |
| "loss": 2.7821, | |
| "theoretical_loss": 3.939476572262754, | |
| "tokens_seen": 471859200 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.00041936589748439566, | |
| "loss": 2.7549, | |
| "theoretical_loss": 3.9365135941913563, | |
| "tokens_seen": 475136000 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0004187748250425572, | |
| "loss": 2.7653, | |
| "theoretical_loss": 3.9335766572934023, | |
| "tokens_seen": 478412800 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0004181837526007187, | |
| "loss": 2.7314, | |
| "theoretical_loss": 3.9306653566696603, | |
| "tokens_seen": 481689600 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0004175926801588803, | |
| "loss": 2.7065, | |
| "theoretical_loss": 3.927779296416332, | |
| "tokens_seen": 484966400 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0004170016077170418, | |
| "loss": 2.7375, | |
| "theoretical_loss": 3.924918089366024, | |
| "tokens_seen": 488243200 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "objective/train/docs_used": 267274, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.6940042972564697, | |
| "objective/train/theoretical_loss": 3.9220813568378707, | |
| "objective/train/tokens_used": 511980000, | |
| "theoretical_loss": 3.9220813568378707, | |
| "tokens_seen": 491520000 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0004164105352752033, | |
| "loss": 2.6869, | |
| "theoretical_loss": 3.9220813568378707, | |
| "tokens_seen": 491520000 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.00041581946283336484, | |
| "loss": 2.7129, | |
| "theoretical_loss": 3.9192687283964096, | |
| "tokens_seen": 494796800 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.00041522839039152636, | |
| "loss": 2.7344, | |
| "theoretical_loss": 3.9164798416188527, | |
| "tokens_seen": 498073600 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0004146373179496879, | |
| "loss": 2.6784, | |
| "theoretical_loss": 3.913714341870409, | |
| "tokens_seen": 501350400 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.00041404624550784946, | |
| "loss": 2.6854, | |
| "theoretical_loss": 3.9109718820873303, | |
| "tokens_seen": 504627200 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.000413455173066011, | |
| "loss": 2.694, | |
| "theoretical_loss": 3.9082521225673625, | |
| "tokens_seen": 507904000 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0004128641006241725, | |
| "loss": 2.6917, | |
| "theoretical_loss": 3.9055547307673075, | |
| "tokens_seen": 511180800 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.000412273028182334, | |
| "loss": 2.7214, | |
| "theoretical_loss": 3.9028793811074056, | |
| "tokens_seen": 514457600 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.00041168195574049554, | |
| "loss": 2.7651, | |
| "theoretical_loss": 3.900225754782274, | |
| "tokens_seen": 517734400 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.00041109088329865706, | |
| "loss": 2.7546, | |
| "theoretical_loss": 3.897593539578138, | |
| "tokens_seen": 521011200 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.00041049981085681863, | |
| "loss": 2.7185, | |
| "theoretical_loss": 3.8949824296961015, | |
| "tokens_seen": 524288000 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.00040990873841498016, | |
| "loss": 2.7248, | |
| "theoretical_loss": 3.8923921255812353, | |
| "tokens_seen": 527564800 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0004093176659731417, | |
| "loss": 2.7266, | |
| "theoretical_loss": 3.8898223337572393, | |
| "tokens_seen": 530841600 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0004087265935313032, | |
| "loss": 2.6847, | |
| "theoretical_loss": 3.88727276666648, | |
| "tokens_seen": 534118400 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0004081355210894647, | |
| "loss": 2.7031, | |
| "theoretical_loss": 3.884743142515184, | |
| "tokens_seen": 537395200 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.00040754444864762624, | |
| "loss": 2.6997, | |
| "theoretical_loss": 3.8822331851235985, | |
| "tokens_seen": 540672000 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0004069533762057878, | |
| "loss": 2.722, | |
| "theoretical_loss": 3.87974262378093, | |
| "tokens_seen": 543948800 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.00040636230376394933, | |
| "loss": 2.6597, | |
| "theoretical_loss": 3.877271193104873, | |
| "tokens_seen": 547225600 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.00040577123132211085, | |
| "loss": 2.6842, | |
| "theoretical_loss": 3.8748186329055736, | |
| "tokens_seen": 550502400 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004051801588802724, | |
| "loss": 2.6774, | |
| "theoretical_loss": 3.87238468805384, | |
| "tokens_seen": 553779200 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004045890864384339, | |
| "loss": 2.6673, | |
| "theoretical_loss": 3.8699691083534633, | |
| "tokens_seen": 557056000 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004039980139965954, | |
| "loss": 2.6858, | |
| "theoretical_loss": 3.8675716484174907, | |
| "tokens_seen": 560332800 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.00040340694155475694, | |
| "loss": 2.6698, | |
| "theoretical_loss": 3.8651920675482936, | |
| "tokens_seen": 563609600 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004028158691129185, | |
| "loss": 2.7021, | |
| "theoretical_loss": 3.862830129621318, | |
| "tokens_seen": 566886400 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.00040222479667108003, | |
| "loss": 2.7151, | |
| "theoretical_loss": 3.8604856029723575, | |
| "tokens_seen": 570163200 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.00040163372422924155, | |
| "loss": 2.6881, | |
| "theoretical_loss": 3.8581582602882447, | |
| "tokens_seen": 573440000 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0004010426517874031, | |
| "loss": 2.7237, | |
| "theoretical_loss": 3.8558478785008203, | |
| "tokens_seen": 576716800 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0004004515793455646, | |
| "loss": 2.7083, | |
| "theoretical_loss": 3.8535542386840778, | |
| "tokens_seen": 579993600 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0003998605069037261, | |
| "loss": 2.6988, | |
| "theoretical_loss": 3.8512771259543586, | |
| "tokens_seen": 583270400 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0003992694344618877, | |
| "loss": 2.7261, | |
| "theoretical_loss": 3.8490163293735082, | |
| "tokens_seen": 586547200 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0003986783620200492, | |
| "loss": 2.7215, | |
| "theoretical_loss": 3.8467716418548648, | |
| "tokens_seen": 589824000 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.00039808728957821073, | |
| "loss": 2.7196, | |
| "theoretical_loss": 3.844542860072007, | |
| "tokens_seen": 593100800 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.00039749621713637225, | |
| "loss": 2.736, | |
| "theoretical_loss": 3.8423297843701496, | |
| "tokens_seen": 596377600 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0003969051446945338, | |
| "loss": 2.7189, | |
| "theoretical_loss": 3.8401322186800995, | |
| "tokens_seen": 599654400 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0003963140722526953, | |
| "loss": 2.7546, | |
| "theoretical_loss": 3.83794997043469, | |
| "tokens_seen": 602931200 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.00039572299981085687, | |
| "loss": 2.7572, | |
| "theoretical_loss": 3.8357828504876004, | |
| "tokens_seen": 606208000 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0003951319273690184, | |
| "loss": 2.7173, | |
| "theoretical_loss": 3.833630673034487, | |
| "tokens_seen": 609484800 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0003945408549271799, | |
| "loss": 2.7306, | |
| "theoretical_loss": 3.831493255536345, | |
| "tokens_seen": 612761600 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.00039394978248534143, | |
| "loss": 2.7288, | |
| "theoretical_loss": 3.8293704186450253, | |
| "tokens_seen": 616038400 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.00039335871004350295, | |
| "loss": 2.723, | |
| "theoretical_loss": 3.827261986130839, | |
| "tokens_seen": 619315200 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0003927676376016645, | |
| "loss": 2.6673, | |
| "theoretical_loss": 3.825167784812175, | |
| "tokens_seen": 622592000 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.000392176565159826, | |
| "loss": 2.6865, | |
| "theoretical_loss": 3.823087644487069, | |
| "tokens_seen": 625868800 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.00039158549271798757, | |
| "loss": 2.6744, | |
| "theoretical_loss": 3.8210213978666565, | |
| "tokens_seen": 629145600 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0003909944202761491, | |
| "loss": 2.6884, | |
| "theoretical_loss": 3.8189688805104476, | |
| "tokens_seen": 632422400 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.00039040334783431056, | |
| "loss": 2.6735, | |
| "theoretical_loss": 3.816929930763374, | |
| "tokens_seen": 635699200 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0003898122753924721, | |
| "loss": 2.702, | |
| "theoretical_loss": 3.8149043896945347, | |
| "tokens_seen": 638976000 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0003892212029506336, | |
| "loss": 2.6783, | |
| "theoretical_loss": 3.812892101037601, | |
| "tokens_seen": 642252800 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0003886301305087951, | |
| "loss": 2.6662, | |
| "theoretical_loss": 3.81089291113282, | |
| "tokens_seen": 645529600 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0003880390580669567, | |
| "loss": 2.6817, | |
| "theoretical_loss": 3.8089066688705673, | |
| "tokens_seen": 648806400 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0003874479856251182, | |
| "loss": 2.6423, | |
| "theoretical_loss": 3.8069332256363992, | |
| "tokens_seen": 652083200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.4830888722420741, | |
| "debugging/distinct-1-grams": 0.8177040787129198, | |
| "debugging/distinct-2-grams": 0.968608367204819, | |
| "debugging/entropy-1-grams": 5.3590605074024396, | |
| "debugging/entropy-2-grams": 5.9214073904158475, | |
| "debugging/length": 564.1666666666666, | |
| "debugging/num_segments": 6, | |
| "epoch": 0.23, | |
| "objective/train/docs_used": 351925, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.8214328289031982, | |
| "objective/train/theoretical_loss": 3.80497243525756, | |
| "objective/train/tokens_used": 675820000, | |
| "theoretical_loss": 3.80497243525756, | |
| "tokens_seen": 655360000 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.00038685691318327974, | |
| "loss": 2.6739, | |
| "theoretical_loss": 3.80497243525756, | |
| "tokens_seen": 655360000 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.00038626584074144126, | |
| "loss": 2.6231, | |
| "theoretical_loss": 3.8030241539508958, | |
| "tokens_seen": 658636800 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003856747682996028, | |
| "loss": 2.6252, | |
| "theoretical_loss": 3.8010882402721324, | |
| "tokens_seen": 661913600 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003850836958577643, | |
| "loss": 2.6658, | |
| "theoretical_loss": 3.7991645550664757, | |
| "tokens_seen": 665190400 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003844926234159259, | |
| "loss": 2.7018, | |
| "theoretical_loss": 3.797252961420492, | |
| "tokens_seen": 668467200 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003839015509740874, | |
| "loss": 2.689, | |
| "theoretical_loss": 3.795353324615228, | |
| "tokens_seen": 671744000 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003833104785322489, | |
| "loss": 2.7145, | |
| "theoretical_loss": 3.793465512080541, | |
| "tokens_seen": 675020800 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.00038271940609041044, | |
| "loss": 2.7397, | |
| "theoretical_loss": 3.791589393350587, | |
| "tokens_seen": 678297600 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.00038212833364857196, | |
| "loss": 2.6861, | |
| "theoretical_loss": 3.7897248400204475, | |
| "tokens_seen": 681574400 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003815372612067335, | |
| "loss": 2.6654, | |
| "theoretical_loss": 3.7878717257038534, | |
| "tokens_seen": 684851200 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.000380946188764895, | |
| "loss": 2.6726, | |
| "theoretical_loss": 3.7860299259919685, | |
| "tokens_seen": 688128000 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0003803551163230566, | |
| "loss": 2.6455, | |
| "theoretical_loss": 3.7841993184132114, | |
| "tokens_seen": 691404800 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0003797640438812181, | |
| "loss": 2.6702, | |
| "theoretical_loss": 3.78237978239408, | |
| "tokens_seen": 694681600 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0003791729714393796, | |
| "loss": 2.6917, | |
| "theoretical_loss": 3.780571199220942, | |
| "tokens_seen": 697958400 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.00037858189899754114, | |
| "loss": 2.6649, | |
| "theoretical_loss": 3.7787734520027803, | |
| "tokens_seen": 701235200 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.00037799082655570266, | |
| "loss": 2.6522, | |
| "theoretical_loss": 3.7769864256348455, | |
| "tokens_seen": 704512000 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.00037741157556270093, | |
| "loss": 2.7032, | |
| "theoretical_loss": 3.775210006763202, | |
| "tokens_seen": 707788800 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0003768205031208625, | |
| "loss": 2.7074, | |
| "theoretical_loss": 3.7734440837501406, | |
| "tokens_seen": 711065600 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.000376229430679024, | |
| "loss": 2.7516, | |
| "theoretical_loss": 3.7716885466404246, | |
| "tokens_seen": 714342400 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.00037563835823718555, | |
| "loss": 2.7007, | |
| "theoretical_loss": 3.769943287128357, | |
| "tokens_seen": 717619200 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.00037504728579534707, | |
| "loss": 2.7292, | |
| "theoretical_loss": 3.7682081985256364, | |
| "tokens_seen": 720896000 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0003744562133535086, | |
| "loss": 2.7581, | |
| "theoretical_loss": 3.7664831757299795, | |
| "tokens_seen": 724172800 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0003738651409116701, | |
| "loss": 2.6871, | |
| "theoretical_loss": 3.7647681151944976, | |
| "tokens_seen": 727449600 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0003732740684698317, | |
| "loss": 2.6436, | |
| "theoretical_loss": 3.7630629148977937, | |
| "tokens_seen": 730726400 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0003726829960279932, | |
| "loss": 2.6503, | |
| "theoretical_loss": 3.761367474314768, | |
| "tokens_seen": 734003200 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0003720919235861547, | |
| "loss": 2.676, | |
| "theoretical_loss": 3.7596816943881084, | |
| "tokens_seen": 737280000 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.00037150085114431625, | |
| "loss": 2.6658, | |
| "theoretical_loss": 3.758005477500451, | |
| "tokens_seen": 740556800 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.00037090977870247777, | |
| "loss": 2.6606, | |
| "theoretical_loss": 3.756338727447186, | |
| "tokens_seen": 743833600 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0003703187062606393, | |
| "loss": 2.6554, | |
| "theoretical_loss": 3.7546813494098945, | |
| "tokens_seen": 747110400 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0003697276338188008, | |
| "loss": 2.6705, | |
| "theoretical_loss": 3.7530332499304007, | |
| "tokens_seen": 750387200 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0003691365613769624, | |
| "loss": 2.6521, | |
| "theoretical_loss": 3.7513943368854195, | |
| "tokens_seen": 753664000 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0003685454889351239, | |
| "loss": 2.6524, | |
| "theoretical_loss": 3.7497645194617863, | |
| "tokens_seen": 756940800 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0003679544164932854, | |
| "loss": 2.6184, | |
| "theoretical_loss": 3.748143708132246, | |
| "tokens_seen": 760217600 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.00036736334405144695, | |
| "loss": 2.6389, | |
| "theoretical_loss": 3.7465318146317994, | |
| "tokens_seen": 763494400 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.00036677227160960847, | |
| "loss": 2.596, | |
| "theoretical_loss": 3.7449287519345766, | |
| "tokens_seen": 766771200 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.00036618119916777, | |
| "loss": 2.6259, | |
| "theoretical_loss": 3.7433344342312385, | |
| "tokens_seen": 770048000 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.00036559012672593156, | |
| "loss": 2.6141, | |
| "theoretical_loss": 3.7417487769068756, | |
| "tokens_seen": 773324800 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0003649990542840931, | |
| "loss": 2.6488, | |
| "theoretical_loss": 3.7401716965194076, | |
| "tokens_seen": 776601600 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0003644079818422546, | |
| "loss": 2.6692, | |
| "theoretical_loss": 3.738603110778461, | |
| "tokens_seen": 779878400 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0003638169094004161, | |
| "loss": 2.6684, | |
| "theoretical_loss": 3.73704293852471, | |
| "tokens_seen": 783155200 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.00036322583695857765, | |
| "loss": 2.6492, | |
| "theoretical_loss": 3.7354910997096793, | |
| "tokens_seen": 786432000 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.00036263476451673917, | |
| "loss": 2.6465, | |
| "theoretical_loss": 3.7339475153759825, | |
| "tokens_seen": 789708800 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.00036204369207490074, | |
| "loss": 2.6071, | |
| "theoretical_loss": 3.732412107638, | |
| "tokens_seen": 792985600 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.00036145261963306226, | |
| "loss": 2.64, | |
| "theoretical_loss": 3.7308847996629724, | |
| "tokens_seen": 796262400 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0003608615471912238, | |
| "loss": 2.6283, | |
| "theoretical_loss": 3.7293655156525043, | |
| "tokens_seen": 799539200 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0003602704747493853, | |
| "loss": 2.6181, | |
| "theoretical_loss": 3.727854180824469, | |
| "tokens_seen": 802816000 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0003596794023075468, | |
| "loss": 2.5809, | |
| "theoretical_loss": 3.7263507213952978, | |
| "tokens_seen": 806092800 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.00035908832986570834, | |
| "loss": 2.5738, | |
| "theoretical_loss": 3.724855064562658, | |
| "tokens_seen": 809369600 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0003584972574238699, | |
| "loss": 2.602, | |
| "theoretical_loss": 3.723367138488488, | |
| "tokens_seen": 812646400 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.00035790618498203144, | |
| "loss": 2.6405, | |
| "theoretical_loss": 3.7218868722824014, | |
| "tokens_seen": 815923200 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "objective/train/docs_used": 436776, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.661590576171875, | |
| "objective/train/theoretical_loss": 3.7204141959854384, | |
| "objective/train/tokens_used": 839660000, | |
| "theoretical_loss": 3.7204141959854384, | |
| "tokens_seen": 819200000 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.00035731511254019296, | |
| "loss": 2.641, | |
| "theoretical_loss": 3.7204141959854384, | |
| "tokens_seen": 819200000 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0003567240400983545, | |
| "loss": 2.6069, | |
| "theoretical_loss": 3.718949040554162, | |
| "tokens_seen": 822476800 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.000356132967656516, | |
| "loss": 2.6626, | |
| "theoretical_loss": 3.7174913378450833, | |
| "tokens_seen": 825753600 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0003555418952146775, | |
| "loss": 2.6566, | |
| "theoretical_loss": 3.7160410205994183, | |
| "tokens_seen": 829030400 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.00035495082277283904, | |
| "loss": 2.6638, | |
| "theoretical_loss": 3.7145980224281585, | |
| "tokens_seen": 832307200 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0003543597503310006, | |
| "loss": 2.6863, | |
| "theoretical_loss": 3.713162277797449, | |
| "tokens_seen": 835584000 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.00035376867788916214, | |
| "loss": 2.6587, | |
| "theoretical_loss": 3.7117337220142748, | |
| "tokens_seen": 838860800 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.00035317760544732366, | |
| "loss": 2.6476, | |
| "theoretical_loss": 3.7103122912124364, | |
| "tokens_seen": 842137600 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0003525865330054852, | |
| "loss": 2.6524, | |
| "theoretical_loss": 3.7088979223388128, | |
| "tokens_seen": 845414400 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.00035199546056364665, | |
| "loss": 2.6555, | |
| "theoretical_loss": 3.70749055313991, | |
| "tokens_seen": 848691200 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.00035140438812180817, | |
| "loss": 2.6511, | |
| "theoretical_loss": 3.7060901221486766, | |
| "tokens_seen": 851968000 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00035081331567996974, | |
| "loss": 2.664, | |
| "theoretical_loss": 3.704696568671591, | |
| "tokens_seen": 855244800 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00035022224323813127, | |
| "loss": 2.6587, | |
| "theoretical_loss": 3.7033098327760063, | |
| "tokens_seen": 858521600 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.0003496429922451296, | |
| "loss": 2.61, | |
| "theoretical_loss": 3.7019298552777533, | |
| "tokens_seen": 861798400 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.0003490519198032911, | |
| "loss": 2.6281, | |
| "theoretical_loss": 3.700556577728988, | |
| "tokens_seen": 865075200 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00034846084736145263, | |
| "loss": 2.629, | |
| "theoretical_loss": 3.6991899424062815, | |
| "tokens_seen": 868352000 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00034786977491961415, | |
| "loss": 2.649, | |
| "theoretical_loss": 3.697829892298951, | |
| "tokens_seen": 871628800 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00034727870247777573, | |
| "loss": 2.624, | |
| "theoretical_loss": 3.696476371097618, | |
| "tokens_seen": 874905600 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00034668763003593725, | |
| "loss": 2.6113, | |
| "theoretical_loss": 3.695129323182993, | |
| "tokens_seen": 878182400 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00034609655759409877, | |
| "loss": 2.5968, | |
| "theoretical_loss": 3.693788693614879, | |
| "tokens_seen": 881459200 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0003455054851522603, | |
| "loss": 2.5875, | |
| "theoretical_loss": 3.6924544281213967, | |
| "tokens_seen": 884736000 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0003449144127104218, | |
| "loss": 2.5903, | |
| "theoretical_loss": 3.691126473088412, | |
| "tokens_seen": 888012800 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.00034432334026858333, | |
| "loss": 2.6052, | |
| "theoretical_loss": 3.689804775549173, | |
| "tokens_seen": 891289600 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.00034373226782674485, | |
| "loss": 2.5897, | |
| "theoretical_loss": 3.688489283174146, | |
| "tokens_seen": 894566400 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.00034314119538490643, | |
| "loss": 2.6264, | |
| "theoretical_loss": 3.6871799442610538, | |
| "tokens_seen": 897843200 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0003425501229430679, | |
| "loss": 2.6031, | |
| "theoretical_loss": 3.685876707725093, | |
| "tokens_seen": 901120000 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0003419590505012294, | |
| "loss": 2.5521, | |
| "theoretical_loss": 3.6845795230893517, | |
| "tokens_seen": 904396800 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.00034136797805939094, | |
| "loss": 2.584, | |
| "theoretical_loss": 3.6832883404754035, | |
| "tokens_seen": 907673600 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00034077690561755246, | |
| "loss": 2.5797, | |
| "theoretical_loss": 3.6820031105940796, | |
| "tokens_seen": 910950400 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.000340185833175714, | |
| "loss": 2.5943, | |
| "theoretical_loss": 3.6807237847364176, | |
| "tokens_seen": 914227200 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00033959476073387555, | |
| "loss": 2.6038, | |
| "theoretical_loss": 3.6794503147647846, | |
| "tokens_seen": 917504000 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0003390036882920371, | |
| "loss": 2.6191, | |
| "theoretical_loss": 3.67818265310416, | |
| "tokens_seen": 920780800 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0003384126158501986, | |
| "loss": 2.6231, | |
| "theoretical_loss": 3.6769207527335888, | |
| "tokens_seen": 924057600 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0003378215434083601, | |
| "loss": 2.6081, | |
| "theoretical_loss": 3.675664567177787, | |
| "tokens_seen": 927334400 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00033723047096652164, | |
| "loss": 2.5985, | |
| "theoretical_loss": 3.674414050498913, | |
| "tokens_seen": 930611200 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00033663939852468316, | |
| "loss": 2.6102, | |
| "theoretical_loss": 3.6731691572884824, | |
| "tokens_seen": 933888000 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00033604832608284473, | |
| "loss": 2.5746, | |
| "theoretical_loss": 3.671929842659438, | |
| "tokens_seen": 937164800 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00033545725364100625, | |
| "loss": 2.5936, | |
| "theoretical_loss": 3.6706960622383624, | |
| "tokens_seen": 940441600 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0003348661811991678, | |
| "loss": 2.5706, | |
| "theoretical_loss": 3.6694677721578377, | |
| "tokens_seen": 943718400 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0003342751087573293, | |
| "loss": 2.5973, | |
| "theoretical_loss": 3.66824492904894, | |
| "tokens_seen": 946995200 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0003336840363154908, | |
| "loss": 2.5857, | |
| "theoretical_loss": 3.667027490033874, | |
| "tokens_seen": 950272000 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00033309296387365234, | |
| "loss": 2.5566, | |
| "theoretical_loss": 3.6658154127187412, | |
| "tokens_seen": 953548800 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00033250189143181386, | |
| "loss": 2.5445, | |
| "theoretical_loss": 3.664608655186437, | |
| "tokens_seen": 956825600 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00033191081898997543, | |
| "loss": 2.6071, | |
| "theoretical_loss": 3.663407175989679, | |
| "tokens_seen": 960102400 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00033131974654813695, | |
| "loss": 2.5979, | |
| "theoretical_loss": 3.662210934144158, | |
| "tokens_seen": 963379200 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003307286741062985, | |
| "loss": 2.5734, | |
| "theoretical_loss": 3.661019889121812, | |
| "tokens_seen": 966656000 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00033013760166446, | |
| "loss": 2.5867, | |
| "theoretical_loss": 3.6598340008442234, | |
| "tokens_seen": 969932800 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003295465292226215, | |
| "loss": 2.6167, | |
| "theoretical_loss": 3.6586532296761285, | |
| "tokens_seen": 973209600 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00032895545678078304, | |
| "loss": 2.621, | |
| "theoretical_loss": 3.657477536419047, | |
| "tokens_seen": 976486400 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003283643843389446, | |
| "loss": 2.631, | |
| "theoretical_loss": 3.656306882305022, | |
| "tokens_seen": 979763200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.5857137347143212, | |
| "debugging/distinct-1-grams": 0.8015824539835295, | |
| "debugging/distinct-2-grams": 0.9770068773134507, | |
| "debugging/entropy-1-grams": 6.057437437902186, | |
| "debugging/entropy-2-grams": 6.967727189100934, | |
| "debugging/length": 546.0, | |
| "debugging/num_segments": 18, | |
| "epoch": 0.35, | |
| "objective/train/docs_used": 521967, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.800149440765381, | |
| "objective/train/theoretical_loss": 3.6551412289904697, | |
| "objective/train/tokens_used": 1003500000, | |
| "theoretical_loss": 3.6551412289904697, | |
| "tokens_seen": 983040000 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00032777331189710613, | |
| "loss": 2.5862, | |
| "theoretical_loss": 3.6551412289904697, | |
| "tokens_seen": 983040000 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00032718223945526765, | |
| "loss": 2.5939, | |
| "theoretical_loss": 3.6539805385501376, | |
| "tokens_seen": 986316800 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003265911670134292, | |
| "loss": 2.5941, | |
| "theoretical_loss": 3.652824773471171, | |
| "tokens_seen": 989593600 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003260000945715907, | |
| "loss": 2.6028, | |
| "theoretical_loss": 3.651673896647277, | |
| "tokens_seen": 992870400 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0003254090221297522, | |
| "loss": 2.5966, | |
| "theoretical_loss": 3.6505278713729985, | |
| "tokens_seen": 996147200 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0003248179496879138, | |
| "loss": 2.5693, | |
| "theoretical_loss": 3.6493866613380774, | |
| "tokens_seen": 999424000 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.00032423869869491206, | |
| "loss": 2.5594, | |
| "theoretical_loss": 3.648250230621924, | |
| "tokens_seen": 1002700800 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0003236476262530736, | |
| "loss": 2.5436, | |
| "theoretical_loss": 3.647118543688179, | |
| "tokens_seen": 1005977600 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0003230565538112351, | |
| "loss": 2.5506, | |
| "theoretical_loss": 3.6459915653793633, | |
| "tokens_seen": 1009254400 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0003224654813693966, | |
| "loss": 2.5771, | |
| "theoretical_loss": 3.644869260911628, | |
| "tokens_seen": 1012531200 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.00032187440892755814, | |
| "loss": 2.5578, | |
| "theoretical_loss": 3.64375159586959, | |
| "tokens_seen": 1015808000 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0003212833364857197, | |
| "loss": 2.5466, | |
| "theoretical_loss": 3.642638536201252, | |
| "tokens_seen": 1019084800 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00032069226404388124, | |
| "loss": 2.561, | |
| "theoretical_loss": 3.6415300482130135, | |
| "tokens_seen": 1022361600 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00032010119160204276, | |
| "loss": 2.5863, | |
| "theoretical_loss": 3.6404260985647667, | |
| "tokens_seen": 1025638400 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0003195101191602043, | |
| "loss": 2.5644, | |
| "theoretical_loss": 3.6393266542650684, | |
| "tokens_seen": 1028915200 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0003189190467183658, | |
| "loss": 2.5471, | |
| "theoretical_loss": 3.638231682666401, | |
| "tokens_seen": 1032192000 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0003183279742765273, | |
| "loss": 2.553, | |
| "theoretical_loss": 3.637141151460505, | |
| "tokens_seen": 1035468800 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00031773690183468884, | |
| "loss": 2.5607, | |
| "theoretical_loss": 3.636055028673799, | |
| "tokens_seen": 1038745600 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0003171458293928504, | |
| "loss": 2.5638, | |
| "theoretical_loss": 3.634973282662864, | |
| "tokens_seen": 1042022400 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00031655475695101194, | |
| "loss": 2.6156, | |
| "theoretical_loss": 3.6338958821100107, | |
| "tokens_seen": 1045299200 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00031596368450917346, | |
| "loss": 2.6571, | |
| "theoretical_loss": 3.63282279601892, | |
| "tokens_seen": 1048576000 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.000315372612067335, | |
| "loss": 2.6717, | |
| "theoretical_loss": 3.631753993710352, | |
| "tokens_seen": 1051852800 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0003147815396254965, | |
| "loss": 2.6583, | |
| "theoretical_loss": 3.630689444817925, | |
| "tokens_seen": 1055129600 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.000314190467183658, | |
| "loss": 2.6705, | |
| "theoretical_loss": 3.629629119283967, | |
| "tokens_seen": 1058406400 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0003135993947418196, | |
| "loss": 2.6554, | |
| "theoretical_loss": 3.628572987355434, | |
| "tokens_seen": 1061683200 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0003130083222999811, | |
| "loss": 2.6439, | |
| "theoretical_loss": 3.6275210195798913, | |
| "tokens_seen": 1064960000 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.00031241724985814264, | |
| "loss": 2.6397, | |
| "theoretical_loss": 3.626473186801564, | |
| "tokens_seen": 1068236800 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.00031182617741630416, | |
| "loss": 2.6653, | |
| "theoretical_loss": 3.6254294601574495, | |
| "tokens_seen": 1071513600 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0003112351049744657, | |
| "loss": 2.61, | |
| "theoretical_loss": 3.624389811073493, | |
| "tokens_seen": 1074790400 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003106440325326272, | |
| "loss": 2.6357, | |
| "theoretical_loss": 3.6233542112608257, | |
| "tokens_seen": 1078067200 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003100529600907888, | |
| "loss": 2.6169, | |
| "theoretical_loss": 3.6223226327120592, | |
| "tokens_seen": 1081344000 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003094618876489503, | |
| "loss": 2.6232, | |
| "theoretical_loss": 3.621295047697644, | |
| "tokens_seen": 1084620800 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003088708152071118, | |
| "loss": 2.6621, | |
| "theoretical_loss": 3.6202714287622833, | |
| "tokens_seen": 1087897600 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00030827974276527334, | |
| "loss": 2.6171, | |
| "theoretical_loss": 3.6192517487214038, | |
| "tokens_seen": 1091174400 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00030768867032343486, | |
| "loss": 2.5856, | |
| "theoretical_loss": 3.6182359806576834, | |
| "tokens_seen": 1094451200 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003070975978815964, | |
| "loss": 2.6144, | |
| "theoretical_loss": 3.6172240979176333, | |
| "tokens_seen": 1097728000 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003065065254397579, | |
| "loss": 2.5866, | |
| "theoretical_loss": 3.616216074108232, | |
| "tokens_seen": 1101004800 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003059154529979195, | |
| "loss": 2.6417, | |
| "theoretical_loss": 3.6152118830936164, | |
| "tokens_seen": 1104281600 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.000305324380556081, | |
| "loss": 2.6339, | |
| "theoretical_loss": 3.6142114989918195, | |
| "tokens_seen": 1107558400 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00030473330811424246, | |
| "loss": 2.626, | |
| "theoretical_loss": 3.6132148961715624, | |
| "tokens_seen": 1110835200 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.000304142235672404, | |
| "loss": 2.6102, | |
| "theoretical_loss": 3.6122220492490964, | |
| "tokens_seen": 1114112000 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0003035511632305655, | |
| "loss": 2.5967, | |
| "theoretical_loss": 3.6112329330850894, | |
| "tokens_seen": 1117388800 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.000302960090788727, | |
| "loss": 2.597, | |
| "theoretical_loss": 3.61024752278157, | |
| "tokens_seen": 1120665600 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0003023690183468886, | |
| "loss": 2.6077, | |
| "theoretical_loss": 3.6092657936789054, | |
| "tokens_seen": 1123942400 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0003017779459050501, | |
| "loss": 2.5608, | |
| "theoretical_loss": 3.6082877213528377, | |
| "tokens_seen": 1127219200 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00030118687346321164, | |
| "loss": 2.5657, | |
| "theoretical_loss": 3.60731328161156, | |
| "tokens_seen": 1130496000 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00030059580102137316, | |
| "loss": 2.6017, | |
| "theoretical_loss": 3.6063424504928365, | |
| "tokens_seen": 1133772800 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0003000047285795347, | |
| "loss": 2.5677, | |
| "theoretical_loss": 3.60537520426117, | |
| "tokens_seen": 1137049600 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0002994136561376962, | |
| "loss": 2.5892, | |
| "theoretical_loss": 3.6044115194050086, | |
| "tokens_seen": 1140326400 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0002988225836958578, | |
| "loss": 2.5556, | |
| "theoretical_loss": 3.603451372633997, | |
| "tokens_seen": 1143603200 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "objective/train/docs_used": 604425, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.6260290145874023, | |
| "objective/train/theoretical_loss": 3.6024947408762698, | |
| "objective/train/tokens_used": 1167340000, | |
| "theoretical_loss": 3.6024947408762698, | |
| "tokens_seen": 1146880000 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0002982433327028561, | |
| "loss": 2.6048, | |
| "theoretical_loss": 3.6024947408762698, | |
| "tokens_seen": 1146880000 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0002976640817098544, | |
| "loss": 2.6085, | |
| "theoretical_loss": 3.601541601275783, | |
| "tokens_seen": 1150156800 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0002970730092680159, | |
| "loss": 2.5966, | |
| "theoretical_loss": 3.6005919311896886, | |
| "tokens_seen": 1153433600 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0002964819368261774, | |
| "loss": 2.5683, | |
| "theoretical_loss": 3.5996457081857454, | |
| "tokens_seen": 1156710400 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00029589086438433894, | |
| "loss": 2.5609, | |
| "theoretical_loss": 3.598702910039772, | |
| "tokens_seen": 1159987200 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0002952997919425005, | |
| "loss": 2.5521, | |
| "theoretical_loss": 3.597763514733133, | |
| "tokens_seen": 1163264000 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00029470871950066204, | |
| "loss": 2.5443, | |
| "theoretical_loss": 3.59682750045027, | |
| "tokens_seen": 1166540800 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00029411764705882356, | |
| "loss": 2.5605, | |
| "theoretical_loss": 3.5958948455762583, | |
| "tokens_seen": 1169817600 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0002935265746169851, | |
| "loss": 2.6041, | |
| "theoretical_loss": 3.594965528694412, | |
| "tokens_seen": 1173094400 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0002929355021751466, | |
| "loss": 2.574, | |
| "theoretical_loss": 3.594039528583913, | |
| "tokens_seen": 1176371200 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0002923444297333081, | |
| "loss": 2.5796, | |
| "theoretical_loss": 3.5931168242174847, | |
| "tokens_seen": 1179648000 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00029175335729146964, | |
| "loss": 2.5703, | |
| "theoretical_loss": 3.59219739475909, | |
| "tokens_seen": 1182924800 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0002911622848496312, | |
| "loss": 2.5559, | |
| "theoretical_loss": 3.5912812195616732, | |
| "tokens_seen": 1186201600 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00029057121240779274, | |
| "loss": 2.5536, | |
| "theoretical_loss": 3.590368278164926, | |
| "tokens_seen": 1189478400 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.00028998013996595426, | |
| "loss": 2.5564, | |
| "theoretical_loss": 3.5894585502930902, | |
| "tokens_seen": 1192755200 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0002893890675241158, | |
| "loss": 2.6011, | |
| "theoretical_loss": 3.588552015852793, | |
| "tokens_seen": 1196032000 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0002887979950822773, | |
| "loss": 2.5488, | |
| "theoretical_loss": 3.5876486549309097, | |
| "tokens_seen": 1199308800 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0002882069226404388, | |
| "loss": 2.5447, | |
| "theoretical_loss": 3.586748447792462, | |
| "tokens_seen": 1202585600 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0002876158501986004, | |
| "loss": 2.5656, | |
| "theoretical_loss": 3.5858513748785423, | |
| "tokens_seen": 1205862400 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0002870247777567619, | |
| "loss": 2.5935, | |
| "theoretical_loss": 3.5849574168042704, | |
| "tokens_seen": 1209139200 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.00028643370531492344, | |
| "loss": 2.6245, | |
| "theoretical_loss": 3.5840665543567782, | |
| "tokens_seen": 1212416000 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0002858426328730849, | |
| "loss": 2.5978, | |
| "theoretical_loss": 3.583178768493222, | |
| "tokens_seen": 1215692800 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002852515604312464, | |
| "loss": 2.5588, | |
| "theoretical_loss": 3.5822940403388284, | |
| "tokens_seen": 1218969600 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00028466048798940794, | |
| "loss": 2.5947, | |
| "theoretical_loss": 3.581412351184958, | |
| "tokens_seen": 1222246400 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002840694155475695, | |
| "loss": 2.5713, | |
| "theoretical_loss": 3.580533682487208, | |
| "tokens_seen": 1225523200 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00028347834310573104, | |
| "loss": 2.5671, | |
| "theoretical_loss": 3.579658015863532, | |
| "tokens_seen": 1228800000 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00028288727066389256, | |
| "loss": 2.5841, | |
| "theoretical_loss": 3.5787853330923927, | |
| "tokens_seen": 1232076800 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002822961982220541, | |
| "loss": 2.5546, | |
| "theoretical_loss": 3.577915616110936, | |
| "tokens_seen": 1235353600 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002817051257802156, | |
| "loss": 2.5536, | |
| "theoretical_loss": 3.577048847013194, | |
| "tokens_seen": 1238630400 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002811140533383771, | |
| "loss": 2.5912, | |
| "theoretical_loss": 3.57618500804831, | |
| "tokens_seen": 1241907200 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00028052298089653864, | |
| "loss": 2.5372, | |
| "theoretical_loss": 3.575324081618793, | |
| "tokens_seen": 1245184000 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.0002799319084547002, | |
| "loss": 2.5759, | |
| "theoretical_loss": 3.5744660502787875, | |
| "tokens_seen": 1248460800 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00027934083601286174, | |
| "loss": 2.5583, | |
| "theoretical_loss": 3.5736108967323794, | |
| "tokens_seen": 1251737600 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00027874976357102326, | |
| "loss": 2.5872, | |
| "theoretical_loss": 3.5727586038319155, | |
| "tokens_seen": 1255014400 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.0002781586911291848, | |
| "loss": 2.5319, | |
| "theoretical_loss": 3.571909154576348, | |
| "tokens_seen": 1258291200 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.0002775676186873463, | |
| "loss": 2.5563, | |
| "theoretical_loss": 3.5710625321096074, | |
| "tokens_seen": 1261568000 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.0002769765462455078, | |
| "loss": 2.553, | |
| "theoretical_loss": 3.570218719718989, | |
| "tokens_seen": 1264844800 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.0002763854738036694, | |
| "loss": 2.5804, | |
| "theoretical_loss": 3.569377700833569, | |
| "tokens_seen": 1268121600 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.0002757944013618309, | |
| "loss": 2.5463, | |
| "theoretical_loss": 3.568539459022639, | |
| "tokens_seen": 1271398400 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00027520332891999244, | |
| "loss": 2.5542, | |
| "theoretical_loss": 3.5677039779941584, | |
| "tokens_seen": 1274675200 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00027461225647815396, | |
| "loss": 2.5695, | |
| "theoretical_loss": 3.566871241593236, | |
| "tokens_seen": 1277952000 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0002740211840363155, | |
| "loss": 2.5521, | |
| "theoretical_loss": 3.5660412338006235, | |
| "tokens_seen": 1281228800 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.000273430111594477, | |
| "loss": 2.5862, | |
| "theoretical_loss": 3.565213938731236, | |
| "tokens_seen": 1284505600 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0002728390391526386, | |
| "loss": 2.5867, | |
| "theoretical_loss": 3.5643893406326868, | |
| "tokens_seen": 1287782400 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0002722479667108001, | |
| "loss": 2.5822, | |
| "theoretical_loss": 3.5635674238838466, | |
| "tokens_seen": 1291059200 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0002716568942689616, | |
| "loss": 2.5458, | |
| "theoretical_loss": 3.5627481729934196, | |
| "tokens_seen": 1294336000 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00027106582182712314, | |
| "loss": 2.5858, | |
| "theoretical_loss": 3.561931572598538, | |
| "tokens_seen": 1297612800 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00027047474938528466, | |
| "loss": 2.5536, | |
| "theoretical_loss": 3.5611176074633777, | |
| "tokens_seen": 1300889600 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0002698836769434462, | |
| "loss": 2.554, | |
| "theoretical_loss": 3.5603062624777895, | |
| "tokens_seen": 1304166400 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0002692926045016077, | |
| "loss": 2.6034, | |
| "theoretical_loss": 3.559497522655951, | |
| "tokens_seen": 1307443200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.6360978963817496, | |
| "debugging/distinct-1-grams": 0.7608773404922611, | |
| "debugging/distinct-2-grams": 0.9487668534919836, | |
| "debugging/entropy-1-grams": 6.132694731605541, | |
| "debugging/entropy-2-grams": 7.054469626558369, | |
| "debugging/length": 633.2352941176471, | |
| "debugging/num_segments": 17, | |
| "epoch": 0.47, | |
| "objective/train/docs_used": 687081, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.799859046936035, | |
| "objective/train/theoretical_loss": 3.5586913731350327, | |
| "objective/train/tokens_used": 1331180000, | |
| "theoretical_loss": 3.5586913731350327, | |
| "tokens_seen": 1310720000 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0002687015320597693, | |
| "loss": 2.6091, | |
| "theoretical_loss": 3.5586913731350327, | |
| "tokens_seen": 1310720000 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0002681104596179308, | |
| "loss": 2.6058, | |
| "theoretical_loss": 3.557887799173889, | |
| "tokens_seen": 1313996800 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0002675193871760923, | |
| "loss": 2.618, | |
| "theoretical_loss": 3.557086786151754, | |
| "tokens_seen": 1317273600 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00026692831473425384, | |
| "loss": 2.5715, | |
| "theoretical_loss": 3.5562883195669697, | |
| "tokens_seen": 1320550400 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00026633724229241536, | |
| "loss": 2.6036, | |
| "theoretical_loss": 3.555492385035719, | |
| "tokens_seen": 1323827200 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0002657461698505769, | |
| "loss": 2.5559, | |
| "theoretical_loss": 3.5546989682907784, | |
| "tokens_seen": 1327104000 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026515509740873846, | |
| "loss": 2.5523, | |
| "theoretical_loss": 3.5539080551802895, | |
| "tokens_seen": 1330380800 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.0002645640249669, | |
| "loss": 2.556, | |
| "theoretical_loss": 3.553119631666546, | |
| "tokens_seen": 1333657600 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.0002639729525250615, | |
| "loss": 2.559, | |
| "theoretical_loss": 3.5523336838247914, | |
| "tokens_seen": 1336934400 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.000263381880083223, | |
| "loss": 2.5567, | |
| "theoretical_loss": 3.55155019784204, | |
| "tokens_seen": 1340211200 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026279080764138454, | |
| "loss": 2.5476, | |
| "theoretical_loss": 3.5507691600159053, | |
| "tokens_seen": 1343488000 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026219973519954606, | |
| "loss": 2.5683, | |
| "theoretical_loss": 3.5499905567534515, | |
| "tokens_seen": 1346764800 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026160866275770763, | |
| "loss": 2.5248, | |
| "theoretical_loss": 3.549214374570052, | |
| "tokens_seen": 1350041600 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026101759031586916, | |
| "loss": 2.5172, | |
| "theoretical_loss": 3.5484406000882665, | |
| "tokens_seen": 1353318400 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.0002604265178740307, | |
| "loss": 2.532, | |
| "theoretical_loss": 3.5476692200367346, | |
| "tokens_seen": 1356595200 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.0002598354454321922, | |
| "loss": 2.4928, | |
| "theoretical_loss": 3.546900221249076, | |
| "tokens_seen": 1359872000 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.0002592443729903537, | |
| "loss": 2.5419, | |
| "theoretical_loss": 3.5461335906628157, | |
| "tokens_seen": 1363148800 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00025865330054851524, | |
| "loss": 2.5409, | |
| "theoretical_loss": 3.54536931531831, | |
| "tokens_seen": 1366425600 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00025806222810667676, | |
| "loss": 2.498, | |
| "theoretical_loss": 3.5446073823576985, | |
| "tokens_seen": 1369702400 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00025747115566483833, | |
| "loss": 2.5569, | |
| "theoretical_loss": 3.543847779023859, | |
| "tokens_seen": 1372979200 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.0002568800832229998, | |
| "loss": 2.4994, | |
| "theoretical_loss": 3.543090492659384, | |
| "tokens_seen": 1376256000 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.0002562890107811613, | |
| "loss": 2.505, | |
| "theoretical_loss": 3.542335510705562, | |
| "tokens_seen": 1379532800 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00025569793833932284, | |
| "loss": 2.5426, | |
| "theoretical_loss": 3.541582820701378, | |
| "tokens_seen": 1382809600 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.00025510686589748436, | |
| "loss": 2.4956, | |
| "theoretical_loss": 3.5408324102825253, | |
| "tokens_seen": 1386086400 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.0002545157934556459, | |
| "loss": 2.5037, | |
| "theoretical_loss": 3.5400842671804265, | |
| "tokens_seen": 1389363200 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.00025392472101380746, | |
| "loss": 2.4902, | |
| "theoretical_loss": 3.5393383792212676, | |
| "tokens_seen": 1392640000 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.000253333648571969, | |
| "loss": 2.5279, | |
| "theoretical_loss": 3.5385947343250486, | |
| "tokens_seen": 1395916800 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.0002527425761301305, | |
| "loss": 2.5259, | |
| "theoretical_loss": 3.53785332050464, | |
| "tokens_seen": 1399193600 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.000252151503688292, | |
| "loss": 2.4974, | |
| "theoretical_loss": 3.537114125864853, | |
| "tokens_seen": 1402470400 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.00025156043124645354, | |
| "loss": 2.5276, | |
| "theoretical_loss": 3.536377138601523, | |
| "tokens_seen": 1405747200 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.00025096935880461506, | |
| "loss": 2.5619, | |
| "theoretical_loss": 3.5356423470006018, | |
| "tokens_seen": 1409024000 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.00025037828636277664, | |
| "loss": 2.5225, | |
| "theoretical_loss": 3.5349097394372615, | |
| "tokens_seen": 1412300800 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.00024978721392093816, | |
| "loss": 2.5266, | |
| "theoretical_loss": 3.5341793043750114, | |
| "tokens_seen": 1415577600 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.0002491961414790997, | |
| "loss": 2.5275, | |
| "theoretical_loss": 3.533451030364823, | |
| "tokens_seen": 1418854400 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.0002486050690372612, | |
| "loss": 2.5188, | |
| "theoretical_loss": 3.5327249060442676, | |
| "tokens_seen": 1422131200 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.0002480139965954227, | |
| "loss": 2.5231, | |
| "theoretical_loss": 3.5320009201366616, | |
| "tokens_seen": 1425408000 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.0002474229241535843, | |
| "loss": 2.5609, | |
| "theoretical_loss": 3.531279061450226, | |
| "tokens_seen": 1428684800 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.0002468318517117458, | |
| "loss": 2.5196, | |
| "theoretical_loss": 3.5305593188772533, | |
| "tokens_seen": 1431961600 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.00024624077926990734, | |
| "loss": 2.5443, | |
| "theoretical_loss": 3.5298416813932865, | |
| "tokens_seen": 1435238400 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.00024564970682806886, | |
| "loss": 2.5678, | |
| "theoretical_loss": 3.529126138056305, | |
| "tokens_seen": 1438515200 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.0002450586343862304, | |
| "loss": 2.5256, | |
| "theoretical_loss": 3.528412678005922, | |
| "tokens_seen": 1441792000 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 0.0002444675619443919, | |
| "loss": 2.5342, | |
| "theoretical_loss": 3.527701290462595, | |
| "tokens_seen": 1445068800 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 0.00024387648950255345, | |
| "loss": 2.523, | |
| "theoretical_loss": 3.5269919647268377, | |
| "tokens_seen": 1448345600 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 0.00024328541706071497, | |
| "loss": 2.5114, | |
| "theoretical_loss": 3.526284690178448, | |
| "tokens_seen": 1451622400 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 0.0002426943446188765, | |
| "loss": 2.5111, | |
| "theoretical_loss": 3.5255794562757443, | |
| "tokens_seen": 1454899200 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 0.00024210327217703804, | |
| "loss": 2.5011, | |
| "theoretical_loss": 3.524876252554808, | |
| "tokens_seen": 1458176000 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 0.00024151219973519956, | |
| "loss": 2.513, | |
| "theoretical_loss": 3.524175068628737, | |
| "tokens_seen": 1461452800 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 0.00024092112729336108, | |
| "loss": 2.5425, | |
| "theoretical_loss": 3.5234758941869075, | |
| "tokens_seen": 1464729600 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 0.0002403300548515226, | |
| "loss": 2.5646, | |
| "theoretical_loss": 3.522778718994246, | |
| "tokens_seen": 1468006400 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.0002397508038585209, | |
| "loss": 2.5519, | |
| "theoretical_loss": 3.5220835328905062, | |
| "tokens_seen": 1471283200 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "objective/train/docs_used": 774317, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.6251282691955566, | |
| "objective/train/theoretical_loss": 3.521390325789559, | |
| "objective/train/tokens_used": 1495020000, | |
| "theoretical_loss": 3.521390325789559, | |
| "tokens_seen": 1474560000 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.00023915973141668242, | |
| "loss": 2.5411, | |
| "theoretical_loss": 3.521390325789559, | |
| "tokens_seen": 1474560000 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.00023856865897484394, | |
| "loss": 2.537, | |
| "theoretical_loss": 3.520699087678688, | |
| "tokens_seen": 1477836800 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.0002379775865330055, | |
| "loss": 2.5459, | |
| "theoretical_loss": 3.5200098086178926, | |
| "tokens_seen": 1481113600 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.000237386514091167, | |
| "loss": 2.5432, | |
| "theoretical_loss": 3.5193224787392015, | |
| "tokens_seen": 1484390400 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.00023679544164932853, | |
| "loss": 2.5357, | |
| "theoretical_loss": 3.518637088245993, | |
| "tokens_seen": 1487667200 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.00023620436920749008, | |
| "loss": 2.5358, | |
| "theoretical_loss": 3.5179536274123233, | |
| "tokens_seen": 1490944000 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.0002356132967656516, | |
| "loss": 2.5185, | |
| "theoretical_loss": 3.5172720865822633, | |
| "tokens_seen": 1494220800 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.00023502222432381312, | |
| "loss": 2.5095, | |
| "theoretical_loss": 3.5165924561692394, | |
| "tokens_seen": 1497497600 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 0.00023443115188197467, | |
| "loss": 2.5051, | |
| "theoretical_loss": 3.5159147266553887, | |
| "tokens_seen": 1500774400 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 0.0002338400794401362, | |
| "loss": 2.5473, | |
| "theoretical_loss": 3.515238888590916, | |
| "tokens_seen": 1504051200 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 0.0002332490069982977, | |
| "loss": 2.5373, | |
| "theoretical_loss": 3.51456493259346, | |
| "tokens_seen": 1507328000 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 0.00023265793455645926, | |
| "loss": 2.5436, | |
| "theoretical_loss": 3.5138928493474664, | |
| "tokens_seen": 1510604800 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 0.00023206686211462078, | |
| "loss": 2.524, | |
| "theoretical_loss": 3.513222629603569, | |
| "tokens_seen": 1513881600 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 0.0002314757896727823, | |
| "loss": 2.5107, | |
| "theoretical_loss": 3.512554264177979, | |
| "tokens_seen": 1517158400 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 0.00023088471723094385, | |
| "loss": 2.4951, | |
| "theoretical_loss": 3.511887743951874, | |
| "tokens_seen": 1520435200 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 0.00023029364478910537, | |
| "loss": 2.4738, | |
| "theoretical_loss": 3.511223059870809, | |
| "tokens_seen": 1523712000 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.0002297025723472669, | |
| "loss": 2.4675, | |
| "theoretical_loss": 3.5105602029441143, | |
| "tokens_seen": 1526988800 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.00022911149990542844, | |
| "loss": 2.471, | |
| "theoretical_loss": 3.5098991642443185, | |
| "tokens_seen": 1530265600 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.00022852042746358996, | |
| "loss": 2.5191, | |
| "theoretical_loss": 3.5092399349065655, | |
| "tokens_seen": 1533542400 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.00022792935502175148, | |
| "loss": 2.4958, | |
| "theoretical_loss": 3.5085825061280445, | |
| "tokens_seen": 1536819200 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.000227338282579913, | |
| "loss": 2.4852, | |
| "theoretical_loss": 3.507926869167425, | |
| "tokens_seen": 1540096000 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.00022674721013807455, | |
| "loss": 2.5191, | |
| "theoretical_loss": 3.507273015344296, | |
| "tokens_seen": 1543372800 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.00022615613769623604, | |
| "loss": 2.4683, | |
| "theoretical_loss": 3.5066209360386154, | |
| "tokens_seen": 1546649600 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.00022556506525439756, | |
| "loss": 2.5015, | |
| "theoretical_loss": 3.5059706226901595, | |
| "tokens_seen": 1549926400 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.0002249739928125591, | |
| "loss": 2.4505, | |
| "theoretical_loss": 3.505322066797988, | |
| "tokens_seen": 1553203200 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 0.00022438292037072063, | |
| "loss": 2.4597, | |
| "theoretical_loss": 3.504675259919904, | |
| "tokens_seen": 1556480000 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 0.00022379184792888215, | |
| "loss": 2.4489, | |
| "theoretical_loss": 3.5040301936719302, | |
| "tokens_seen": 1559756800 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 0.0002232007754870437, | |
| "loss": 2.4646, | |
| "theoretical_loss": 3.5033868597277813, | |
| "tokens_seen": 1563033600 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 0.00022260970304520522, | |
| "loss": 2.4352, | |
| "theoretical_loss": 3.502745249818351, | |
| "tokens_seen": 1566310400 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 0.00022201863060336674, | |
| "loss": 2.4722, | |
| "theoretical_loss": 3.5021053557311985, | |
| "tokens_seen": 1569587200 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 0.0002214275581615283, | |
| "loss": 2.4657, | |
| "theoretical_loss": 3.501467169310044, | |
| "tokens_seen": 1572864000 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 0.0002208364857196898, | |
| "loss": 2.4585, | |
| "theoretical_loss": 3.500830682454268, | |
| "tokens_seen": 1576140800 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 0.00022024541327785133, | |
| "loss": 2.4841, | |
| "theoretical_loss": 3.5001958871184167, | |
| "tokens_seen": 1579417600 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.00021965434083601288, | |
| "loss": 2.5061, | |
| "theoretical_loss": 3.4995627753117127, | |
| "tokens_seen": 1582694400 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.0002190632683941744, | |
| "loss": 2.5077, | |
| "theoretical_loss": 3.498931339097571, | |
| "tokens_seen": 1585971200 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.00021847219595233592, | |
| "loss": 2.4916, | |
| "theoretical_loss": 3.4983015705931226, | |
| "tokens_seen": 1589248000 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.00021788112351049747, | |
| "loss": 2.4953, | |
| "theoretical_loss": 3.4976734619687377, | |
| "tokens_seen": 1592524800 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.000217290051068659, | |
| "loss": 2.5063, | |
| "theoretical_loss": 3.497047005447558, | |
| "tokens_seen": 1595801600 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.0002166989786268205, | |
| "loss": 2.5219, | |
| "theoretical_loss": 3.4964221933050377, | |
| "tokens_seen": 1599078400 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.00021610790618498203, | |
| "loss": 2.5385, | |
| "theoretical_loss": 3.495799017868479, | |
| "tokens_seen": 1602355200 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.00021551683374314358, | |
| "loss": 2.5294, | |
| "theoretical_loss": 3.495177471516584, | |
| "tokens_seen": 1605632000 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.0002149257613013051, | |
| "loss": 2.5031, | |
| "theoretical_loss": 3.4945575466790024, | |
| "tokens_seen": 1608908800 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 0.00021433468885946662, | |
| "loss": 2.5209, | |
| "theoretical_loss": 3.4939392358358923, | |
| "tokens_seen": 1612185600 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 0.00021374361641762817, | |
| "loss": 2.5568, | |
| "theoretical_loss": 3.493322531517478, | |
| "tokens_seen": 1615462400 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 0.0002131525439757897, | |
| "loss": 2.5458, | |
| "theoretical_loss": 3.4927074263036157, | |
| "tokens_seen": 1618739200 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 0.00021257329298278796, | |
| "loss": 2.5147, | |
| "theoretical_loss": 3.492093912823367, | |
| "tokens_seen": 1622016000 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 0.0002119822205409495, | |
| "loss": 2.5227, | |
| "theoretical_loss": 3.4914819837545705, | |
| "tokens_seen": 1625292800 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 0.00021139114809911103, | |
| "loss": 2.529, | |
| "theoretical_loss": 3.490871631823426, | |
| "tokens_seen": 1628569600 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 0.00021080007565727255, | |
| "loss": 2.5455, | |
| "theoretical_loss": 3.490262849804073, | |
| "tokens_seen": 1631846400 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 0.0002102090032154341, | |
| "loss": 2.5266, | |
| "theoretical_loss": 3.4896556305181834, | |
| "tokens_seen": 1635123200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.5275920475271991, | |
| "debugging/distinct-1-grams": 0.7407784153685145, | |
| "debugging/distinct-2-grams": 0.9366109467220821, | |
| "debugging/entropy-1-grams": 5.798263193796952, | |
| "debugging/entropy-2-grams": 6.65668849844512, | |
| "debugging/length": 561.4615384615385, | |
| "debugging/num_segments": 13, | |
| "epoch": 0.59, | |
| "objective/train/docs_used": 859391, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.4356160163879395, | |
| "objective/train/theoretical_loss": 3.4890499668345516, | |
| "objective/train/tokens_used": 1658860000, | |
| "theoretical_loss": 3.4890499668345516, | |
| "tokens_seen": 1638400000 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.00020961793077359562, | |
| "loss": 2.5196, | |
| "theoretical_loss": 3.4890499668345516, | |
| "tokens_seen": 1638400000 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.00020902685833175714, | |
| "loss": 2.4829, | |
| "theoretical_loss": 3.4884458516686947, | |
| "tokens_seen": 1641676800 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.00020843578588991869, | |
| "loss": 2.4802, | |
| "theoretical_loss": 3.487843277982451, | |
| "tokens_seen": 1644953600 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.0002078447134480802, | |
| "loss": 2.4863, | |
| "theoretical_loss": 3.4872422387835877, | |
| "tokens_seen": 1648230400 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.00020725364100624173, | |
| "loss": 2.5052, | |
| "theoretical_loss": 3.4866427271254077, | |
| "tokens_seen": 1651507200 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.00020666256856440328, | |
| "loss": 2.4742, | |
| "theoretical_loss": 3.4860447361063667, | |
| "tokens_seen": 1654784000 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.0002060714961225648, | |
| "loss": 2.4871, | |
| "theoretical_loss": 3.48544825886969, | |
| "tokens_seen": 1658060800 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.00020548042368072632, | |
| "loss": 2.4904, | |
| "theoretical_loss": 3.4848532886029924, | |
| "tokens_seen": 1661337600 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.00020488935123888786, | |
| "loss": 2.4644, | |
| "theoretical_loss": 3.4842598185379066, | |
| "tokens_seen": 1664614400 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "learning_rate": 0.00020429827879704939, | |
| "loss": 2.5134, | |
| "theoretical_loss": 3.4836678419497105, | |
| "tokens_seen": 1667891200 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "learning_rate": 0.0002037072063552109, | |
| "loss": 2.508, | |
| "theoretical_loss": 3.483077352156964, | |
| "tokens_seen": 1671168000 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "learning_rate": 0.00020311613391337243, | |
| "loss": 2.516, | |
| "theoretical_loss": 3.4824883425211413, | |
| "tokens_seen": 1674444800 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "learning_rate": 0.00020252506147153395, | |
| "loss": 2.5011, | |
| "theoretical_loss": 3.4819008064462764, | |
| "tokens_seen": 1677721600 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "learning_rate": 0.00020193398902969547, | |
| "loss": 2.4943, | |
| "theoretical_loss": 3.481314737378605, | |
| "tokens_seen": 1680998400 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "learning_rate": 0.000201342916587857, | |
| "loss": 2.4885, | |
| "theoretical_loss": 3.4807301288062145, | |
| "tokens_seen": 1684275200 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "learning_rate": 0.00020075184414601854, | |
| "loss": 2.4542, | |
| "theoretical_loss": 3.4801469742586955, | |
| "tokens_seen": 1687552000 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "learning_rate": 0.00020016077170418006, | |
| "loss": 2.4444, | |
| "theoretical_loss": 3.4795652673067954, | |
| "tokens_seen": 1690828800 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.00019956969926234158, | |
| "loss": 2.485, | |
| "theoretical_loss": 3.478985001562082, | |
| "tokens_seen": 1694105600 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.00019897862682050313, | |
| "loss": 2.5134, | |
| "theoretical_loss": 3.4784061706766, | |
| "tokens_seen": 1697382400 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.00019838755437866465, | |
| "loss": 2.5253, | |
| "theoretical_loss": 3.477828768342543, | |
| "tokens_seen": 1700659200 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.00019779648193682617, | |
| "loss": 2.5363, | |
| "theoretical_loss": 3.477252788291917, | |
| "tokens_seen": 1703936000 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.00019720540949498772, | |
| "loss": 2.4912, | |
| "theoretical_loss": 3.4766782242962186, | |
| "tokens_seen": 1707212800 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.00019661433705314924, | |
| "loss": 2.4705, | |
| "theoretical_loss": 3.476105070166107, | |
| "tokens_seen": 1710489600 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.00019602326461131076, | |
| "loss": 2.4601, | |
| "theoretical_loss": 3.475533319751085, | |
| "tokens_seen": 1713766400 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.0001954321921694723, | |
| "loss": 2.455, | |
| "theoretical_loss": 3.4749629669391813, | |
| "tokens_seen": 1717043200 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.00019484111972763383, | |
| "loss": 2.4814, | |
| "theoretical_loss": 3.474394005656636, | |
| "tokens_seen": 1720320000 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.00019425004728579535, | |
| "loss": 2.4868, | |
| "theoretical_loss": 3.473826429867589, | |
| "tokens_seen": 1723596800 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.0001936589748439569, | |
| "loss": 2.4733, | |
| "theoretical_loss": 3.4732602335737734, | |
| "tokens_seen": 1726873600 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.00019306790240211842, | |
| "loss": 2.4747, | |
| "theoretical_loss": 3.47269541081421, | |
| "tokens_seen": 1730150400 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.00019247682996027994, | |
| "loss": 2.4431, | |
| "theoretical_loss": 3.4721319556649046, | |
| "tokens_seen": 1733427200 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.00019188575751844146, | |
| "loss": 2.4598, | |
| "theoretical_loss": 3.4715698622385487, | |
| "tokens_seen": 1736704000 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.000191294685076603, | |
| "loss": 2.4833, | |
| "theoretical_loss": 3.471009124684226, | |
| "tokens_seen": 1739980800 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.00019070361263476453, | |
| "loss": 2.4769, | |
| "theoretical_loss": 3.470449737187118, | |
| "tokens_seen": 1743257600 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.00019011254019292605, | |
| "loss": 2.525, | |
| "theoretical_loss": 3.4698916939682123, | |
| "tokens_seen": 1746534400 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.0001895214677510876, | |
| "loss": 2.4752, | |
| "theoretical_loss": 3.4693349892840186, | |
| "tokens_seen": 1749811200 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 0.00018893039530924912, | |
| "loss": 2.5101, | |
| "theoretical_loss": 3.4687796174262826, | |
| "tokens_seen": 1753088000 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 0.0001883393228674106, | |
| "loss": 2.4915, | |
| "theoretical_loss": 3.468225572721704, | |
| "tokens_seen": 1756364800 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 0.00018774825042557216, | |
| "loss": 2.4702, | |
| "theoretical_loss": 3.4676728495316587, | |
| "tokens_seen": 1759641600 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 0.00018715717798373368, | |
| "loss": 2.4589, | |
| "theoretical_loss": 3.467121442251922, | |
| "tokens_seen": 1762918400 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 0.0001865661055418952, | |
| "loss": 2.4993, | |
| "theoretical_loss": 3.4665713453123965, | |
| "tokens_seen": 1766195200 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 0.00018597503310005675, | |
| "loss": 2.4633, | |
| "theoretical_loss": 3.46602255317684, | |
| "tokens_seen": 1769472000 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 0.00018538396065821827, | |
| "loss": 2.4303, | |
| "theoretical_loss": 3.465475060342598, | |
| "tokens_seen": 1772748800 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 0.0001847928882163798, | |
| "loss": 2.4376, | |
| "theoretical_loss": 3.4649288613403386, | |
| "tokens_seen": 1776025600 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.00018420181577454134, | |
| "loss": 2.4964, | |
| "theoretical_loss": 3.4643839507337892, | |
| "tokens_seen": 1779302400 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.00018361074333270286, | |
| "loss": 2.4933, | |
| "theoretical_loss": 3.4638403231194763, | |
| "tokens_seen": 1782579200 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.00018301967089086438, | |
| "loss": 2.44, | |
| "theoretical_loss": 3.463297973126469, | |
| "tokens_seen": 1785856000 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.00018242859844902593, | |
| "loss": 2.4002, | |
| "theoretical_loss": 3.462756895416121, | |
| "tokens_seen": 1789132800 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.00018183752600718745, | |
| "loss": 2.4189, | |
| "theoretical_loss": 3.4622170846818205, | |
| "tokens_seen": 1792409600 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.00018124645356534897, | |
| "loss": 2.4505, | |
| "theoretical_loss": 3.4616785356487387, | |
| "tokens_seen": 1795686400 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.00018065538112351052, | |
| "loss": 2.4389, | |
| "theoretical_loss": 3.461141243073582, | |
| "tokens_seen": 1798963200 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "objective/train/docs_used": 944730, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.517244338989258, | |
| "objective/train/theoretical_loss": 3.4606052017443467, | |
| "objective/train/tokens_used": 1822700000, | |
| "theoretical_loss": 3.4606052017443467, | |
| "tokens_seen": 1802240000 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.00018006430868167204, | |
| "loss": 2.4612, | |
| "theoretical_loss": 3.4606052017443467, | |
| "tokens_seen": 1802240000 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.00017947323623983356, | |
| "loss": 2.4456, | |
| "theoretical_loss": 3.4600704064800745, | |
| "tokens_seen": 1805516800 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 0.00017888216379799508, | |
| "loss": 2.4879, | |
| "theoretical_loss": 3.459536852130613, | |
| "tokens_seen": 1808793600 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 0.00017829109135615663, | |
| "loss": 2.4678, | |
| "theoretical_loss": 3.4590045335763775, | |
| "tokens_seen": 1812070400 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 0.00017770001891431815, | |
| "loss": 2.5004, | |
| "theoretical_loss": 3.4584734457281128, | |
| "tokens_seen": 1815347200 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 0.00017710894647247967, | |
| "loss": 2.4577, | |
| "theoretical_loss": 3.457943583526662, | |
| "tokens_seen": 1818624000 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 0.00017651787403064122, | |
| "loss": 2.4999, | |
| "theoretical_loss": 3.45741494194273, | |
| "tokens_seen": 1821900800 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 0.00017592680158880274, | |
| "loss": 2.5114, | |
| "theoretical_loss": 3.4568875159766606, | |
| "tokens_seen": 1825177600 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 0.00017533572914696426, | |
| "loss": 2.4777, | |
| "theoretical_loss": 3.4563613006582017, | |
| "tokens_seen": 1828454400 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 0.0001747446567051258, | |
| "loss": 2.4537, | |
| "theoretical_loss": 3.4558362910462863, | |
| "tokens_seen": 1831731200 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.00017415358426328733, | |
| "loss": 2.5086, | |
| "theoretical_loss": 3.4553124822288055, | |
| "tokens_seen": 1835008000 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.00017356251182144882, | |
| "loss": 2.4897, | |
| "theoretical_loss": 3.4547898693223873, | |
| "tokens_seen": 1838284800 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.00017297143937961037, | |
| "loss": 2.4912, | |
| "theoretical_loss": 3.4542684474721805, | |
| "tokens_seen": 1841561600 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.0001723803669377719, | |
| "loss": 2.4588, | |
| "theoretical_loss": 3.453748211851636, | |
| "tokens_seen": 1844838400 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.0001717892944959334, | |
| "loss": 2.435, | |
| "theoretical_loss": 3.453229157662291, | |
| "tokens_seen": 1848115200 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.00017119822205409496, | |
| "loss": 2.4488, | |
| "theoretical_loss": 3.452711280133558, | |
| "tokens_seen": 1851392000 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.00017060714961225648, | |
| "loss": 2.5002, | |
| "theoretical_loss": 3.4521945745225118, | |
| "tokens_seen": 1854668800 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.000170016077170418, | |
| "loss": 2.4873, | |
| "theoretical_loss": 3.451679036113682, | |
| "tokens_seen": 1857945600 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.00016942500472857955, | |
| "loss": 2.5013, | |
| "theoretical_loss": 3.451164660218848, | |
| "tokens_seen": 1861222400 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 0.00016883393228674107, | |
| "loss": 2.4984, | |
| "theoretical_loss": 3.4506514421768273, | |
| "tokens_seen": 1864499200 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 0.0001682428598449026, | |
| "loss": 2.4879, | |
| "theoretical_loss": 3.45013937735328, | |
| "tokens_seen": 1867776000 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 0.0001676517874030641, | |
| "loss": 2.4779, | |
| "theoretical_loss": 3.4496284611405024, | |
| "tokens_seen": 1871052800 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 0.00016706071496122566, | |
| "loss": 2.5358, | |
| "theoretical_loss": 3.4491186889572303, | |
| "tokens_seen": 1874329600 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 0.00016646964251938718, | |
| "loss": 2.4927, | |
| "theoretical_loss": 3.4486100562484396, | |
| "tokens_seen": 1877606400 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 0.0001658785700775487, | |
| "loss": 2.4922, | |
| "theoretical_loss": 3.448102558485152, | |
| "tokens_seen": 1880883200 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 0.00016528749763571025, | |
| "loss": 2.4792, | |
| "theoretical_loss": 3.447596191164239, | |
| "tokens_seen": 1884160000 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 0.00016469642519387177, | |
| "loss": 2.5167, | |
| "theoretical_loss": 3.447090949808233, | |
| "tokens_seen": 1887436800 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.0001641053527520333, | |
| "loss": 2.5146, | |
| "theoretical_loss": 3.446586829965132, | |
| "tokens_seen": 1890713600 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.00016351428031019484, | |
| "loss": 2.506, | |
| "theoretical_loss": 3.4460838272082173, | |
| "tokens_seen": 1893990400 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.00016292320786835636, | |
| "loss": 2.5007, | |
| "theoretical_loss": 3.4455819371358594, | |
| "tokens_seen": 1897267200 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.00016233213542651788, | |
| "loss": 2.4907, | |
| "theoretical_loss": 3.445081155371338, | |
| "tokens_seen": 1900544000 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.00016174106298467942, | |
| "loss": 2.4986, | |
| "theoretical_loss": 3.4445814775626564, | |
| "tokens_seen": 1903820800 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.00016114999054284095, | |
| "loss": 2.4743, | |
| "theoretical_loss": 3.4440828993823605, | |
| "tokens_seen": 1907097600 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.00016055891810100247, | |
| "loss": 2.5036, | |
| "theoretical_loss": 3.4435854165273567, | |
| "tokens_seen": 1910374400 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.00015996784565916401, | |
| "loss": 2.4968, | |
| "theoretical_loss": 3.443089024718735, | |
| "tokens_seen": 1913651200 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.0001593767732173255, | |
| "loss": 2.4838, | |
| "theoretical_loss": 3.4425937197015912, | |
| "tokens_seen": 1916928000 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 0.0001587975222243238, | |
| "loss": 2.4618, | |
| "theoretical_loss": 3.4420994972448513, | |
| "tokens_seen": 1920204800 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 0.00015820644978248535, | |
| "loss": 2.4683, | |
| "theoretical_loss": 3.4416063531410988, | |
| "tokens_seen": 1923481600 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 0.00015761537734064688, | |
| "loss": 2.4569, | |
| "theoretical_loss": 3.4411142832064003, | |
| "tokens_seen": 1926758400 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 0.0001570243048988084, | |
| "loss": 2.463, | |
| "theoretical_loss": 3.4406232832801376, | |
| "tokens_seen": 1930035200 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 0.00015643323245696994, | |
| "loss": 2.4744, | |
| "theoretical_loss": 3.440133349224835, | |
| "tokens_seen": 1933312000 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 0.00015584216001513147, | |
| "loss": 2.4696, | |
| "theoretical_loss": 3.439644476925995, | |
| "tokens_seen": 1936588800 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 0.00015525108757329299, | |
| "loss": 2.4582, | |
| "theoretical_loss": 3.4391566622919303, | |
| "tokens_seen": 1939865600 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 0.00015468365802912806, | |
| "loss": 2.4682, | |
| "theoretical_loss": 3.438669901253598, | |
| "tokens_seen": 1943142400 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.00015409258558728958, | |
| "loss": 2.4491, | |
| "theoretical_loss": 3.4381841897644385, | |
| "tokens_seen": 1946419200 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.0001535015131454511, | |
| "loss": 2.428, | |
| "theoretical_loss": 3.4376995238002137, | |
| "tokens_seen": 1949696000 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.00015291044070361263, | |
| "loss": 2.445, | |
| "theoretical_loss": 3.4372158993588444, | |
| "tokens_seen": 1952972800 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.00015231936826177417, | |
| "loss": 2.4509, | |
| "theoretical_loss": 3.436733312460253, | |
| "tokens_seen": 1956249600 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.0001517282958199357, | |
| "loss": 2.4591, | |
| "theoretical_loss": 3.436251759146206, | |
| "tokens_seen": 1959526400 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.00015113722337809722, | |
| "loss": 2.4193, | |
| "theoretical_loss": 3.435771235480157, | |
| "tokens_seen": 1962803200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.6358145103933808, | |
| "debugging/distinct-1-grams": 0.7973884824416121, | |
| "debugging/distinct-2-grams": 0.9612689210381616, | |
| "debugging/entropy-1-grams": 6.295285576201537, | |
| "debugging/entropy-2-grams": 7.202343872602045, | |
| "debugging/length": 588.1363636363636, | |
| "debugging/num_segments": 22, | |
| "epoch": 0.7, | |
| "objective/train/docs_used": 1029498, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.5518174171447754, | |
| "objective/train/theoretical_loss": 3.435291737547092, | |
| "objective/train/tokens_used": 1986540000, | |
| "theoretical_loss": 3.435291737547092, | |
| "tokens_seen": 1966080000 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.00015054615093625876, | |
| "loss": 2.4617, | |
| "theoretical_loss": 3.435291737547092, | |
| "tokens_seen": 1966080000 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.00014995507849442028, | |
| "loss": 2.4557, | |
| "theoretical_loss": 3.434813261453378, | |
| "tokens_seen": 1969356800 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.0001493640060525818, | |
| "loss": 2.4401, | |
| "theoretical_loss": 3.434335803326607, | |
| "tokens_seen": 1972633600 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "learning_rate": 0.00014877293361074335, | |
| "loss": 2.4421, | |
| "theoretical_loss": 3.433859359315448, | |
| "tokens_seen": 1975910400 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "learning_rate": 0.00014818186116890487, | |
| "loss": 2.4439, | |
| "theoretical_loss": 3.433383925589498, | |
| "tokens_seen": 1979187200 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "learning_rate": 0.0001475907887270664, | |
| "loss": 2.4225, | |
| "theoretical_loss": 3.432909498339132, | |
| "tokens_seen": 1982464000 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "learning_rate": 0.00014699971628522794, | |
| "loss": 2.448, | |
| "theoretical_loss": 3.432436073775358, | |
| "tokens_seen": 1985740800 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "learning_rate": 0.00014640864384338946, | |
| "loss": 2.473, | |
| "theoretical_loss": 3.4319636481296687, | |
| "tokens_seen": 1989017600 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "learning_rate": 0.00014581757140155096, | |
| "loss": 2.4937, | |
| "theoretical_loss": 3.4314922176539002, | |
| "tokens_seen": 1992294400 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "learning_rate": 0.0001452264989597125, | |
| "loss": 2.4887, | |
| "theoretical_loss": 3.431021778620088, | |
| "tokens_seen": 1995571200 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "learning_rate": 0.00014463542651787403, | |
| "loss": 2.4961, | |
| "theoretical_loss": 3.4305523273203233, | |
| "tokens_seen": 1998848000 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.00014404435407603555, | |
| "loss": 2.5226, | |
| "theoretical_loss": 3.430083860066617, | |
| "tokens_seen": 2002124800 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.0001434532816341971, | |
| "loss": 2.5104, | |
| "theoretical_loss": 3.429616373190753, | |
| "tokens_seen": 2005401600 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.00014286220919235861, | |
| "loss": 2.5127, | |
| "theoretical_loss": 3.4291498630441586, | |
| "tokens_seen": 2008678400 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.00014227113675052014, | |
| "loss": 2.4843, | |
| "theoretical_loss": 3.4286843259977613, | |
| "tokens_seen": 2011955200 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.00014168006430868168, | |
| "loss": 2.4846, | |
| "theoretical_loss": 3.4282197584418546, | |
| "tokens_seen": 2015232000 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.0001410889918668432, | |
| "loss": 2.4995, | |
| "theoretical_loss": 3.427756156785965, | |
| "tokens_seen": 2018508800 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.00014049791942500472, | |
| "loss": 2.4556, | |
| "theoretical_loss": 3.427293517458717, | |
| "tokens_seen": 2021785600 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.00013990684698316625, | |
| "loss": 2.4325, | |
| "theoretical_loss": 3.4268318369076995, | |
| "tokens_seen": 2025062400 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.0001393157745413278, | |
| "loss": 2.4547, | |
| "theoretical_loss": 3.4263711115993383, | |
| "tokens_seen": 2028339200 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.00013872470209948931, | |
| "loss": 2.417, | |
| "theoretical_loss": 3.425911338018763, | |
| "tokens_seen": 2031616000 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.00013813362965765084, | |
| "loss": 2.4276, | |
| "theoretical_loss": 3.425452512669678, | |
| "tokens_seen": 2034892800 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.00013754255721581238, | |
| "loss": 2.4107, | |
| "theoretical_loss": 3.4249946320742364, | |
| "tokens_seen": 2038169600 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.0001369514847739739, | |
| "loss": 2.4422, | |
| "theoretical_loss": 3.4245376927729128, | |
| "tokens_seen": 2041446400 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.00013636041233213542, | |
| "loss": 2.4285, | |
| "theoretical_loss": 3.424081691324375, | |
| "tokens_seen": 2044723200 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.00013576933989029697, | |
| "loss": 2.4423, | |
| "theoretical_loss": 3.4236266243053626, | |
| "tokens_seen": 2048000000 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.0001351782674484585, | |
| "loss": 2.4339, | |
| "theoretical_loss": 3.4231724883105628, | |
| "tokens_seen": 2051276800 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.00013458719500662001, | |
| "loss": 2.4185, | |
| "theoretical_loss": 3.422719279952486, | |
| "tokens_seen": 2054553600 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.00013399612256478156, | |
| "loss": 2.4286, | |
| "theoretical_loss": 3.422266995861345, | |
| "tokens_seen": 2057830400 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 0.00013340505012294308, | |
| "loss": 2.3685, | |
| "theoretical_loss": 3.421815632684937, | |
| "tokens_seen": 2061107200 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 0.0001328139776811046, | |
| "loss": 2.3959, | |
| "theoretical_loss": 3.4213651870885196, | |
| "tokens_seen": 2064384000 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 0.00013222290523926615, | |
| "loss": 2.4091, | |
| "theoretical_loss": 3.4209156557546954, | |
| "tokens_seen": 2067660800 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 0.00013163183279742767, | |
| "loss": 2.4388, | |
| "theoretical_loss": 3.420467035383294, | |
| "tokens_seen": 2070937600 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 0.00013104076035558917, | |
| "loss": 2.4569, | |
| "theoretical_loss": 3.4200193226912554, | |
| "tokens_seen": 2074214400 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 0.00013044968791375071, | |
| "loss": 2.4921, | |
| "theoretical_loss": 3.419572514412513, | |
| "tokens_seen": 2077491200 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 0.00012985861547191223, | |
| "loss": 2.51, | |
| "theoretical_loss": 3.4191266072978816, | |
| "tokens_seen": 2080768000 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 0.00012926754303007376, | |
| "loss": 2.482, | |
| "theoretical_loss": 3.418681598114942, | |
| "tokens_seen": 2084044800 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.00012867647058823528, | |
| "loss": 2.488, | |
| "theoretical_loss": 3.4182374836479266, | |
| "tokens_seen": 2087321600 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.00012808539814639682, | |
| "loss": 2.4943, | |
| "theoretical_loss": 3.4177942606976126, | |
| "tokens_seen": 2090598400 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.00012749432570455834, | |
| "loss": 2.5187, | |
| "theoretical_loss": 3.4173519260812064, | |
| "tokens_seen": 2093875200 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.00012690325326271987, | |
| "loss": 2.4943, | |
| "theoretical_loss": 3.4169104766322356, | |
| "tokens_seen": 2097152000 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.0001263121808208814, | |
| "loss": 2.463, | |
| "theoretical_loss": 3.4164699092004405, | |
| "tokens_seen": 2100428800 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.00012572110837904293, | |
| "loss": 2.457, | |
| "theoretical_loss": 3.416030220651664, | |
| "tokens_seen": 2103705600 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.00012513003593720446, | |
| "loss": 2.4413, | |
| "theoretical_loss": 3.4155914078677467, | |
| "tokens_seen": 2106982400 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.000124538963495366, | |
| "loss": 2.463, | |
| "theoretical_loss": 3.4151534677464195, | |
| "tokens_seen": 2110259200 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.00012394789105352752, | |
| "loss": 2.4897, | |
| "theoretical_loss": 3.414716397201196, | |
| "tokens_seen": 2113536000 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "learning_rate": 0.00012335681861168904, | |
| "loss": 2.4496, | |
| "theoretical_loss": 3.4142801931612725, | |
| "tokens_seen": 2116812800 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "learning_rate": 0.00012276574616985057, | |
| "loss": 2.4572, | |
| "theoretical_loss": 3.41384485257142, | |
| "tokens_seen": 2120089600 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "learning_rate": 0.0001221746737280121, | |
| "loss": 2.4456, | |
| "theoretical_loss": 3.4134103723918834, | |
| "tokens_seen": 2123366400 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "learning_rate": 0.00012158360128617365, | |
| "loss": 2.4428, | |
| "theoretical_loss": 3.412976749598278, | |
| "tokens_seen": 2126643200 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "objective/train/docs_used": 1116355, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.4785542488098145, | |
| "objective/train/theoretical_loss": 3.412543981181491, | |
| "objective/train/tokens_used": 2150380000, | |
| "theoretical_loss": 3.412543981181491, | |
| "tokens_seen": 2129920000 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "learning_rate": 0.00012099252884433517, | |
| "loss": 2.4431, | |
| "theoretical_loss": 3.412543981181491, | |
| "tokens_seen": 2129920000 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "learning_rate": 0.00012040145640249669, | |
| "loss": 2.4237, | |
| "theoretical_loss": 3.412112064147577, | |
| "tokens_seen": 2133196800 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "learning_rate": 0.00011981038396065821, | |
| "loss": 2.4269, | |
| "theoretical_loss": 3.4116809955176617, | |
| "tokens_seen": 2136473600 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "learning_rate": 0.00011921931151881974, | |
| "loss": 2.4261, | |
| "theoretical_loss": 3.4112507723278416, | |
| "tokens_seen": 2139750400 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.00011862823907698128, | |
| "loss": 2.4408, | |
| "theoretical_loss": 3.410821391629086, | |
| "tokens_seen": 2143027200 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.0001180371666351428, | |
| "loss": 2.47, | |
| "theoretical_loss": 3.4103928504871406, | |
| "tokens_seen": 2146304000 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.00011744609419330433, | |
| "loss": 2.4524, | |
| "theoretical_loss": 3.4099651459824294, | |
| "tokens_seen": 2149580800 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.00011685502175146587, | |
| "loss": 2.4462, | |
| "theoretical_loss": 3.409538275209961, | |
| "tokens_seen": 2152857600 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.00011626394930962739, | |
| "loss": 2.5174, | |
| "theoretical_loss": 3.409112235279233, | |
| "tokens_seen": 2156134400 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.00011567287686778892, | |
| "loss": 2.5022, | |
| "theoretical_loss": 3.4086870233141355, | |
| "tokens_seen": 2159411200 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.00011508180442595046, | |
| "loss": 2.4735, | |
| "theoretical_loss": 3.4082626364528625, | |
| "tokens_seen": 2162688000 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.00011449073198411198, | |
| "loss": 2.4847, | |
| "theoretical_loss": 3.4078390718478153, | |
| "tokens_seen": 2165964800 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.00011389965954227351, | |
| "loss": 2.488, | |
| "theoretical_loss": 3.4074163266655115, | |
| "tokens_seen": 2169241600 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "learning_rate": 0.00011330858710043502, | |
| "loss": 2.4826, | |
| "theoretical_loss": 3.4069943980864945, | |
| "tokens_seen": 2172518400 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "learning_rate": 0.00011271751465859655, | |
| "loss": 2.4763, | |
| "theoretical_loss": 3.4065732833052436, | |
| "tokens_seen": 2175795200 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "learning_rate": 0.00011212644221675809, | |
| "loss": 2.4631, | |
| "theoretical_loss": 3.406152979530081, | |
| "tokens_seen": 2179072000 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "learning_rate": 0.00011153536977491961, | |
| "loss": 2.4776, | |
| "theoretical_loss": 3.405733483983088, | |
| "tokens_seen": 2182348800 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "learning_rate": 0.00011094429733308114, | |
| "loss": 2.4251, | |
| "theoretical_loss": 3.4053147939000117, | |
| "tokens_seen": 2185625600 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "learning_rate": 0.00011035322489124268, | |
| "loss": 2.4133, | |
| "theoretical_loss": 3.4048969065301806, | |
| "tokens_seen": 2188902400 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "learning_rate": 0.0001097621524494042, | |
| "loss": 2.4359, | |
| "theoretical_loss": 3.4044798191364163, | |
| "tokens_seen": 2192179200 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "learning_rate": 0.00010917108000756573, | |
| "loss": 2.3925, | |
| "theoretical_loss": 3.4040635289949472, | |
| "tokens_seen": 2195456000 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.00010858000756572725, | |
| "loss": 2.4049, | |
| "theoretical_loss": 3.403648033395324, | |
| "tokens_seen": 2198732800 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.00010798893512388879, | |
| "loss": 2.4003, | |
| "theoretical_loss": 3.403233329640333, | |
| "tokens_seen": 2202009600 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.00010739786268205032, | |
| "loss": 2.3901, | |
| "theoretical_loss": 3.4028194150459137, | |
| "tokens_seen": 2205286400 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.00010680679024021184, | |
| "loss": 2.406, | |
| "theoretical_loss": 3.4024062869410745, | |
| "tokens_seen": 2208563200 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.00010621571779837336, | |
| "loss": 2.4326, | |
| "theoretical_loss": 3.4019939426678096, | |
| "tokens_seen": 2211840000 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.0001056246453565349, | |
| "loss": 2.4203, | |
| "theoretical_loss": 3.401582379581017, | |
| "tokens_seen": 2215116800 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.00010503357291469642, | |
| "loss": 2.4086, | |
| "theoretical_loss": 3.401171595048417, | |
| "tokens_seen": 2218393600 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.00010444250047285795, | |
| "loss": 2.3859, | |
| "theoretical_loss": 3.40076158645047, | |
| "tokens_seen": 2221670400 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.00010385142803101949, | |
| "loss": 2.423, | |
| "theoretical_loss": 3.400352351180298, | |
| "tokens_seen": 2224947200 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 0.00010326035558918101, | |
| "loss": 2.3967, | |
| "theoretical_loss": 3.3999438866436047, | |
| "tokens_seen": 2228224000 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 0.00010266928314734254, | |
| "loss": 2.4206, | |
| "theoretical_loss": 3.399536190258594, | |
| "tokens_seen": 2231500800 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 0.00010207821070550406, | |
| "loss": 2.4364, | |
| "theoretical_loss": 3.399129259455895, | |
| "tokens_seen": 2234777600 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 0.0001014871382636656, | |
| "loss": 2.4146, | |
| "theoretical_loss": 3.3987230916784803, | |
| "tokens_seen": 2238054400 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 0.00010089606582182713, | |
| "loss": 2.4274, | |
| "theoretical_loss": 3.398317684381593, | |
| "tokens_seen": 2241331200 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 0.00010030499337998865, | |
| "loss": 2.4752, | |
| "theoretical_loss": 3.397913035032666, | |
| "tokens_seen": 2244608000 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 9.971392093815019e-05, | |
| "loss": 2.4483, | |
| "theoretical_loss": 3.3975091411112484, | |
| "tokens_seen": 2247884800 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 9.912284849631172e-05, | |
| "loss": 2.4255, | |
| "theoretical_loss": 3.397106000108929, | |
| "tokens_seen": 2251161600 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 9.853177605447323e-05, | |
| "loss": 2.3979, | |
| "theoretical_loss": 3.3967036095292613, | |
| "tokens_seen": 2254438400 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 9.794070361263476e-05, | |
| "loss": 2.4317, | |
| "theoretical_loss": 3.3963019668876897, | |
| "tokens_seen": 2257715200 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 9.73496311707963e-05, | |
| "loss": 2.4463, | |
| "theoretical_loss": 3.395901069711476, | |
| "tokens_seen": 2260992000 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 9.675855872895782e-05, | |
| "loss": 2.4395, | |
| "theoretical_loss": 3.3955009155396247, | |
| "tokens_seen": 2264268800 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 9.616748628711935e-05, | |
| "loss": 2.4744, | |
| "theoretical_loss": 3.3951015019228117, | |
| "tokens_seen": 2267545600 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 9.557641384528087e-05, | |
| "loss": 2.4439, | |
| "theoretical_loss": 3.3947028264233112, | |
| "tokens_seen": 2270822400 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 9.498534140344241e-05, | |
| "loss": 2.4644, | |
| "theoretical_loss": 3.3943048866149255, | |
| "tokens_seen": 2274099200 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 9.439426896160394e-05, | |
| "loss": 2.4711, | |
| "theoretical_loss": 3.3939076800829118, | |
| "tokens_seen": 2277376000 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 9.380319651976546e-05, | |
| "loss": 2.4571, | |
| "theoretical_loss": 3.393511204423914, | |
| "tokens_seen": 2280652800 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 9.3212124077927e-05, | |
| "loss": 2.4423, | |
| "theoretical_loss": 3.3931154572458917, | |
| "tokens_seen": 2283929600 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 9.262105163608853e-05, | |
| "loss": 2.4321, | |
| "theoretical_loss": 3.3927204361680503, | |
| "tokens_seen": 2287206400 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 9.202997919425005e-05, | |
| "loss": 2.4256, | |
| "theoretical_loss": 3.3923261388207733, | |
| "tokens_seen": 2290483200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.5751823723746227, | |
| "debugging/distinct-1-grams": 0.796003660541112, | |
| "debugging/distinct-2-grams": 0.9749584818496995, | |
| "debugging/entropy-1-grams": 5.825073103154919, | |
| "debugging/entropy-2-grams": 6.675736664304104, | |
| "debugging/length": 551.2307692307693, | |
| "debugging/num_segments": 13, | |
| "epoch": 0.82, | |
| "objective/train/docs_used": 1203856, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.4898712635040283, | |
| "objective/train/theoretical_loss": 3.3919325628455534, | |
| "objective/train/tokens_used": 2314220000, | |
| "theoretical_loss": 3.3919325628455534, | |
| "tokens_seen": 2293760000 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 9.143890675241157e-05, | |
| "loss": 2.4448, | |
| "theoretical_loss": 3.3919325628455534, | |
| "tokens_seen": 2293760000 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 9.08478343105731e-05, | |
| "loss": 2.3993, | |
| "theoretical_loss": 3.3915397058949233, | |
| "tokens_seen": 2297036800 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 9.025676186873463e-05, | |
| "loss": 2.4074, | |
| "theoretical_loss": 3.3911475656323904, | |
| "tokens_seen": 2300313600 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 8.966568942689616e-05, | |
| "loss": 2.4156, | |
| "theoretical_loss": 3.3907561397323693, | |
| "tokens_seen": 2303590400 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 8.907461698505768e-05, | |
| "loss": 2.4241, | |
| "theoretical_loss": 3.3903654258801152, | |
| "tokens_seen": 2306867200 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 8.848354454321922e-05, | |
| "loss": 2.4299, | |
| "theoretical_loss": 3.389975421771658, | |
| "tokens_seen": 2310144000 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 8.789247210138075e-05, | |
| "loss": 2.451, | |
| "theoretical_loss": 3.389586125113736, | |
| "tokens_seen": 2313420800 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 8.730139965954227e-05, | |
| "loss": 2.4232, | |
| "theoretical_loss": 3.389197533623734, | |
| "tokens_seen": 2316697600 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 8.671032721770381e-05, | |
| "loss": 2.4304, | |
| "theoretical_loss": 3.3888096450296152, | |
| "tokens_seen": 2319974400 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 8.611925477586534e-05, | |
| "loss": 2.4102, | |
| "theoretical_loss": 3.3884224570698613, | |
| "tokens_seen": 2323251200 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 8.552818233402686e-05, | |
| "loss": 2.4055, | |
| "theoretical_loss": 3.388035967493405, | |
| "tokens_seen": 2326528000 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 8.49371098921884e-05, | |
| "loss": 2.4286, | |
| "theoretical_loss": 3.387650174059569, | |
| "tokens_seen": 2329804800 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 8.43460374503499e-05, | |
| "loss": 2.4148, | |
| "theoretical_loss": 3.3872650745380044, | |
| "tokens_seen": 2333081600 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 8.375496500851144e-05, | |
| "loss": 2.4489, | |
| "theoretical_loss": 3.3868806667086275, | |
| "tokens_seen": 2336358400 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 8.316389256667297e-05, | |
| "loss": 2.4279, | |
| "theoretical_loss": 3.3864969483615575, | |
| "tokens_seen": 2339635200 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 8.25728201248345e-05, | |
| "loss": 2.4401, | |
| "theoretical_loss": 3.3861139172970582, | |
| "tokens_seen": 2342912000 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 8.198174768299603e-05, | |
| "loss": 2.4289, | |
| "theoretical_loss": 3.385731571325474, | |
| "tokens_seen": 2346188800 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 8.139067524115756e-05, | |
| "loss": 2.4212, | |
| "theoretical_loss": 3.385349908267171, | |
| "tokens_seen": 2349465600 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 8.079960279931908e-05, | |
| "loss": 2.4094, | |
| "theoretical_loss": 3.3849689259524784, | |
| "tokens_seen": 2352742400 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 8.020853035748062e-05, | |
| "loss": 2.4289, | |
| "theoretical_loss": 3.3845886222216275, | |
| "tokens_seen": 2356019200 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 7.961745791564215e-05, | |
| "loss": 2.4385, | |
| "theoretical_loss": 3.384208994924694, | |
| "tokens_seen": 2359296000 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 7.902638547380367e-05, | |
| "loss": 2.4133, | |
| "theoretical_loss": 3.3838300419215384, | |
| "tokens_seen": 2362572800 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 7.843531303196521e-05, | |
| "loss": 2.4363, | |
| "theoretical_loss": 3.383451761081749, | |
| "tokens_seen": 2365849600 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "learning_rate": 7.784424059012673e-05, | |
| "loss": 2.4336, | |
| "theoretical_loss": 3.383074150284584, | |
| "tokens_seen": 2369126400 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "learning_rate": 7.725316814828825e-05, | |
| "loss": 2.4655, | |
| "theoretical_loss": 3.3826972074189134, | |
| "tokens_seen": 2372403200 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "learning_rate": 7.667391715528656e-05, | |
| "loss": 2.4284, | |
| "theoretical_loss": 3.3823209303831634, | |
| "tokens_seen": 2375680000 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "learning_rate": 7.608284471344808e-05, | |
| "loss": 2.468, | |
| "theoretical_loss": 3.3819453170852602, | |
| "tokens_seen": 2378956800 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "learning_rate": 7.549177227160962e-05, | |
| "loss": 2.5631, | |
| "theoretical_loss": 3.381570365442573, | |
| "tokens_seen": 2382233600 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "learning_rate": 7.490069982977114e-05, | |
| "loss": 2.6305, | |
| "theoretical_loss": 3.3811960733818585, | |
| "tokens_seen": 2385510400 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "learning_rate": 7.430962738793266e-05, | |
| "loss": 2.6647, | |
| "theoretical_loss": 3.380822438839207, | |
| "tokens_seen": 2388787200 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "learning_rate": 7.371855494609419e-05, | |
| "loss": 2.6687, | |
| "theoretical_loss": 3.380449459759986, | |
| "tokens_seen": 2392064000 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 7.312748250425573e-05, | |
| "loss": 2.6164, | |
| "theoretical_loss": 3.380077134098787, | |
| "tokens_seen": 2395340800 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 7.253641006241725e-05, | |
| "loss": 2.6181, | |
| "theoretical_loss": 3.3797054598193723, | |
| "tokens_seen": 2398617600 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 7.194533762057878e-05, | |
| "loss": 2.5822, | |
| "theoretical_loss": 3.379334434894619, | |
| "tokens_seen": 2401894400 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 7.13542651787403e-05, | |
| "loss": 2.5412, | |
| "theoretical_loss": 3.378964057306467, | |
| "tokens_seen": 2405171200 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 7.076319273690184e-05, | |
| "loss": 2.5279, | |
| "theoretical_loss": 3.378594325045867, | |
| "tokens_seen": 2408448000 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 7.017212029506337e-05, | |
| "loss": 2.4702, | |
| "theoretical_loss": 3.378225236112729, | |
| "tokens_seen": 2411724800 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 6.958104785322489e-05, | |
| "loss": 2.5029, | |
| "theoretical_loss": 3.377856788515865, | |
| "tokens_seen": 2415001600 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 6.898997541138643e-05, | |
| "loss": 2.4748, | |
| "theoretical_loss": 3.3774889802729446, | |
| "tokens_seen": 2418278400 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 6.839890296954796e-05, | |
| "loss": 2.4242, | |
| "theoretical_loss": 3.377121809410438, | |
| "tokens_seen": 2421555200 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "learning_rate": 6.780783052770947e-05, | |
| "loss": 2.4535, | |
| "theoretical_loss": 3.3767552739635684, | |
| "tokens_seen": 2424832000 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "learning_rate": 6.7216758085871e-05, | |
| "loss": 2.4235, | |
| "theoretical_loss": 3.37638937197626, | |
| "tokens_seen": 2428108800 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "learning_rate": 6.662568564403252e-05, | |
| "loss": 2.4109, | |
| "theoretical_loss": 3.3760241015010872, | |
| "tokens_seen": 2431385600 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "learning_rate": 6.603461320219406e-05, | |
| "loss": 2.4288, | |
| "theoretical_loss": 3.375659460599226, | |
| "tokens_seen": 2434662400 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "learning_rate": 6.544354076035559e-05, | |
| "loss": 2.4338, | |
| "theoretical_loss": 3.375295447340404, | |
| "tokens_seen": 2437939200 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "learning_rate": 6.485246831851711e-05, | |
| "loss": 2.4174, | |
| "theoretical_loss": 3.3749320598028527, | |
| "tokens_seen": 2441216000 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "learning_rate": 6.426139587667865e-05, | |
| "loss": 2.4269, | |
| "theoretical_loss": 3.3745692960732545, | |
| "tokens_seen": 2444492800 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "learning_rate": 6.367032343484018e-05, | |
| "loss": 2.4221, | |
| "theoretical_loss": 3.374207154246699, | |
| "tokens_seen": 2447769600 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 6.30792509930017e-05, | |
| "loss": 2.388, | |
| "theoretical_loss": 3.373845632426633, | |
| "tokens_seen": 2451046400 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 6.248817855116324e-05, | |
| "loss": 2.4176, | |
| "theoretical_loss": 3.373484728724812, | |
| "tokens_seen": 2454323200 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "objective/train/docs_used": 1283757, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.537339210510254, | |
| "objective/train/theoretical_loss": 3.3731244412612527, | |
| "objective/train/tokens_used": 2478060000, | |
| "theoretical_loss": 3.3731244412612527, | |
| "tokens_seen": 2457600000 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 6.189710610932476e-05, | |
| "loss": 2.4178, | |
| "theoretical_loss": 3.3731244412612527, | |
| "tokens_seen": 2457600000 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 6.130603366748629e-05, | |
| "loss": 2.4165, | |
| "theoretical_loss": 3.3727647681641884, | |
| "tokens_seen": 2460876800 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 6.071496122564782e-05, | |
| "loss": 2.4275, | |
| "theoretical_loss": 3.3724057075700182, | |
| "tokens_seen": 2464153600 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 6.012388878380935e-05, | |
| "loss": 2.4218, | |
| "theoretical_loss": 3.372047257623265, | |
| "tokens_seen": 2467430400 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 5.9532816341970874e-05, | |
| "loss": 2.41, | |
| "theoretical_loss": 3.3716894164765256, | |
| "tokens_seen": 2470707200 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 5.8941743900132395e-05, | |
| "loss": 2.4097, | |
| "theoretical_loss": 3.371332182290427, | |
| "tokens_seen": 2473984000 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 5.835067145829393e-05, | |
| "loss": 2.4489, | |
| "theoretical_loss": 3.37097555323358, | |
| "tokens_seen": 2477260800 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "learning_rate": 5.775959901645546e-05, | |
| "loss": 2.4814, | |
| "theoretical_loss": 3.370619527482535, | |
| "tokens_seen": 2480537600 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "learning_rate": 5.7168526574616985e-05, | |
| "loss": 2.5471, | |
| "theoretical_loss": 3.370264103221736, | |
| "tokens_seen": 2483814400 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "learning_rate": 5.657745413277852e-05, | |
| "loss": 2.6286, | |
| "theoretical_loss": 3.369909278643477, | |
| "tokens_seen": 2487091200 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "learning_rate": 5.5986381690940047e-05, | |
| "loss": 2.5661, | |
| "theoretical_loss": 3.369555051947857, | |
| "tokens_seen": 2490368000 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "learning_rate": 5.539530924910157e-05, | |
| "loss": 2.5628, | |
| "theoretical_loss": 3.369201421342737, | |
| "tokens_seen": 2493644800 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "learning_rate": 5.4804236807263095e-05, | |
| "loss": 2.5182, | |
| "theoretical_loss": 3.3688483850436945, | |
| "tokens_seen": 2496921600 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "learning_rate": 5.421316436542463e-05, | |
| "loss": 2.4975, | |
| "theoretical_loss": 3.368495941273984, | |
| "tokens_seen": 2500198400 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "learning_rate": 5.362209192358616e-05, | |
| "loss": 2.4749, | |
| "theoretical_loss": 3.3681440882644886, | |
| "tokens_seen": 2503475200 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 5.3031019481747684e-05, | |
| "loss": 2.4717, | |
| "theoretical_loss": 3.3677928242536828, | |
| "tokens_seen": 2506752000 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 5.243994703990921e-05, | |
| "loss": 2.4832, | |
| "theoretical_loss": 3.3674421474875853, | |
| "tokens_seen": 2510028800 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 5.1848874598070746e-05, | |
| "loss": 2.4656, | |
| "theoretical_loss": 3.367092056219721, | |
| "tokens_seen": 2513305600 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 5.126962360506904e-05, | |
| "loss": 2.4354, | |
| "theoretical_loss": 3.3667425487110756, | |
| "tokens_seen": 2516582400 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 5.0678551163230566e-05, | |
| "loss": 2.4233, | |
| "theoretical_loss": 3.366393623230057, | |
| "tokens_seen": 2519859200 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 5.00874787213921e-05, | |
| "loss": 2.4454, | |
| "theoretical_loss": 3.366045278052451, | |
| "tokens_seen": 2523136000 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 4.949640627955362e-05, | |
| "loss": 2.4168, | |
| "theoretical_loss": 3.365697511461384, | |
| "tokens_seen": 2526412800 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 4.890533383771515e-05, | |
| "loss": 2.4534, | |
| "theoretical_loss": 3.365350321747279, | |
| "tokens_seen": 2529689600 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 4.8314261395876676e-05, | |
| "loss": 2.4315, | |
| "theoretical_loss": 3.365003707207816, | |
| "tokens_seen": 2532966400 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "learning_rate": 4.772318895403821e-05, | |
| "loss": 2.4742, | |
| "theoretical_loss": 3.3646576661478944, | |
| "tokens_seen": 2536243200 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "learning_rate": 4.713211651219974e-05, | |
| "loss": 2.445, | |
| "theoretical_loss": 3.3643121968795895, | |
| "tokens_seen": 2539520000 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "learning_rate": 4.6541044070361266e-05, | |
| "loss": 2.4477, | |
| "theoretical_loss": 3.3639672977221142, | |
| "tokens_seen": 2542796800 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "learning_rate": 4.594997162852279e-05, | |
| "loss": 2.4849, | |
| "theoretical_loss": 3.3636229670017808, | |
| "tokens_seen": 2546073600 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "learning_rate": 4.535889918668432e-05, | |
| "loss": 2.4969, | |
| "theoretical_loss": 3.3632792030519605, | |
| "tokens_seen": 2549350400 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "learning_rate": 4.476782674484585e-05, | |
| "loss": 2.4901, | |
| "theoretical_loss": 3.362936004213045, | |
| "tokens_seen": 2552627200 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "learning_rate": 4.4176754303007376e-05, | |
| "loss": 2.4372, | |
| "theoretical_loss": 3.362593368832409, | |
| "tokens_seen": 2555904000 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "learning_rate": 4.358568186116891e-05, | |
| "loss": 2.4694, | |
| "theoretical_loss": 3.36225129526437, | |
| "tokens_seen": 2559180800 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 4.299460941933044e-05, | |
| "loss": 2.4687, | |
| "theoretical_loss": 3.3619097818701515, | |
| "tokens_seen": 2562457600 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 4.240353697749196e-05, | |
| "loss": 2.4342, | |
| "theoretical_loss": 3.3615688270178454, | |
| "tokens_seen": 2565734400 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 4.1812464535653486e-05, | |
| "loss": 2.4402, | |
| "theoretical_loss": 3.3612284290823737, | |
| "tokens_seen": 2569011200 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 4.122139209381502e-05, | |
| "loss": 2.4133, | |
| "theoretical_loss": 3.3608885864454523, | |
| "tokens_seen": 2572288000 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 4.063031965197655e-05, | |
| "loss": 2.5042, | |
| "theoretical_loss": 3.360549297495553, | |
| "tokens_seen": 2575564800 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 4.0039247210138076e-05, | |
| "loss": 2.4613, | |
| "theoretical_loss": 3.3602105606278663, | |
| "tokens_seen": 2578841600 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 3.9448174768299603e-05, | |
| "loss": 2.4788, | |
| "theoretical_loss": 3.359872374244266, | |
| "tokens_seen": 2582118400 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 3.885710232646113e-05, | |
| "loss": 2.4714, | |
| "theoretical_loss": 3.3595347367532744, | |
| "tokens_seen": 2585395200 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 3.826602988462266e-05, | |
| "loss": 2.4494, | |
| "theoretical_loss": 3.359197646570022, | |
| "tokens_seen": 2588672000 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 3.7674957442784186e-05, | |
| "loss": 2.4483, | |
| "theoretical_loss": 3.3588611021162156, | |
| "tokens_seen": 2591948800 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 3.7083885000945714e-05, | |
| "loss": 2.4542, | |
| "theoretical_loss": 3.3585251018201006, | |
| "tokens_seen": 2595225600 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 3.649281255910725e-05, | |
| "loss": 2.4252, | |
| "theoretical_loss": 3.358189644116427, | |
| "tokens_seen": 2598502400 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 3.5901740117268776e-05, | |
| "loss": 2.4209, | |
| "theoretical_loss": 3.3578547274464134, | |
| "tokens_seen": 2601779200 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 3.5310667675430296e-05, | |
| "loss": 2.4062, | |
| "theoretical_loss": 3.357520350257713, | |
| "tokens_seen": 2605056000 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 3.471959523359183e-05, | |
| "loss": 2.3822, | |
| "theoretical_loss": 3.357186511004378, | |
| "tokens_seen": 2608332800 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 3.412852279175336e-05, | |
| "loss": 2.429, | |
| "theoretical_loss": 3.356853208146826, | |
| "tokens_seen": 2611609600 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 3.3537450349914886e-05, | |
| "loss": 2.4334, | |
| "theoretical_loss": 3.356520440151805, | |
| "tokens_seen": 2614886400 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 3.2946377908076413e-05, | |
| "loss": 2.4384, | |
| "theoretical_loss": 3.3561882054923613, | |
| "tokens_seen": 2618163200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.6491709479476812, | |
| "debugging/distinct-1-grams": 0.7803105142159686, | |
| "debugging/distinct-2-grams": 0.9531070625455311, | |
| "debugging/entropy-1-grams": 6.2449327748027414, | |
| "debugging/entropy-2-grams": 7.178441337028067, | |
| "debugging/length": 599.6363636363636, | |
| "debugging/num_segments": 22, | |
| "epoch": 0.94, | |
| "objective/train/docs_used": 1365303, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.3327574729919434, | |
| "objective/train/theoretical_loss": 3.3558565026478027, | |
| "objective/train/tokens_used": 2641900000, | |
| "theoretical_loss": 3.3558565026478027, | |
| "tokens_seen": 2621440000 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 3.235530546623795e-05, | |
| "loss": 2.4403, | |
| "theoretical_loss": 3.3558565026478027, | |
| "tokens_seen": 2621440000 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 3.1764233024399475e-05, | |
| "loss": 2.4274, | |
| "theoretical_loss": 3.3555253301036685, | |
| "tokens_seen": 2624716800 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 3.1173160582560996e-05, | |
| "loss": 2.405, | |
| "theoretical_loss": 3.3551946863516937, | |
| "tokens_seen": 2627993600 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 3.058208814072253e-05, | |
| "loss": 2.4075, | |
| "theoretical_loss": 3.3548645698897777, | |
| "tokens_seen": 2631270400 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 2.9991015698884055e-05, | |
| "loss": 2.3794, | |
| "theoretical_loss": 3.3545349792219494, | |
| "tokens_seen": 2634547200 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 2.9399943257045586e-05, | |
| "loss": 2.3998, | |
| "theoretical_loss": 3.354205912858337, | |
| "tokens_seen": 2637824000 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 2.880887081520711e-05, | |
| "loss": 2.4031, | |
| "theoretical_loss": 3.353877369315133, | |
| "tokens_seen": 2641100800 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 2.821779837336864e-05, | |
| "loss": 2.4198, | |
| "theoretical_loss": 3.353549347114565, | |
| "tokens_seen": 2644377600 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 2.762672593153017e-05, | |
| "loss": 2.4331, | |
| "theoretical_loss": 3.3532218447848616, | |
| "tokens_seen": 2647654400 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 2.7035653489691696e-05, | |
| "loss": 2.4011, | |
| "theoretical_loss": 3.352894860860221, | |
| "tokens_seen": 2650931200 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 2.6444581047853227e-05, | |
| "loss": 2.4215, | |
| "theoretical_loss": 3.352568393880779, | |
| "tokens_seen": 2654208000 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 2.5853508606014754e-05, | |
| "loss": 2.4038, | |
| "theoretical_loss": 3.3522424423925794, | |
| "tokens_seen": 2657484800 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 2.5262436164176282e-05, | |
| "loss": 2.4491, | |
| "theoretical_loss": 3.3519170049475413, | |
| "tokens_seen": 2660761600 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 2.467136372233781e-05, | |
| "loss": 2.4333, | |
| "theoretical_loss": 3.351592080103429, | |
| "tokens_seen": 2664038400 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 2.408029128049934e-05, | |
| "loss": 2.4016, | |
| "theoretical_loss": 3.3512676664238192, | |
| "tokens_seen": 2667315200 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 2.3489218838660865e-05, | |
| "loss": 2.4081, | |
| "theoretical_loss": 3.350943762478075, | |
| "tokens_seen": 2670592000 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 2.2898146396822396e-05, | |
| "loss": 2.4127, | |
| "theoretical_loss": 3.3506203668413095, | |
| "tokens_seen": 2673868800 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 2.2307073954983923e-05, | |
| "loss": 2.4085, | |
| "theoretical_loss": 3.3502974780943613, | |
| "tokens_seen": 2677145600 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 2.171600151314545e-05, | |
| "loss": 2.404, | |
| "theoretical_loss": 3.34997509482376, | |
| "tokens_seen": 2680422400 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 2.112492907130698e-05, | |
| "loss": 2.392, | |
| "theoretical_loss": 3.3496532156217005, | |
| "tokens_seen": 2683699200 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 2.053385662946851e-05, | |
| "loss": 2.3624, | |
| "theoretical_loss": 3.34933183908601, | |
| "tokens_seen": 2686976000 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 1.9942784187630033e-05, | |
| "loss": 2.3477, | |
| "theoretical_loss": 3.3490109638201195, | |
| "tokens_seen": 2690252800 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 1.9351711745791564e-05, | |
| "loss": 2.3776, | |
| "theoretical_loss": 3.348690588433038, | |
| "tokens_seen": 2693529600 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 1.8760639303953092e-05, | |
| "loss": 2.3969, | |
| "theoretical_loss": 3.3483707115393164, | |
| "tokens_seen": 2696806400 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 1.816956686211462e-05, | |
| "loss": 2.3887, | |
| "theoretical_loss": 3.3480513317590264, | |
| "tokens_seen": 2700083200 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 1.757849442027615e-05, | |
| "loss": 2.4019, | |
| "theoretical_loss": 3.347732447717727, | |
| "tokens_seen": 2703360000 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 1.6987421978437678e-05, | |
| "loss": 2.4098, | |
| "theoretical_loss": 3.347414058046437, | |
| "tokens_seen": 2706636800 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 1.639634953659921e-05, | |
| "loss": 2.3954, | |
| "theoretical_loss": 3.3470961613816086, | |
| "tokens_seen": 2709913600 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 1.5805277094760733e-05, | |
| "loss": 2.4158, | |
| "theoretical_loss": 3.3467787563650955, | |
| "tokens_seen": 2713190400 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 1.5214204652922262e-05, | |
| "loss": 2.4098, | |
| "theoretical_loss": 3.34646184164413, | |
| "tokens_seen": 2716467200 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 1.462313221108379e-05, | |
| "loss": 2.3758, | |
| "theoretical_loss": 3.346145415871291, | |
| "tokens_seen": 2719744000 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 1.4032059769245318e-05, | |
| "loss": 2.4073, | |
| "theoretical_loss": 3.345829477704478, | |
| "tokens_seen": 2723020800 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 1.3440987327406847e-05, | |
| "loss": 2.2802, | |
| "theoretical_loss": 3.345514025806884, | |
| "tokens_seen": 2726297600 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 1.2849914885568376e-05, | |
| "loss": 2.3309, | |
| "theoretical_loss": 3.3451990588469687, | |
| "tokens_seen": 2729574400 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 1.2258842443729904e-05, | |
| "loss": 2.3645, | |
| "theoretical_loss": 3.3448845754984307, | |
| "tokens_seen": 2732851200 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 1.1667770001891433e-05, | |
| "loss": 2.3476, | |
| "theoretical_loss": 3.344570574440179, | |
| "tokens_seen": 2736128000 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 1.107669756005296e-05, | |
| "loss": 2.3579, | |
| "theoretical_loss": 3.3442570543563104, | |
| "tokens_seen": 2739404800 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 1.0485625118214488e-05, | |
| "loss": 2.3574, | |
| "theoretical_loss": 3.34394401393608, | |
| "tokens_seen": 2742681600 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 9.894552676376017e-06, | |
| "loss": 2.3552, | |
| "theoretical_loss": 3.343631451873874, | |
| "tokens_seen": 2745958400 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 9.303480234537545e-06, | |
| "loss": 2.369, | |
| "theoretical_loss": 3.343319366869187, | |
| "tokens_seen": 2749235200 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 8.712407792699074e-06, | |
| "loss": 2.3748, | |
| "theoretical_loss": 3.343007757626593, | |
| "tokens_seen": 2752512000 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 8.121335350860602e-06, | |
| "loss": 2.3482, | |
| "theoretical_loss": 3.34269662285572, | |
| "tokens_seen": 2755788800 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 7.530262909022129e-06, | |
| "loss": 2.3381, | |
| "theoretical_loss": 3.342385961271226, | |
| "tokens_seen": 2759065600 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 6.939190467183658e-06, | |
| "loss": 2.3855, | |
| "theoretical_loss": 3.3420757715927705, | |
| "tokens_seen": 2762342400 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 6.348118025345187e-06, | |
| "loss": 2.3993, | |
| "theoretical_loss": 3.341766052544993, | |
| "tokens_seen": 2765619200 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 5.757045583506715e-06, | |
| "loss": 2.3942, | |
| "theoretical_loss": 3.341456802857484, | |
| "tokens_seen": 2768896000 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 5.165973141668243e-06, | |
| "loss": 2.3254, | |
| "theoretical_loss": 3.341148021264762, | |
| "tokens_seen": 2772172800 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 4.574900699829771e-06, | |
| "loss": 2.3807, | |
| "theoretical_loss": 3.3408397065062485, | |
| "tokens_seen": 2775449600 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 3.9838282579913e-06, | |
| "loss": 2.3766, | |
| "theoretical_loss": 3.3405318573262424, | |
| "tokens_seen": 2778726400 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 3.4045772649895976e-06, | |
| "loss": 2.3754, | |
| "theoretical_loss": 3.3402244724738965, | |
| "tokens_seen": 2782003200 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "objective/train/docs_used": 1451838, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.3760557174682617, | |
| "objective/train/theoretical_loss": 3.3399175507031913, | |
| "objective/train/tokens_used": 2805740000, | |
| "theoretical_loss": 3.3399175507031913, | |
| "tokens_seen": 2785280000 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 2.8135048231511256e-06, | |
| "loss": 2.3865, | |
| "theoretical_loss": 3.3399175507031913, | |
| "tokens_seen": 2785280000 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "learning_rate": 2.2224323813126536e-06, | |
| "loss": 2.3635, | |
| "theoretical_loss": 3.3396110907729133, | |
| "tokens_seen": 2788556800 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "learning_rate": 1.631359939474182e-06, | |
| "loss": 2.3786, | |
| "theoretical_loss": 3.3393050914466285, | |
| "tokens_seen": 2791833600 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "learning_rate": 1.0402874976357102e-06, | |
| "loss": 2.3985, | |
| "theoretical_loss": 3.33899955149266, | |
| "tokens_seen": 2795110400 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "learning_rate": 4.4921505579723853e-07, | |
| "loss": 2.3652, | |
| "theoretical_loss": 3.338694469684062, | |
| "tokens_seen": 2798387200 | |
| } | |
| ], | |
| "max_steps": 42724, | |
| "num_train_epochs": 9223372036854775807, | |
| "total_flos": 1.428920419221504e+18, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |