| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.5, | |
| "global_step": 21362, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 1.1682242990654204e-06, | |
| "loss": 10.8708, | |
| "theoretical_loss": 20.81281176760504, | |
| "tokens_seen": 65536 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 5.841121495327103e-05, | |
| "loss": 8.9956, | |
| "theoretical_loss": 8.563476630668958, | |
| "tokens_seen": 3276800 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00011682242990654206, | |
| "loss": 6.8219, | |
| "theoretical_loss": 7.477752684105921, | |
| "tokens_seen": 6553600 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00017523364485981307, | |
| "loss": 5.9071, | |
| "theoretical_loss": 6.9337484549527915, | |
| "tokens_seen": 9830400 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00023364485981308412, | |
| "loss": 5.53, | |
| "theoretical_loss": 6.5835601944843045, | |
| "tokens_seen": 13107200 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00029205607476635517, | |
| "loss": 5.2608, | |
| "theoretical_loss": 6.3307075311739744, | |
| "tokens_seen": 16384000 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00035046728971962614, | |
| "loss": 5.0625, | |
| "theoretical_loss": 6.135523197998216, | |
| "tokens_seen": 19660800 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004088785046728972, | |
| "loss": 4.9312, | |
| "theoretical_loss": 5.978095549927499, | |
| "tokens_seen": 22937600 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00046728971962616824, | |
| "loss": 4.8328, | |
| "theoretical_loss": 5.847111292323815, | |
| "tokens_seen": 26214400 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000499739928125591, | |
| "loss": 4.7038, | |
| "theoretical_loss": 5.735570781940016, | |
| "tokens_seen": 29491200 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004991488556837526, | |
| "loss": 4.5907, | |
| "theoretical_loss": 5.638864110129244, | |
| "tokens_seen": 32768000 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004985577832419141, | |
| "loss": 4.4397, | |
| "theoretical_loss": 5.553806347902798, | |
| "tokens_seen": 36044800 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004979667108000757, | |
| "loss": 4.3365, | |
| "theoretical_loss": 5.478112046614329, | |
| "tokens_seen": 39321600 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004973756383582371, | |
| "loss": 4.2684, | |
| "theoretical_loss": 5.410089925637252, | |
| "tokens_seen": 42598400 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004967845659163987, | |
| "loss": 4.1903, | |
| "theoretical_loss": 5.348456049793725, | |
| "tokens_seen": 45875200 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004961934934745603, | |
| "loss": 4.1478, | |
| "theoretical_loss": 5.292214532995457, | |
| "tokens_seen": 49152000 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004956024210327218, | |
| "loss": 4.0707, | |
| "theoretical_loss": 5.240578591827869, | |
| "tokens_seen": 52428800 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004950113485908833, | |
| "loss": 3.9832, | |
| "theoretical_loss": 5.192916690583679, | |
| "tokens_seen": 55705600 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004944202761490448, | |
| "loss": 3.9481, | |
| "theoretical_loss": 5.148714829414331, | |
| "tokens_seen": 58982400 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004938292037072064, | |
| "loss": 3.8208, | |
| "theoretical_loss": 5.107549528462992, | |
| "tokens_seen": 62259200 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004932381312653678, | |
| "loss": 3.705, | |
| "theoretical_loss": 5.069068083201136, | |
| "tokens_seen": 65536000 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004926470588235294, | |
| "loss": 3.6661, | |
| "theoretical_loss": 5.032973875895897, | |
| "tokens_seen": 68812800 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.000492055986381691, | |
| "loss": 3.5935, | |
| "theoretical_loss": 4.999015274282555, | |
| "tokens_seen": 72089600 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004914649139398525, | |
| "loss": 3.5328, | |
| "theoretical_loss": 4.966977121409853, | |
| "tokens_seen": 75366400 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.000490873841498014, | |
| "loss": 3.4939, | |
| "theoretical_loss": 4.936674127683027, | |
| "tokens_seen": 78643200 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004902827690561755, | |
| "loss": 3.4581, | |
| "theoretical_loss": 4.907945679887972, | |
| "tokens_seen": 81920000 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004896916966143371, | |
| "loss": 3.4217, | |
| "theoretical_loss": 4.8806517198708175, | |
| "tokens_seen": 85196800 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004891006241724985, | |
| "loss": 3.4207, | |
| "theoretical_loss": 4.85466944053967, | |
| "tokens_seen": 88473600 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004885095517306601, | |
| "loss": 3.3716, | |
| "theoretical_loss": 4.829890613366704, | |
| "tokens_seen": 91750400 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004879184792888217, | |
| "loss": 3.3598, | |
| "theoretical_loss": 4.806219408835812, | |
| "tokens_seen": 95027200 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004873274068469832, | |
| "loss": 3.2831, | |
| "theoretical_loss": 4.783570605334148, | |
| "tokens_seen": 98304000 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.00048673633440514467, | |
| "loss": 3.2658, | |
| "theoretical_loss": 4.761868106830299, | |
| "tokens_seen": 101580800 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004861452619633062, | |
| "loss": 3.315, | |
| "theoretical_loss": 4.741043708020364, | |
| "tokens_seen": 104857600 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004855541895214677, | |
| "loss": 3.2891, | |
| "theoretical_loss": 4.721036059306941, | |
| "tokens_seen": 108134400 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004849631170796293, | |
| "loss": 3.2322, | |
| "theoretical_loss": 4.701789794289756, | |
| "tokens_seen": 111411200 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004843720446377908, | |
| "loss": 3.2645, | |
| "theoretical_loss": 4.68325479029382, | |
| "tokens_seen": 114688000 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.00048378097219595233, | |
| "loss": 3.263, | |
| "theoretical_loss": 4.6653855384841725, | |
| "tokens_seen": 117964800 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.00048318989975411385, | |
| "loss": 3.2297, | |
| "theoretical_loss": 4.6481406047776295, | |
| "tokens_seen": 121241600 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.00048259882731227537, | |
| "loss": 3.2348, | |
| "theoretical_loss": 4.631482166397534, | |
| "tokens_seen": 124518400 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0004820077548704369, | |
| "loss": 3.1816, | |
| "theoretical_loss": 4.615375611773608, | |
| "tokens_seen": 127795200 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00048141668242859847, | |
| "loss": 3.2168, | |
| "theoretical_loss": 4.5997891937483955, | |
| "tokens_seen": 131072000 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00048082560998676, | |
| "loss": 3.1731, | |
| "theoretical_loss": 4.584693727850565, | |
| "tokens_seen": 134348800 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0004802345375449215, | |
| "loss": 3.2125, | |
| "theoretical_loss": 4.570062328836407, | |
| "tokens_seen": 137625600 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00047964346510308303, | |
| "loss": 3.1873, | |
| "theoretical_loss": 4.5558701798619285, | |
| "tokens_seen": 140902400 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00047905239266124455, | |
| "loss": 3.1537, | |
| "theoretical_loss": 4.542094329588689, | |
| "tokens_seen": 144179200 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00047846132021940607, | |
| "loss": 3.1768, | |
| "theoretical_loss": 4.528713513292708, | |
| "tokens_seen": 147456000 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0004778702477775676, | |
| "loss": 3.1132, | |
| "theoretical_loss": 4.515707994672887, | |
| "tokens_seen": 150732800 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047727917533572917, | |
| "loss": 3.1237, | |
| "theoretical_loss": 4.503059425571229, | |
| "tokens_seen": 154009600 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0004766881028938907, | |
| "loss": 3.12, | |
| "theoretical_loss": 4.490750721243157, | |
| "tokens_seen": 157286400 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0004760970304520522, | |
| "loss": 3.0787, | |
| "theoretical_loss": 4.478765949169858, | |
| "tokens_seen": 160563200 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "objective/train/docs_used": 96807, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 3.022380828857422, | |
| "objective/train/theoretical_loss": 4.46709022969911, | |
| "objective/train/tokens_used": 184300000, | |
| "theoretical_loss": 4.46709022969911, | |
| "tokens_seen": 163840000 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047550595801021373, | |
| "loss": 3.1056, | |
| "theoretical_loss": 4.46709022969911, | |
| "tokens_seen": 163840000 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047491488556837525, | |
| "loss": 3.1107, | |
| "theoretical_loss": 4.455709647047437, | |
| "tokens_seen": 167116800 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047432381312653677, | |
| "loss": 3.1133, | |
| "theoretical_loss": 4.444611169403321, | |
| "tokens_seen": 170393600 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047373274068469835, | |
| "loss": 3.0543, | |
| "theoretical_loss": 4.4337825770455375, | |
| "tokens_seen": 173670400 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047314166824285987, | |
| "loss": 3.0612, | |
| "theoretical_loss": 4.423212397538051, | |
| "tokens_seen": 176947200 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0004725505958010214, | |
| "loss": 3.0913, | |
| "theoretical_loss": 4.4128898471880325, | |
| "tokens_seen": 180224000 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0004719595233591829, | |
| "loss": 3.0501, | |
| "theoretical_loss": 4.40280477805997, | |
| "tokens_seen": 183500800 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00047136845091734443, | |
| "loss": 3.0715, | |
| "theoretical_loss": 4.392947629929752, | |
| "tokens_seen": 186777600 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00047077737847550595, | |
| "loss": 3.0423, | |
| "theoretical_loss": 4.383309386640423, | |
| "tokens_seen": 190054400 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0004701863060336675, | |
| "loss": 3.026, | |
| "theoretical_loss": 4.373881536388167, | |
| "tokens_seen": 193331200 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00046959523359182905, | |
| "loss": 3.0387, | |
| "theoretical_loss": 4.364656035524595, | |
| "tokens_seen": 196608000 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00046900416114999057, | |
| "loss": 3.0178, | |
| "theoretical_loss": 4.355625275511174, | |
| "tokens_seen": 199884800 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0004684130887081521, | |
| "loss": 3.0059, | |
| "theoretical_loss": 4.346782052704563, | |
| "tokens_seen": 203161600 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0004678220162663136, | |
| "loss": 3.0195, | |
| "theoretical_loss": 4.338119540689052, | |
| "tokens_seen": 206438400 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00046723094382447513, | |
| "loss": 3.0093, | |
| "theoretical_loss": 4.329631264904703, | |
| "tokens_seen": 209715200 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.00046663987138263665, | |
| "loss": 3.0085, | |
| "theoretical_loss": 4.321311079348144, | |
| "tokens_seen": 212992000 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0004660487989407982, | |
| "loss": 2.9952, | |
| "theoretical_loss": 4.313153145147683, | |
| "tokens_seen": 216268800 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.00046545772649895975, | |
| "loss": 2.9473, | |
| "theoretical_loss": 4.305151910836119, | |
| "tokens_seen": 219545600 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.00046486665405712127, | |
| "loss": 2.9939, | |
| "theoretical_loss": 4.2973020941635784, | |
| "tokens_seen": 222822400 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0004642755816152828, | |
| "loss": 3.0031, | |
| "theoretical_loss": 4.28959866530949, | |
| "tokens_seen": 226099200 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0004636845091734443, | |
| "loss": 2.9483, | |
| "theoretical_loss": 4.282036831367506, | |
| "tokens_seen": 229376000 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.00046309343673160583, | |
| "loss": 2.9299, | |
| "theoretical_loss": 4.274612021990189, | |
| "tokens_seen": 232652800 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0004625023642897674, | |
| "loss": 2.9692, | |
| "theoretical_loss": 4.267319876091788, | |
| "tokens_seen": 235929600 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004619112918479289, | |
| "loss": 2.9406, | |
| "theoretical_loss": 4.260156229517635, | |
| "tokens_seen": 239206400 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.00046132021940609044, | |
| "loss": 2.9809, | |
| "theoretical_loss": 4.253117103597704, | |
| "tokens_seen": 242483200 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.00046072914696425197, | |
| "loss": 2.9968, | |
| "theoretical_loss": 4.246198694509945, | |
| "tokens_seen": 245760000 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004601380745224135, | |
| "loss": 2.979, | |
| "theoretical_loss": 4.239397363386152, | |
| "tokens_seen": 249036800 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.000459547002080575, | |
| "loss": 2.951, | |
| "theoretical_loss": 4.232709627099522, | |
| "tokens_seen": 252313600 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004589559296387366, | |
| "loss": 2.9322, | |
| "theoretical_loss": 4.226132149678757, | |
| "tokens_seen": 255590400 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004583648571968981, | |
| "loss": 2.9567, | |
| "theoretical_loss": 4.219661734298666, | |
| "tokens_seen": 258867200 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004577737847550596, | |
| "loss": 2.9331, | |
| "theoretical_loss": 4.213295315801815, | |
| "tokens_seen": 262144000 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004571827123132211, | |
| "loss": 2.8995, | |
| "theoretical_loss": 4.207029953709861, | |
| "tokens_seen": 265420800 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0004565916398713826, | |
| "loss": 2.9239, | |
| "theoretical_loss": 4.200862825686893, | |
| "tokens_seen": 268697600 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.00045600056742954413, | |
| "loss": 2.9299, | |
| "theoretical_loss": 4.19479122142044, | |
| "tokens_seen": 271974400 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0004554094949877057, | |
| "loss": 2.9014, | |
| "theoretical_loss": 4.188812536888775, | |
| "tokens_seen": 275251200 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.00045481842254586723, | |
| "loss": 2.8966, | |
| "theoretical_loss": 4.182924268985855, | |
| "tokens_seen": 278528000 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.00045422735010402875, | |
| "loss": 2.8785, | |
| "theoretical_loss": 4.177124010477671, | |
| "tokens_seen": 281804800 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.00045363627766219027, | |
| "loss": 2.8701, | |
| "theoretical_loss": 4.171409445265983, | |
| "tokens_seen": 285081600 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0004530452052203518, | |
| "loss": 2.869, | |
| "theoretical_loss": 4.165778343937409, | |
| "tokens_seen": 288358400 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0004524541327785133, | |
| "loss": 2.8472, | |
| "theoretical_loss": 4.160228559577659, | |
| "tokens_seen": 291635200 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.00045186306033667483, | |
| "loss": 2.7937, | |
| "theoretical_loss": 4.15475802383233, | |
| "tokens_seen": 294912000 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0004512719878948364, | |
| "loss": 2.8514, | |
| "theoretical_loss": 4.149364743197177, | |
| "tokens_seen": 298188800 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.00045068091545299793, | |
| "loss": 2.8571, | |
| "theoretical_loss": 4.14404679552214, | |
| "tokens_seen": 301465600 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.00045008984301115945, | |
| "loss": 2.8575, | |
| "theoretical_loss": 4.138802326714632, | |
| "tokens_seen": 304742400 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.00044949877056932097, | |
| "loss": 2.8846, | |
| "theoretical_loss": 4.133629547628726, | |
| "tokens_seen": 308019200 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0004489076981274825, | |
| "loss": 2.9035, | |
| "theoretical_loss": 4.128526731127894, | |
| "tokens_seen": 311296000 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.000448316625685644, | |
| "loss": 2.87, | |
| "theoretical_loss": 4.123492209309923, | |
| "tokens_seen": 314572800 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0004477255532438056, | |
| "loss": 2.8807, | |
| "theoretical_loss": 4.118524370883447, | |
| "tokens_seen": 317849600 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0004471344808019671, | |
| "loss": 2.8528, | |
| "theoretical_loss": 4.113621658686355, | |
| "tokens_seen": 321126400 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.00044654340836012863, | |
| "loss": 2.8407, | |
| "theoretical_loss": 4.108782567337039, | |
| "tokens_seen": 324403200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.6097476930643404, | |
| "debugging/distinct-1-grams": 0.7770138893542506, | |
| "debugging/distinct-2-grams": 0.9535605375181951, | |
| "debugging/entropy-1-grams": 6.010126735429775, | |
| "debugging/entropy-2-grams": 6.85493648784113, | |
| "debugging/length": 575.7333333333333, | |
| "debugging/num_segments": 15, | |
| "epoch": 0.12, | |
| "objective/train/docs_used": 182636, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.6651382446289062, | |
| "objective/train/theoretical_loss": 4.104005641010112, | |
| "objective/train/tokens_used": 348140000, | |
| "theoretical_loss": 4.104005641010112, | |
| "tokens_seen": 327680000 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.00044595233591829015, | |
| "loss": 2.839, | |
| "theoretical_loss": 4.104005641010112, | |
| "tokens_seen": 327680000 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.00044536126347645167, | |
| "loss": 2.807, | |
| "theoretical_loss": 4.099289471328812, | |
| "tokens_seen": 330956800 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0004447701910346132, | |
| "loss": 2.8074, | |
| "theoretical_loss": 4.094632695366921, | |
| "tokens_seen": 334233600 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.00044417911859277476, | |
| "loss": 2.8079, | |
| "theoretical_loss": 4.090033993753448, | |
| "tokens_seen": 337510400 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0004435880461509363, | |
| "loss": 2.7971, | |
| "theoretical_loss": 4.085492088873883, | |
| "tokens_seen": 340787200 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0004429969737090978, | |
| "loss": 2.8108, | |
| "theoretical_loss": 4.081005743162224, | |
| "tokens_seen": 344064000 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0004424059012672593, | |
| "loss": 2.7982, | |
| "theoretical_loss": 4.076573757478361, | |
| "tokens_seen": 347340800 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00044181482882542085, | |
| "loss": 2.8143, | |
| "theoretical_loss": 4.072194969565807, | |
| "tokens_seen": 350617600 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00044122375638358237, | |
| "loss": 2.817, | |
| "theoretical_loss": 4.067868252585089, | |
| "tokens_seen": 353894400 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0004406326839417439, | |
| "loss": 2.7987, | |
| "theoretical_loss": 4.063592513718411, | |
| "tokens_seen": 357171200 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00044004161149990546, | |
| "loss": 2.796, | |
| "theoretical_loss": 4.059366692841521, | |
| "tokens_seen": 360448000 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.000439450539058067, | |
| "loss": 2.8249, | |
| "theoretical_loss": 4.055189761258959, | |
| "tokens_seen": 363724800 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0004388594666162285, | |
| "loss": 2.7585, | |
| "theoretical_loss": 4.051060720499127, | |
| "tokens_seen": 367001600 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00043826839417439, | |
| "loss": 2.7594, | |
| "theoretical_loss": 4.046978601165831, | |
| "tokens_seen": 370278400 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00043767732173255155, | |
| "loss": 2.7873, | |
| "theoretical_loss": 4.042942461843204, | |
| "tokens_seen": 373555200 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00043708624929071307, | |
| "loss": 2.8348, | |
| "theoretical_loss": 4.038951388051044, | |
| "tokens_seen": 376832000 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.00043649517684887464, | |
| "loss": 2.8196, | |
| "theoretical_loss": 4.035004491247873, | |
| "tokens_seen": 380108800 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.00043590410440703616, | |
| "loss": 2.8058, | |
| "theoretical_loss": 4.031100907879109, | |
| "tokens_seen": 383385600 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0004353130319651977, | |
| "loss": 2.8149, | |
| "theoretical_loss": 4.02723979846797, | |
| "tokens_seen": 386662400 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0004347219595233592, | |
| "loss": 2.797, | |
| "theoretical_loss": 4.023420346746835, | |
| "tokens_seen": 389939200 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0004341308870815207, | |
| "loss": 2.7989, | |
| "theoretical_loss": 4.019641758826938, | |
| "tokens_seen": 393216000 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.00043353981463968225, | |
| "loss": 2.8114, | |
| "theoretical_loss": 4.015903262404413, | |
| "tokens_seen": 396492800 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0004329487421978438, | |
| "loss": 2.762, | |
| "theoretical_loss": 4.012204106000786, | |
| "tokens_seen": 399769600 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.00043235766975600534, | |
| "loss": 2.7748, | |
| "theoretical_loss": 4.008543558236181, | |
| "tokens_seen": 403046400 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.00043176659731416686, | |
| "loss": 2.7641, | |
| "theoretical_loss": 4.004920907133565, | |
| "tokens_seen": 406323200 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004311755248723284, | |
| "loss": 2.7425, | |
| "theoretical_loss": 4.001335459452449, | |
| "tokens_seen": 409600000 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004305844524304899, | |
| "loss": 2.7494, | |
| "theoretical_loss": 3.997786540050617, | |
| "tokens_seen": 412876800 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004299933799886514, | |
| "loss": 2.7559, | |
| "theoretical_loss": 3.9942734912724456, | |
| "tokens_seen": 416153600 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.00042940230754681295, | |
| "loss": 2.7416, | |
| "theoretical_loss": 3.9907956723625375, | |
| "tokens_seen": 419430400 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004288112351049745, | |
| "loss": 2.7749, | |
| "theoretical_loss": 3.9873524589034224, | |
| "tokens_seen": 422707200 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.000428220162663136, | |
| "loss": 2.7583, | |
| "theoretical_loss": 3.9839432422761556, | |
| "tokens_seen": 425984000 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004276290902212975, | |
| "loss": 2.7776, | |
| "theoretical_loss": 3.980567429142721, | |
| "tokens_seen": 429260800 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.00042703801777945903, | |
| "loss": 2.7914, | |
| "theoretical_loss": 3.977224440949197, | |
| "tokens_seen": 432537600 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042644694533762055, | |
| "loss": 2.7796, | |
| "theoretical_loss": 3.9739137134486917, | |
| "tokens_seen": 435814400 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042585587289578207, | |
| "loss": 2.7826, | |
| "theoretical_loss": 3.9706346962431396, | |
| "tokens_seen": 439091200 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042526480045394365, | |
| "loss": 2.8278, | |
| "theoretical_loss": 3.9673868523430564, | |
| "tokens_seen": 442368000 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042467372801210517, | |
| "loss": 2.7916, | |
| "theoretical_loss": 3.9641696577444376, | |
| "tokens_seen": 445644800 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0004240826555702667, | |
| "loss": 2.7818, | |
| "theoretical_loss": 3.9609826010220033, | |
| "tokens_seen": 448921600 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0004234915831284282, | |
| "loss": 2.8396, | |
| "theoretical_loss": 3.9578251829380506, | |
| "tokens_seen": 452198400 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042290051068658973, | |
| "loss": 2.7652, | |
| "theoretical_loss": 3.954696916066199, | |
| "tokens_seen": 455475200 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042230943824475125, | |
| "loss": 2.787, | |
| "theoretical_loss": 3.9515973244293643, | |
| "tokens_seen": 458752000 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0004217183658029128, | |
| "loss": 2.7561, | |
| "theoretical_loss": 3.948525943151326, | |
| "tokens_seen": 462028800 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.00042112729336107435, | |
| "loss": 2.7778, | |
| "theoretical_loss": 3.9454823181212815, | |
| "tokens_seen": 465305600 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.00042053622091923587, | |
| "loss": 2.777, | |
| "theoretical_loss": 3.9424660056708167, | |
| "tokens_seen": 468582400 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0004199451484773974, | |
| "loss": 2.7708, | |
| "theoretical_loss": 3.939476572262754, | |
| "tokens_seen": 471859200 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0004193540760355589, | |
| "loss": 2.7721, | |
| "theoretical_loss": 3.9365135941913563, | |
| "tokens_seen": 475136000 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.00041876300359372043, | |
| "loss": 2.7698, | |
| "theoretical_loss": 3.9335766572934023, | |
| "tokens_seen": 478412800 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.00041817193115188195, | |
| "loss": 2.7543, | |
| "theoretical_loss": 3.9306653566696603, | |
| "tokens_seen": 481689600 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0004175808587100435, | |
| "loss": 2.7272, | |
| "theoretical_loss": 3.927779296416332, | |
| "tokens_seen": 484966400 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.00041698978626820505, | |
| "loss": 2.7367, | |
| "theoretical_loss": 3.924918089366024, | |
| "tokens_seen": 488243200 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "objective/train/docs_used": 266945, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.685297727584839, | |
| "objective/train/theoretical_loss": 3.9220813568378707, | |
| "objective/train/tokens_used": 511980000, | |
| "theoretical_loss": 3.9220813568378707, | |
| "tokens_seen": 491520000 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.00041639871382636657, | |
| "loss": 2.698, | |
| "theoretical_loss": 3.9220813568378707, | |
| "tokens_seen": 491520000 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0004158076413845281, | |
| "loss": 2.7082, | |
| "theoretical_loss": 3.9192687283964096, | |
| "tokens_seen": 494796800 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0004152165689426896, | |
| "loss": 2.7273, | |
| "theoretical_loss": 3.9164798416188527, | |
| "tokens_seen": 498073600 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.00041462549650085113, | |
| "loss": 2.7106, | |
| "theoretical_loss": 3.913714341870409, | |
| "tokens_seen": 501350400 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0004140344240590127, | |
| "loss": 2.6887, | |
| "theoretical_loss": 3.9109718820873303, | |
| "tokens_seen": 504627200 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0004134433516171742, | |
| "loss": 2.6851, | |
| "theoretical_loss": 3.9082521225673625, | |
| "tokens_seen": 507904000 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.00041285227917533575, | |
| "loss": 2.6773, | |
| "theoretical_loss": 3.9055547307673075, | |
| "tokens_seen": 511180800 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.00041226120673349727, | |
| "loss": 2.7175, | |
| "theoretical_loss": 3.9028793811074056, | |
| "tokens_seen": 514457600 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0004116701342916588, | |
| "loss": 2.7491, | |
| "theoretical_loss": 3.900225754782274, | |
| "tokens_seen": 517734400 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0004110790618498203, | |
| "loss": 2.7338, | |
| "theoretical_loss": 3.897593539578138, | |
| "tokens_seen": 521011200 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0004104879894079819, | |
| "loss": 2.7031, | |
| "theoretical_loss": 3.8949824296961015, | |
| "tokens_seen": 524288000 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0004098969169661434, | |
| "loss": 2.7107, | |
| "theoretical_loss": 3.8923921255812353, | |
| "tokens_seen": 527564800 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0004093058445243049, | |
| "loss": 2.7432, | |
| "theoretical_loss": 3.8898223337572393, | |
| "tokens_seen": 530841600 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.00040871477208246645, | |
| "loss": 2.6887, | |
| "theoretical_loss": 3.88727276666648, | |
| "tokens_seen": 534118400 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.00040812369964062797, | |
| "loss": 2.6839, | |
| "theoretical_loss": 3.884743142515184, | |
| "tokens_seen": 537395200 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0004075326271987895, | |
| "loss": 2.7072, | |
| "theoretical_loss": 3.8822331851235985, | |
| "tokens_seen": 540672000 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0004069533762057878, | |
| "loss": 2.7285, | |
| "theoretical_loss": 3.87974262378093, | |
| "tokens_seen": 543948800 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.00040636230376394933, | |
| "loss": 2.6995, | |
| "theoretical_loss": 3.877271193104873, | |
| "tokens_seen": 547225600 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004057830527709476, | |
| "loss": 2.7322, | |
| "theoretical_loss": 3.8748186329055736, | |
| "tokens_seen": 550502400 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004051919803291091, | |
| "loss": 2.7092, | |
| "theoretical_loss": 3.87238468805384, | |
| "tokens_seen": 553779200 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.00040460090788727065, | |
| "loss": 2.6705, | |
| "theoretical_loss": 3.8699691083534633, | |
| "tokens_seen": 557056000 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.00040400983544543217, | |
| "loss": 2.6637, | |
| "theoretical_loss": 3.8675716484174907, | |
| "tokens_seen": 560332800 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004034187630035937, | |
| "loss": 2.7249, | |
| "theoretical_loss": 3.8651920675482936, | |
| "tokens_seen": 563609600 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.00040282769056175526, | |
| "loss": 2.6851, | |
| "theoretical_loss": 3.862830129621318, | |
| "tokens_seen": 566886400 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004022366181199168, | |
| "loss": 2.7408, | |
| "theoretical_loss": 3.8604856029723575, | |
| "tokens_seen": 570163200 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004016455456780783, | |
| "loss": 2.675, | |
| "theoretical_loss": 3.8581582602882447, | |
| "tokens_seen": 573440000 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0004010544732362398, | |
| "loss": 2.6936, | |
| "theoretical_loss": 3.8558478785008203, | |
| "tokens_seen": 576716800 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.00040046340079440135, | |
| "loss": 2.6972, | |
| "theoretical_loss": 3.8535542386840778, | |
| "tokens_seen": 579993600 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.00039987232835256287, | |
| "loss": 2.7192, | |
| "theoretical_loss": 3.8512771259543586, | |
| "tokens_seen": 583270400 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.00039928125591072444, | |
| "loss": 2.7147, | |
| "theoretical_loss": 3.8490163293735082, | |
| "tokens_seen": 586547200 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.00039869018346888596, | |
| "loss": 2.7246, | |
| "theoretical_loss": 3.8467716418548648, | |
| "tokens_seen": 589824000 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0003980991110270475, | |
| "loss": 2.6779, | |
| "theoretical_loss": 3.844542860072007, | |
| "tokens_seen": 593100800 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.000397508038585209, | |
| "loss": 2.7299, | |
| "theoretical_loss": 3.8423297843701496, | |
| "tokens_seen": 596377600 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0003969169661433705, | |
| "loss": 2.7322, | |
| "theoretical_loss": 3.8401322186800995, | |
| "tokens_seen": 599654400 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.00039632589370153205, | |
| "loss": 2.7438, | |
| "theoretical_loss": 3.83794997043469, | |
| "tokens_seen": 602931200 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0003957348212596936, | |
| "loss": 2.7522, | |
| "theoretical_loss": 3.8357828504876004, | |
| "tokens_seen": 606208000 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.00039514374881785514, | |
| "loss": 2.7625, | |
| "theoretical_loss": 3.833630673034487, | |
| "tokens_seen": 609484800 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.00039455267637601666, | |
| "loss": 2.7712, | |
| "theoretical_loss": 3.831493255536345, | |
| "tokens_seen": 612761600 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0003939616039341782, | |
| "loss": 2.7518, | |
| "theoretical_loss": 3.8293704186450253, | |
| "tokens_seen": 616038400 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0003933705314923397, | |
| "loss": 2.7159, | |
| "theoretical_loss": 3.827261986130839, | |
| "tokens_seen": 619315200 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0003927794590505012, | |
| "loss": 2.6877, | |
| "theoretical_loss": 3.825167784812175, | |
| "tokens_seen": 622592000 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.00039218838660866275, | |
| "loss": 2.6786, | |
| "theoretical_loss": 3.823087644487069, | |
| "tokens_seen": 625868800 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0003915973141668243, | |
| "loss": 2.6904, | |
| "theoretical_loss": 3.8210213978666565, | |
| "tokens_seen": 629145600 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.00039100624172498584, | |
| "loss": 2.6902, | |
| "theoretical_loss": 3.8189688805104476, | |
| "tokens_seen": 632422400 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.00039041516928314736, | |
| "loss": 2.6763, | |
| "theoretical_loss": 3.816929930763374, | |
| "tokens_seen": 635699200 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0003898240968413089, | |
| "loss": 2.6795, | |
| "theoretical_loss": 3.8149043896945347, | |
| "tokens_seen": 638976000 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0003892330243994704, | |
| "loss": 2.6677, | |
| "theoretical_loss": 3.812892101037601, | |
| "tokens_seen": 642252800 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0003886419519576319, | |
| "loss": 2.6813, | |
| "theoretical_loss": 3.81089291113282, | |
| "tokens_seen": 645529600 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0003880508795157935, | |
| "loss": 2.682, | |
| "theoretical_loss": 3.8089066688705673, | |
| "tokens_seen": 648806400 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.000387459807073955, | |
| "loss": 2.6562, | |
| "theoretical_loss": 3.8069332256363992, | |
| "tokens_seen": 652083200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.46032904668873204, | |
| "debugging/distinct-1-grams": 0.8086458165007655, | |
| "debugging/distinct-2-grams": 0.9712022459970895, | |
| "debugging/entropy-1-grams": 5.567510239963637, | |
| "debugging/entropy-2-grams": 6.342889956495089, | |
| "debugging/length": 518.7777777777778, | |
| "debugging/num_segments": 9, | |
| "epoch": 0.23, | |
| "objective/train/docs_used": 351952, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.591569423675537, | |
| "objective/train/theoretical_loss": 3.80497243525756, | |
| "objective/train/tokens_used": 675820000, | |
| "theoretical_loss": 3.80497243525756, | |
| "tokens_seen": 655360000 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.00038686873463211654, | |
| "loss": 2.6468, | |
| "theoretical_loss": 3.80497243525756, | |
| "tokens_seen": 655360000 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.00038627766219027806, | |
| "loss": 2.6231, | |
| "theoretical_loss": 3.8030241539508958, | |
| "tokens_seen": 658636800 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003856865897484396, | |
| "loss": 2.6129, | |
| "theoretical_loss": 3.8010882402721324, | |
| "tokens_seen": 661913600 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003850955173066011, | |
| "loss": 2.6584, | |
| "theoretical_loss": 3.7991645550664757, | |
| "tokens_seen": 665190400 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003845044448647627, | |
| "loss": 2.718, | |
| "theoretical_loss": 3.797252961420492, | |
| "tokens_seen": 668467200 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003839133724229242, | |
| "loss": 2.68, | |
| "theoretical_loss": 3.795353324615228, | |
| "tokens_seen": 671744000 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003833222999810857, | |
| "loss": 2.6926, | |
| "theoretical_loss": 3.793465512080541, | |
| "tokens_seen": 675020800 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.00038273122753924724, | |
| "loss": 2.7406, | |
| "theoretical_loss": 3.791589393350587, | |
| "tokens_seen": 678297600 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.00038214015509740876, | |
| "loss": 2.6932, | |
| "theoretical_loss": 3.7897248400204475, | |
| "tokens_seen": 681574400 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.00038154908265557023, | |
| "loss": 2.6717, | |
| "theoretical_loss": 3.7878717257038534, | |
| "tokens_seen": 684851200 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.00038095801021373175, | |
| "loss": 2.6572, | |
| "theoretical_loss": 3.7860299259919685, | |
| "tokens_seen": 688128000 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0003803669377718933, | |
| "loss": 2.6723, | |
| "theoretical_loss": 3.7841993184132114, | |
| "tokens_seen": 691404800 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.00037977586533005485, | |
| "loss": 2.6649, | |
| "theoretical_loss": 3.78237978239408, | |
| "tokens_seen": 694681600 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.00037918479288821637, | |
| "loss": 2.681, | |
| "theoretical_loss": 3.780571199220942, | |
| "tokens_seen": 697958400 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0003785937204463779, | |
| "loss": 2.6608, | |
| "theoretical_loss": 3.7787734520027803, | |
| "tokens_seen": 701235200 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0003780026480045394, | |
| "loss": 2.6806, | |
| "theoretical_loss": 3.7769864256348455, | |
| "tokens_seen": 704512000 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.00037741157556270093, | |
| "loss": 2.7127, | |
| "theoretical_loss": 3.775210006763202, | |
| "tokens_seen": 707788800 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0003768205031208625, | |
| "loss": 2.69, | |
| "theoretical_loss": 3.7734440837501406, | |
| "tokens_seen": 711065600 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.000376229430679024, | |
| "loss": 2.6837, | |
| "theoretical_loss": 3.7716885466404246, | |
| "tokens_seen": 714342400 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.00037563835823718555, | |
| "loss": 2.7071, | |
| "theoretical_loss": 3.769943287128357, | |
| "tokens_seen": 717619200 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.00037504728579534707, | |
| "loss": 2.7125, | |
| "theoretical_loss": 3.7682081985256364, | |
| "tokens_seen": 720896000 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0003744562133535086, | |
| "loss": 2.7311, | |
| "theoretical_loss": 3.7664831757299795, | |
| "tokens_seen": 724172800 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0003738651409116701, | |
| "loss": 2.6893, | |
| "theoretical_loss": 3.7647681151944976, | |
| "tokens_seen": 727449600 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0003732740684698317, | |
| "loss": 2.6703, | |
| "theoretical_loss": 3.7630629148977937, | |
| "tokens_seen": 730726400 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0003726829960279932, | |
| "loss": 2.6717, | |
| "theoretical_loss": 3.761367474314768, | |
| "tokens_seen": 734003200 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0003720919235861547, | |
| "loss": 2.6687, | |
| "theoretical_loss": 3.7596816943881084, | |
| "tokens_seen": 737280000 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.00037150085114431625, | |
| "loss": 2.651, | |
| "theoretical_loss": 3.758005477500451, | |
| "tokens_seen": 740556800 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.00037090977870247777, | |
| "loss": 2.6678, | |
| "theoretical_loss": 3.756338727447186, | |
| "tokens_seen": 743833600 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0003703187062606393, | |
| "loss": 2.6621, | |
| "theoretical_loss": 3.7546813494098945, | |
| "tokens_seen": 747110400 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0003697276338188008, | |
| "loss": 2.6662, | |
| "theoretical_loss": 3.7530332499304007, | |
| "tokens_seen": 750387200 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0003691365613769624, | |
| "loss": 2.6735, | |
| "theoretical_loss": 3.7513943368854195, | |
| "tokens_seen": 753664000 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0003685454889351239, | |
| "loss": 2.637, | |
| "theoretical_loss": 3.7497645194617863, | |
| "tokens_seen": 756940800 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0003679544164932854, | |
| "loss": 2.6189, | |
| "theoretical_loss": 3.748143708132246, | |
| "tokens_seen": 760217600 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.00036736334405144695, | |
| "loss": 2.614, | |
| "theoretical_loss": 3.7465318146317994, | |
| "tokens_seen": 763494400 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.00036677227160960847, | |
| "loss": 2.5969, | |
| "theoretical_loss": 3.7449287519345766, | |
| "tokens_seen": 766771200 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.00036618119916777, | |
| "loss": 2.5948, | |
| "theoretical_loss": 3.7433344342312385, | |
| "tokens_seen": 770048000 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.00036559012672593156, | |
| "loss": 2.6212, | |
| "theoretical_loss": 3.7417487769068756, | |
| "tokens_seen": 773324800 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0003649990542840931, | |
| "loss": 2.6494, | |
| "theoretical_loss": 3.7401716965194076, | |
| "tokens_seen": 776601600 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0003644079818422546, | |
| "loss": 2.6427, | |
| "theoretical_loss": 3.738603110778461, | |
| "tokens_seen": 779878400 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0003638169094004161, | |
| "loss": 2.6702, | |
| "theoretical_loss": 3.73704293852471, | |
| "tokens_seen": 783155200 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.00036322583695857765, | |
| "loss": 2.6439, | |
| "theoretical_loss": 3.7354910997096793, | |
| "tokens_seen": 786432000 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.00036263476451673917, | |
| "loss": 2.6477, | |
| "theoretical_loss": 3.7339475153759825, | |
| "tokens_seen": 789708800 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.00036204369207490074, | |
| "loss": 2.6351, | |
| "theoretical_loss": 3.732412107638, | |
| "tokens_seen": 792985600 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.00036145261963306226, | |
| "loss": 2.6209, | |
| "theoretical_loss": 3.7308847996629724, | |
| "tokens_seen": 796262400 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0003608615471912238, | |
| "loss": 2.6421, | |
| "theoretical_loss": 3.7293655156525043, | |
| "tokens_seen": 799539200 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0003602704747493853, | |
| "loss": 2.6204, | |
| "theoretical_loss": 3.727854180824469, | |
| "tokens_seen": 802816000 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0003596794023075468, | |
| "loss": 2.5734, | |
| "theoretical_loss": 3.7263507213952978, | |
| "tokens_seen": 806092800 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.00035908832986570834, | |
| "loss": 2.5821, | |
| "theoretical_loss": 3.724855064562658, | |
| "tokens_seen": 809369600 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0003584972574238699, | |
| "loss": 2.6083, | |
| "theoretical_loss": 3.723367138488488, | |
| "tokens_seen": 812646400 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.00035790618498203144, | |
| "loss": 2.6159, | |
| "theoretical_loss": 3.7218868722824014, | |
| "tokens_seen": 815923200 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "objective/train/docs_used": 436799, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.6990954875946045, | |
| "objective/train/theoretical_loss": 3.7204141959854384, | |
| "objective/train/tokens_used": 839660000, | |
| "theoretical_loss": 3.7204141959854384, | |
| "tokens_seen": 819200000 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.00035731511254019296, | |
| "loss": 2.6223, | |
| "theoretical_loss": 3.7204141959854384, | |
| "tokens_seen": 819200000 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0003567240400983545, | |
| "loss": 2.6423, | |
| "theoretical_loss": 3.718949040554162, | |
| "tokens_seen": 822476800 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.000356132967656516, | |
| "loss": 2.6522, | |
| "theoretical_loss": 3.7174913378450833, | |
| "tokens_seen": 825753600 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0003555418952146775, | |
| "loss": 2.6592, | |
| "theoretical_loss": 3.7160410205994183, | |
| "tokens_seen": 829030400 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.00035495082277283904, | |
| "loss": 2.6816, | |
| "theoretical_loss": 3.7145980224281585, | |
| "tokens_seen": 832307200 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0003543597503310006, | |
| "loss": 2.6836, | |
| "theoretical_loss": 3.713162277797449, | |
| "tokens_seen": 835584000 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.00035376867788916214, | |
| "loss": 2.6382, | |
| "theoretical_loss": 3.7117337220142748, | |
| "tokens_seen": 838860800 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.00035317760544732366, | |
| "loss": 2.6612, | |
| "theoretical_loss": 3.7103122912124364, | |
| "tokens_seen": 842137600 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0003525865330054852, | |
| "loss": 2.6661, | |
| "theoretical_loss": 3.7088979223388128, | |
| "tokens_seen": 845414400 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.00035199546056364665, | |
| "loss": 2.6261, | |
| "theoretical_loss": 3.70749055313991, | |
| "tokens_seen": 848691200 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.00035140438812180817, | |
| "loss": 2.6546, | |
| "theoretical_loss": 3.7060901221486766, | |
| "tokens_seen": 851968000 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00035081331567996974, | |
| "loss": 2.6723, | |
| "theoretical_loss": 3.704696568671591, | |
| "tokens_seen": 855244800 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00035022224323813127, | |
| "loss": 2.6357, | |
| "theoretical_loss": 3.7033098327760063, | |
| "tokens_seen": 858521600 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.0003496311707962928, | |
| "loss": 2.6499, | |
| "theoretical_loss": 3.7019298552777533, | |
| "tokens_seen": 861798400 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.0003490400983544543, | |
| "loss": 2.648, | |
| "theoretical_loss": 3.700556577728988, | |
| "tokens_seen": 865075200 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00034844902591261583, | |
| "loss": 2.589, | |
| "theoretical_loss": 3.6991899424062815, | |
| "tokens_seen": 868352000 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00034785795347077735, | |
| "loss": 2.6425, | |
| "theoretical_loss": 3.697829892298951, | |
| "tokens_seen": 871628800 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.0003472668810289389, | |
| "loss": 2.6254, | |
| "theoretical_loss": 3.696476371097618, | |
| "tokens_seen": 874905600 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00034667580858710044, | |
| "loss": 2.5811, | |
| "theoretical_loss": 3.695129323182993, | |
| "tokens_seen": 878182400 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00034608473614526196, | |
| "loss": 2.6145, | |
| "theoretical_loss": 3.693788693614879, | |
| "tokens_seen": 881459200 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0003454936637034235, | |
| "loss": 2.5792, | |
| "theoretical_loss": 3.6924544281213967, | |
| "tokens_seen": 884736000 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0003449144127104218, | |
| "loss": 2.5563, | |
| "theoretical_loss": 3.691126473088412, | |
| "tokens_seen": 888012800 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.00034432334026858333, | |
| "loss": 2.5797, | |
| "theoretical_loss": 3.689804775549173, | |
| "tokens_seen": 891289600 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.00034373226782674485, | |
| "loss": 2.5937, | |
| "theoretical_loss": 3.688489283174146, | |
| "tokens_seen": 894566400 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.00034314119538490643, | |
| "loss": 2.6142, | |
| "theoretical_loss": 3.6871799442610538, | |
| "tokens_seen": 897843200 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0003425501229430679, | |
| "loss": 2.6138, | |
| "theoretical_loss": 3.685876707725093, | |
| "tokens_seen": 901120000 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0003419590505012294, | |
| "loss": 2.567, | |
| "theoretical_loss": 3.6845795230893517, | |
| "tokens_seen": 904396800 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.00034136797805939094, | |
| "loss": 2.5838, | |
| "theoretical_loss": 3.6832883404754035, | |
| "tokens_seen": 907673600 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00034077690561755246, | |
| "loss": 2.5798, | |
| "theoretical_loss": 3.6820031105940796, | |
| "tokens_seen": 910950400 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.000340185833175714, | |
| "loss": 2.5926, | |
| "theoretical_loss": 3.6807237847364176, | |
| "tokens_seen": 914227200 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00033959476073387555, | |
| "loss": 2.5927, | |
| "theoretical_loss": 3.6794503147647846, | |
| "tokens_seen": 917504000 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0003390036882920371, | |
| "loss": 2.6248, | |
| "theoretical_loss": 3.67818265310416, | |
| "tokens_seen": 920780800 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0003384126158501986, | |
| "loss": 2.6389, | |
| "theoretical_loss": 3.6769207527335888, | |
| "tokens_seen": 924057600 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0003378215434083601, | |
| "loss": 2.6146, | |
| "theoretical_loss": 3.675664567177787, | |
| "tokens_seen": 927334400 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00033723047096652164, | |
| "loss": 2.6046, | |
| "theoretical_loss": 3.674414050498913, | |
| "tokens_seen": 930611200 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00033663939852468316, | |
| "loss": 2.5907, | |
| "theoretical_loss": 3.6731691572884824, | |
| "tokens_seen": 933888000 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00033604832608284473, | |
| "loss": 2.5949, | |
| "theoretical_loss": 3.671929842659438, | |
| "tokens_seen": 937164800 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00033545725364100625, | |
| "loss": 2.6068, | |
| "theoretical_loss": 3.6706960622383624, | |
| "tokens_seen": 940441600 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0003348661811991678, | |
| "loss": 2.6191, | |
| "theoretical_loss": 3.6694677721578377, | |
| "tokens_seen": 943718400 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0003342751087573293, | |
| "loss": 2.5971, | |
| "theoretical_loss": 3.66824492904894, | |
| "tokens_seen": 946995200 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0003336840363154908, | |
| "loss": 2.5676, | |
| "theoretical_loss": 3.667027490033874, | |
| "tokens_seen": 950272000 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00033309296387365234, | |
| "loss": 2.5524, | |
| "theoretical_loss": 3.6658154127187412, | |
| "tokens_seen": 953548800 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00033250189143181386, | |
| "loss": 2.5736, | |
| "theoretical_loss": 3.664608655186437, | |
| "tokens_seen": 956825600 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00033191081898997543, | |
| "loss": 2.6165, | |
| "theoretical_loss": 3.663407175989679, | |
| "tokens_seen": 960102400 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00033131974654813695, | |
| "loss": 2.6047, | |
| "theoretical_loss": 3.662210934144158, | |
| "tokens_seen": 963379200 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003307286741062985, | |
| "loss": 2.58, | |
| "theoretical_loss": 3.661019889121812, | |
| "tokens_seen": 966656000 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00033013760166446, | |
| "loss": 2.6282, | |
| "theoretical_loss": 3.6598340008442234, | |
| "tokens_seen": 969932800 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003295465292226215, | |
| "loss": 2.5975, | |
| "theoretical_loss": 3.6586532296761285, | |
| "tokens_seen": 973209600 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00032895545678078304, | |
| "loss": 2.5975, | |
| "theoretical_loss": 3.657477536419047, | |
| "tokens_seen": 976486400 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003283643843389446, | |
| "loss": 2.6145, | |
| "theoretical_loss": 3.656306882305022, | |
| "tokens_seen": 979763200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.5262804053974194, | |
| "debugging/distinct-1-grams": 0.8078874712972218, | |
| "debugging/distinct-2-grams": 0.9705681917257234, | |
| "debugging/entropy-1-grams": 5.820230432223256, | |
| "debugging/entropy-2-grams": 6.432853055042453, | |
| "debugging/length": 602.8888888888889, | |
| "debugging/num_segments": 9, | |
| "epoch": 0.35, | |
| "objective/train/docs_used": 521997, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.8230371475219727, | |
| "objective/train/theoretical_loss": 3.6551412289904697, | |
| "objective/train/tokens_used": 1003500000, | |
| "theoretical_loss": 3.6551412289904697, | |
| "tokens_seen": 983040000 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00032777331189710613, | |
| "loss": 2.5744, | |
| "theoretical_loss": 3.6551412289904697, | |
| "tokens_seen": 983040000 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00032718223945526765, | |
| "loss": 2.588, | |
| "theoretical_loss": 3.6539805385501376, | |
| "tokens_seen": 986316800 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003265911670134292, | |
| "loss": 2.6134, | |
| "theoretical_loss": 3.652824773471171, | |
| "tokens_seen": 989593600 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003260000945715907, | |
| "loss": 2.6078, | |
| "theoretical_loss": 3.651673896647277, | |
| "tokens_seen": 992870400 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0003254090221297522, | |
| "loss": 2.5771, | |
| "theoretical_loss": 3.6505278713729985, | |
| "tokens_seen": 996147200 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0003248179496879138, | |
| "loss": 2.5559, | |
| "theoretical_loss": 3.6493866613380774, | |
| "tokens_seen": 999424000 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0003242268772460753, | |
| "loss": 2.5702, | |
| "theoretical_loss": 3.648250230621924, | |
| "tokens_seen": 1002700800 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.00032363580480423683, | |
| "loss": 2.5514, | |
| "theoretical_loss": 3.647118543688179, | |
| "tokens_seen": 1005977600 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.00032304473236239835, | |
| "loss": 2.5431, | |
| "theoretical_loss": 3.6459915653793633, | |
| "tokens_seen": 1009254400 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.00032245365992055987, | |
| "loss": 2.5538, | |
| "theoretical_loss": 3.644869260911628, | |
| "tokens_seen": 1012531200 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0003218625874787214, | |
| "loss": 2.5063, | |
| "theoretical_loss": 3.64375159586959, | |
| "tokens_seen": 1015808000 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0003212715150368829, | |
| "loss": 2.5415, | |
| "theoretical_loss": 3.642638536201252, | |
| "tokens_seen": 1019084800 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0003206804425950445, | |
| "loss": 2.5497, | |
| "theoretical_loss": 3.6415300482130135, | |
| "tokens_seen": 1022361600 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.000320089370153206, | |
| "loss": 2.6019, | |
| "theoretical_loss": 3.6404260985647667, | |
| "tokens_seen": 1025638400 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00031949829771136753, | |
| "loss": 2.559, | |
| "theoretical_loss": 3.6393266542650684, | |
| "tokens_seen": 1028915200 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00031890722526952905, | |
| "loss": 2.5516, | |
| "theoretical_loss": 3.638231682666401, | |
| "tokens_seen": 1032192000 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00031831615282769057, | |
| "loss": 2.5422, | |
| "theoretical_loss": 3.637141151460505, | |
| "tokens_seen": 1035468800 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0003177250803858521, | |
| "loss": 2.5578, | |
| "theoretical_loss": 3.636055028673799, | |
| "tokens_seen": 1038745600 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00031713400794401367, | |
| "loss": 2.5707, | |
| "theoretical_loss": 3.634973282662864, | |
| "tokens_seen": 1042022400 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0003165429355021752, | |
| "loss": 2.6216, | |
| "theoretical_loss": 3.6338958821100107, | |
| "tokens_seen": 1045299200 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0003159518630603367, | |
| "loss": 2.6277, | |
| "theoretical_loss": 3.63282279601892, | |
| "tokens_seen": 1048576000 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.00031536079061849823, | |
| "loss": 2.6734, | |
| "theoretical_loss": 3.631753993710352, | |
| "tokens_seen": 1051852800 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.00031476971817665975, | |
| "loss": 2.6774, | |
| "theoretical_loss": 3.630689444817925, | |
| "tokens_seen": 1055129600 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0003141786457348212, | |
| "loss": 2.6487, | |
| "theoretical_loss": 3.629629119283967, | |
| "tokens_seen": 1058406400 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0003135875732929828, | |
| "loss": 2.6663, | |
| "theoretical_loss": 3.628572987355434, | |
| "tokens_seen": 1061683200 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0003129965008511443, | |
| "loss": 2.6252, | |
| "theoretical_loss": 3.6275210195798913, | |
| "tokens_seen": 1064960000 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.00031240542840930583, | |
| "loss": 2.636, | |
| "theoretical_loss": 3.626473186801564, | |
| "tokens_seen": 1068236800 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.00031181435596746736, | |
| "loss": 2.6865, | |
| "theoretical_loss": 3.6254294601574495, | |
| "tokens_seen": 1071513600 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0003112232835256289, | |
| "loss": 2.637, | |
| "theoretical_loss": 3.624389811073493, | |
| "tokens_seen": 1074790400 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003106322110837904, | |
| "loss": 2.6161, | |
| "theoretical_loss": 3.6233542112608257, | |
| "tokens_seen": 1078067200 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003100411386419519, | |
| "loss": 2.6063, | |
| "theoretical_loss": 3.6223226327120592, | |
| "tokens_seen": 1081344000 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003094500662001135, | |
| "loss": 2.6153, | |
| "theoretical_loss": 3.621295047697644, | |
| "tokens_seen": 1084620800 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.000308858993758275, | |
| "loss": 2.6256, | |
| "theoretical_loss": 3.6202714287622833, | |
| "tokens_seen": 1087897600 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00030826792131643653, | |
| "loss": 2.6079, | |
| "theoretical_loss": 3.6192517487214038, | |
| "tokens_seen": 1091174400 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00030767684887459806, | |
| "loss": 2.6153, | |
| "theoretical_loss": 3.6182359806576834, | |
| "tokens_seen": 1094451200 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003070857764327596, | |
| "loss": 2.6394, | |
| "theoretical_loss": 3.6172240979176333, | |
| "tokens_seen": 1097728000 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003064947039909211, | |
| "loss": 2.6104, | |
| "theoretical_loss": 3.616216074108232, | |
| "tokens_seen": 1101004800 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00030590363154908267, | |
| "loss": 2.6492, | |
| "theoretical_loss": 3.6152118830936164, | |
| "tokens_seen": 1104281600 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0003053125591072442, | |
| "loss": 2.6321, | |
| "theoretical_loss": 3.6142114989918195, | |
| "tokens_seen": 1107558400 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0003047214866654057, | |
| "loss": 2.6388, | |
| "theoretical_loss": 3.6132148961715624, | |
| "tokens_seen": 1110835200 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00030413041422356723, | |
| "loss": 2.596, | |
| "theoretical_loss": 3.6122220492490964, | |
| "tokens_seen": 1114112000 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00030353934178172875, | |
| "loss": 2.5997, | |
| "theoretical_loss": 3.6112329330850894, | |
| "tokens_seen": 1117388800 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0003029482693398903, | |
| "loss": 2.609, | |
| "theoretical_loss": 3.61024752278157, | |
| "tokens_seen": 1120665600 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00030235719689805185, | |
| "loss": 2.5537, | |
| "theoretical_loss": 3.6092657936789054, | |
| "tokens_seen": 1123942400 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00030176612445621337, | |
| "loss": 2.5706, | |
| "theoretical_loss": 3.6082877213528377, | |
| "tokens_seen": 1127219200 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0003011750520143749, | |
| "loss": 2.5722, | |
| "theoretical_loss": 3.60731328161156, | |
| "tokens_seen": 1130496000 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0003005839795725364, | |
| "loss": 2.5611, | |
| "theoretical_loss": 3.6063424504928365, | |
| "tokens_seen": 1133772800 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00029999290713069793, | |
| "loss": 2.5825, | |
| "theoretical_loss": 3.60537520426117, | |
| "tokens_seen": 1137049600 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00029940183468885945, | |
| "loss": 2.6026, | |
| "theoretical_loss": 3.6044115194050086, | |
| "tokens_seen": 1140326400 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00029881076224702103, | |
| "loss": 2.5636, | |
| "theoretical_loss": 3.603451372633997, | |
| "tokens_seen": 1143603200 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "objective/train/docs_used": 604461, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.532255172729492, | |
| "objective/train/theoretical_loss": 3.6024947408762698, | |
| "objective/train/tokens_used": 1167340000, | |
| "theoretical_loss": 3.6024947408762698, | |
| "tokens_seen": 1146880000 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00029821968980518255, | |
| "loss": 2.6136, | |
| "theoretical_loss": 3.6024947408762698, | |
| "tokens_seen": 1146880000 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00029762861736334407, | |
| "loss": 2.5975, | |
| "theoretical_loss": 3.601541601275783, | |
| "tokens_seen": 1150156800 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00029704936637034234, | |
| "loss": 2.5864, | |
| "theoretical_loss": 3.6005919311896886, | |
| "tokens_seen": 1153433600 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00029645829392850386, | |
| "loss": 2.545, | |
| "theoretical_loss": 3.5996457081857454, | |
| "tokens_seen": 1156710400 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0002958672214866654, | |
| "loss": 2.5602, | |
| "theoretical_loss": 3.598702910039772, | |
| "tokens_seen": 1159987200 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0002952761490448269, | |
| "loss": 2.549, | |
| "theoretical_loss": 3.597763514733133, | |
| "tokens_seen": 1163264000 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0002946850766029885, | |
| "loss": 2.5652, | |
| "theoretical_loss": 3.59682750045027, | |
| "tokens_seen": 1166540800 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00029409400416115, | |
| "loss": 2.545, | |
| "theoretical_loss": 3.5958948455762583, | |
| "tokens_seen": 1169817600 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0002935029317193115, | |
| "loss": 2.6002, | |
| "theoretical_loss": 3.594965528694412, | |
| "tokens_seen": 1173094400 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00029291185927747304, | |
| "loss": 2.5693, | |
| "theoretical_loss": 3.594039528583913, | |
| "tokens_seen": 1176371200 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00029232078683563456, | |
| "loss": 2.5443, | |
| "theoretical_loss": 3.5931168242174847, | |
| "tokens_seen": 1179648000 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0002917297143937961, | |
| "loss": 2.5721, | |
| "theoretical_loss": 3.59219739475909, | |
| "tokens_seen": 1182924800 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00029113864195195766, | |
| "loss": 2.5627, | |
| "theoretical_loss": 3.5912812195616732, | |
| "tokens_seen": 1186201600 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0002905475695101192, | |
| "loss": 2.5564, | |
| "theoretical_loss": 3.590368278164926, | |
| "tokens_seen": 1189478400 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0002899564970682807, | |
| "loss": 2.5749, | |
| "theoretical_loss": 3.5894585502930902, | |
| "tokens_seen": 1192755200 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0002893654246264422, | |
| "loss": 2.5601, | |
| "theoretical_loss": 3.588552015852793, | |
| "tokens_seen": 1196032000 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.00028877435218460374, | |
| "loss": 2.5395, | |
| "theoretical_loss": 3.5876486549309097, | |
| "tokens_seen": 1199308800 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.00028818327974276526, | |
| "loss": 2.5493, | |
| "theoretical_loss": 3.586748447792462, | |
| "tokens_seen": 1202585600 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.00028759220730092684, | |
| "loss": 2.5885, | |
| "theoretical_loss": 3.5858513748785423, | |
| "tokens_seen": 1205862400 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.00028700113485908836, | |
| "loss": 2.6059, | |
| "theoretical_loss": 3.5849574168042704, | |
| "tokens_seen": 1209139200 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0002864100624172499, | |
| "loss": 2.6414, | |
| "theoretical_loss": 3.5840665543567782, | |
| "tokens_seen": 1212416000 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0002858189899754114, | |
| "loss": 2.584, | |
| "theoretical_loss": 3.583178768493222, | |
| "tokens_seen": 1215692800 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002852279175335729, | |
| "loss": 2.5749, | |
| "theoretical_loss": 3.5822940403388284, | |
| "tokens_seen": 1218969600 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00028463684509173444, | |
| "loss": 2.5516, | |
| "theoretical_loss": 3.581412351184958, | |
| "tokens_seen": 1222246400 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00028404577264989596, | |
| "loss": 2.5768, | |
| "theoretical_loss": 3.580533682487208, | |
| "tokens_seen": 1225523200 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00028345470020805754, | |
| "loss": 2.5683, | |
| "theoretical_loss": 3.579658015863532, | |
| "tokens_seen": 1228800000 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00028286362776621906, | |
| "loss": 2.5747, | |
| "theoretical_loss": 3.5787853330923927, | |
| "tokens_seen": 1232076800 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002822725553243806, | |
| "loss": 2.5668, | |
| "theoretical_loss": 3.577915616110936, | |
| "tokens_seen": 1235353600 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002816814828825421, | |
| "loss": 2.5884, | |
| "theoretical_loss": 3.577048847013194, | |
| "tokens_seen": 1238630400 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002810904104407036, | |
| "loss": 2.5668, | |
| "theoretical_loss": 3.57618500804831, | |
| "tokens_seen": 1241907200 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00028049933799886514, | |
| "loss": 2.5411, | |
| "theoretical_loss": 3.575324081618793, | |
| "tokens_seen": 1245184000 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.0002799082655570267, | |
| "loss": 2.5809, | |
| "theoretical_loss": 3.5744660502787875, | |
| "tokens_seen": 1248460800 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00027931719311518824, | |
| "loss": 2.5486, | |
| "theoretical_loss": 3.5736108967323794, | |
| "tokens_seen": 1251737600 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00027872612067334976, | |
| "loss": 2.5723, | |
| "theoretical_loss": 3.5727586038319155, | |
| "tokens_seen": 1255014400 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.0002781350482315113, | |
| "loss": 2.5451, | |
| "theoretical_loss": 3.571909154576348, | |
| "tokens_seen": 1258291200 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.0002775439757896728, | |
| "loss": 2.5707, | |
| "theoretical_loss": 3.5710625321096074, | |
| "tokens_seen": 1261568000 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.0002769529033478343, | |
| "loss": 2.5454, | |
| "theoretical_loss": 3.570218719718989, | |
| "tokens_seen": 1264844800 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.0002763618309059959, | |
| "loss": 2.5501, | |
| "theoretical_loss": 3.569377700833569, | |
| "tokens_seen": 1268121600 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00027577075846415736, | |
| "loss": 2.52, | |
| "theoretical_loss": 3.568539459022639, | |
| "tokens_seen": 1271398400 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0002751796860223189, | |
| "loss": 2.5709, | |
| "theoretical_loss": 3.5677039779941584, | |
| "tokens_seen": 1274675200 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0002745886135804804, | |
| "loss": 2.5565, | |
| "theoretical_loss": 3.566871241593236, | |
| "tokens_seen": 1277952000 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0002739975411386419, | |
| "loss": 2.5588, | |
| "theoretical_loss": 3.5660412338006235, | |
| "tokens_seen": 1281228800 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00027340646869680345, | |
| "loss": 2.5931, | |
| "theoretical_loss": 3.565213938731236, | |
| "tokens_seen": 1284505600 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00027281539625496497, | |
| "loss": 2.5872, | |
| "theoretical_loss": 3.5643893406326868, | |
| "tokens_seen": 1287782400 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00027222432381312654, | |
| "loss": 2.569, | |
| "theoretical_loss": 3.5635674238838466, | |
| "tokens_seen": 1291059200 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00027163325137128806, | |
| "loss": 2.5535, | |
| "theoretical_loss": 3.5627481729934196, | |
| "tokens_seen": 1294336000 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0002710421789294496, | |
| "loss": 2.5693, | |
| "theoretical_loss": 3.561931572598538, | |
| "tokens_seen": 1297612800 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0002704511064876111, | |
| "loss": 2.5675, | |
| "theoretical_loss": 3.5611176074633777, | |
| "tokens_seen": 1300889600 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0002698600340457726, | |
| "loss": 2.5699, | |
| "theoretical_loss": 3.5603062624777895, | |
| "tokens_seen": 1304166400 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00026928078305277095, | |
| "loss": 2.5992, | |
| "theoretical_loss": 3.559497522655951, | |
| "tokens_seen": 1307443200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.6120150397965989, | |
| "debugging/distinct-1-grams": 0.8028883415407951, | |
| "debugging/distinct-2-grams": 0.9694194448751143, | |
| "debugging/entropy-1-grams": 5.953641862246872, | |
| "debugging/entropy-2-grams": 6.684913997392006, | |
| "debugging/length": 651.1818181818181, | |
| "debugging/num_segments": 11, | |
| "epoch": 0.47, | |
| "objective/train/docs_used": 687197, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.7964653968811035, | |
| "objective/train/theoretical_loss": 3.5586913731350327, | |
| "objective/train/tokens_used": 1331180000, | |
| "theoretical_loss": 3.5586913731350327, | |
| "tokens_seen": 1310720000 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0002686897106109325, | |
| "loss": 2.6245, | |
| "theoretical_loss": 3.5586913731350327, | |
| "tokens_seen": 1310720000 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00026809863816909405, | |
| "loss": 2.6121, | |
| "theoretical_loss": 3.557887799173889, | |
| "tokens_seen": 1313996800 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00026750756572725557, | |
| "loss": 2.596, | |
| "theoretical_loss": 3.557086786151754, | |
| "tokens_seen": 1317273600 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0002669164932854171, | |
| "loss": 2.6216, | |
| "theoretical_loss": 3.5562883195669697, | |
| "tokens_seen": 1320550400 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00026632542084357855, | |
| "loss": 2.57, | |
| "theoretical_loss": 3.555492385035719, | |
| "tokens_seen": 1323827200 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0002657343484017401, | |
| "loss": 2.5643, | |
| "theoretical_loss": 3.5546989682907784, | |
| "tokens_seen": 1327104000 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026514327595990165, | |
| "loss": 2.5717, | |
| "theoretical_loss": 3.5539080551802895, | |
| "tokens_seen": 1330380800 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026455220351806317, | |
| "loss": 2.5581, | |
| "theoretical_loss": 3.553119631666546, | |
| "tokens_seen": 1333657600 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.0002639611310762247, | |
| "loss": 2.571, | |
| "theoretical_loss": 3.5523336838247914, | |
| "tokens_seen": 1336934400 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.0002633700586343862, | |
| "loss": 2.5432, | |
| "theoretical_loss": 3.55155019784204, | |
| "tokens_seen": 1340211200 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026277898619254773, | |
| "loss": 2.5438, | |
| "theoretical_loss": 3.5507691600159053, | |
| "tokens_seen": 1343488000 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026218791375070925, | |
| "loss": 2.5601, | |
| "theoretical_loss": 3.5499905567534515, | |
| "tokens_seen": 1346764800 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026159684130887083, | |
| "loss": 2.543, | |
| "theoretical_loss": 3.549214374570052, | |
| "tokens_seen": 1350041600 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026100576886703235, | |
| "loss": 2.5039, | |
| "theoretical_loss": 3.5484406000882665, | |
| "tokens_seen": 1353318400 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026041469642519387, | |
| "loss": 2.5037, | |
| "theoretical_loss": 3.5476692200367346, | |
| "tokens_seen": 1356595200 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.0002598236239833554, | |
| "loss": 2.5184, | |
| "theoretical_loss": 3.546900221249076, | |
| "tokens_seen": 1359872000 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.0002592325515415169, | |
| "loss": 2.5474, | |
| "theoretical_loss": 3.5461335906628157, | |
| "tokens_seen": 1363148800 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00025864147909967843, | |
| "loss": 2.5192, | |
| "theoretical_loss": 3.54536931531831, | |
| "tokens_seen": 1366425600 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00025805040665783995, | |
| "loss": 2.5314, | |
| "theoretical_loss": 3.5446073823576985, | |
| "tokens_seen": 1369702400 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00025745933421600153, | |
| "loss": 2.5159, | |
| "theoretical_loss": 3.543847779023859, | |
| "tokens_seen": 1372979200 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00025686826177416305, | |
| "loss": 2.5402, | |
| "theoretical_loss": 3.543090492659384, | |
| "tokens_seen": 1376256000 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00025627718933232457, | |
| "loss": 2.5155, | |
| "theoretical_loss": 3.542335510705562, | |
| "tokens_seen": 1379532800 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.0002556861168904861, | |
| "loss": 2.5134, | |
| "theoretical_loss": 3.541582820701378, | |
| "tokens_seen": 1382809600 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.0002550950444486476, | |
| "loss": 2.5125, | |
| "theoretical_loss": 3.5408324102825253, | |
| "tokens_seen": 1386086400 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.00025450397200680913, | |
| "loss": 2.519, | |
| "theoretical_loss": 3.5400842671804265, | |
| "tokens_seen": 1389363200 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.0002539128995649707, | |
| "loss": 2.4963, | |
| "theoretical_loss": 3.5393383792212676, | |
| "tokens_seen": 1392640000 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.00025332182712313223, | |
| "loss": 2.514, | |
| "theoretical_loss": 3.5385947343250486, | |
| "tokens_seen": 1395916800 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.00025273075468129375, | |
| "loss": 2.522, | |
| "theoretical_loss": 3.53785332050464, | |
| "tokens_seen": 1399193600 | |
| } | |
| ], | |
| "max_steps": 42724, | |
| "num_train_epochs": 9223372036854775807, | |
| "total_flos": 7.14460209610752e+17, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |