|
{ |
|
"best_metric": null, |
|
"best_model_checkpoint": null, |
|
"epoch": 1.0, |
|
"global_step": 42724, |
|
"is_hyper_param_search": false, |
|
"is_local_process_zero": true, |
|
"is_world_process_zero": true, |
|
"log_history": [ |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 1.6355140186915887e-06, |
|
"loss": 10.8008, |
|
"theoretical_loss": 20.81281780154715, |
|
"tokens_seen": 65536 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 8.177570093457944e-05, |
|
"loss": 8.7055, |
|
"theoretical_loss": 8.563482664611069, |
|
"tokens_seen": 3276800 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 0.0001635514018691589, |
|
"loss": 6.6732, |
|
"theoretical_loss": 7.4777587180480305, |
|
"tokens_seen": 6553600 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 0.0002453271028037383, |
|
"loss": 5.9716, |
|
"theoretical_loss": 6.9337544888949, |
|
"tokens_seen": 9830400 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 0.0003271028037383178, |
|
"loss": 5.5864, |
|
"theoretical_loss": 6.583566228426414, |
|
"tokens_seen": 13107200 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0004088785046728972, |
|
"loss": 5.3924, |
|
"theoretical_loss": 6.330713565116083, |
|
"tokens_seen": 16384000 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0004906542056074766, |
|
"loss": 5.2081, |
|
"theoretical_loss": 6.135529231940326, |
|
"tokens_seen": 19660800 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0005724299065420561, |
|
"loss": 5.0469, |
|
"theoretical_loss": 5.978101583869607, |
|
"tokens_seen": 22937600 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0006542056074766356, |
|
"loss": 4.9522, |
|
"theoretical_loss": 5.8471173262659235, |
|
"tokens_seen": 26214400 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0006996358993758275, |
|
"loss": 4.7919, |
|
"theoretical_loss": 5.7355768158821245, |
|
"tokens_seen": 29491200 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0006988083979572536, |
|
"loss": 4.7314, |
|
"theoretical_loss": 5.638870144071353, |
|
"tokens_seen": 32768000 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0006979808965386797, |
|
"loss": 4.6075, |
|
"theoretical_loss": 5.553812381844907, |
|
"tokens_seen": 36044800 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.000697153395120106, |
|
"loss": 4.5189, |
|
"theoretical_loss": 5.478118080556438, |
|
"tokens_seen": 39321600 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0006963258937015321, |
|
"loss": 4.4329, |
|
"theoretical_loss": 5.410095959579362, |
|
"tokens_seen": 42598400 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0006954983922829582, |
|
"loss": 4.4025, |
|
"theoretical_loss": 5.348462083735834, |
|
"tokens_seen": 45875200 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0006946708908643843, |
|
"loss": 4.25, |
|
"theoretical_loss": 5.292220566937567, |
|
"tokens_seen": 49152000 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0006938433894458105, |
|
"loss": 4.2174, |
|
"theoretical_loss": 5.240584625769978, |
|
"tokens_seen": 52428800 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0006930158880272367, |
|
"loss": 4.1421, |
|
"theoretical_loss": 5.192922724525789, |
|
"tokens_seen": 55705600 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0006921883866086628, |
|
"loss": 4.0643, |
|
"theoretical_loss": 5.1487208633564405, |
|
"tokens_seen": 58982400 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0006913608851900889, |
|
"loss": 3.9375, |
|
"theoretical_loss": 5.107555562405102, |
|
"tokens_seen": 62259200 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.000690533383771515, |
|
"loss": 3.8331, |
|
"theoretical_loss": 5.069074117143246, |
|
"tokens_seen": 65536000 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0006897058823529412, |
|
"loss": 3.7941, |
|
"theoretical_loss": 5.032979909838007, |
|
"tokens_seen": 68812800 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0006888783809343674, |
|
"loss": 3.7297, |
|
"theoretical_loss": 4.999021308224664, |
|
"tokens_seen": 72089600 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0006880508795157935, |
|
"loss": 3.6773, |
|
"theoretical_loss": 4.966983155351962, |
|
"tokens_seen": 75366400 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0006872233780972196, |
|
"loss": 3.5973, |
|
"theoretical_loss": 4.9366801616251355, |
|
"tokens_seen": 78643200 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0006863958766786457, |
|
"loss": 3.5893, |
|
"theoretical_loss": 4.907951713830082, |
|
"tokens_seen": 81920000 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0006855683752600718, |
|
"loss": 3.5262, |
|
"theoretical_loss": 4.880657753812926, |
|
"tokens_seen": 85196800 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.000684740873841498, |
|
"loss": 3.5248, |
|
"theoretical_loss": 4.854675474481779, |
|
"tokens_seen": 88473600 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0006839133724229242, |
|
"loss": 3.4372, |
|
"theoretical_loss": 4.8298966473088125, |
|
"tokens_seen": 91750400 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0006830858710043503, |
|
"loss": 3.4666, |
|
"theoretical_loss": 4.8062254427779205, |
|
"tokens_seen": 95027200 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0006822583695857764, |
|
"loss": 3.4167, |
|
"theoretical_loss": 4.783576639276257, |
|
"tokens_seen": 98304000 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0006814308681672025, |
|
"loss": 3.4719, |
|
"theoretical_loss": 4.761874140772408, |
|
"tokens_seen": 101580800 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0006806033667486286, |
|
"loss": 3.4687, |
|
"theoretical_loss": 4.741049741962473, |
|
"tokens_seen": 104857600 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0006797758653300548, |
|
"loss": 3.4353, |
|
"theoretical_loss": 4.721042093249051, |
|
"tokens_seen": 108134400 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.000678948363911481, |
|
"loss": 3.4114, |
|
"theoretical_loss": 4.701795828231866, |
|
"tokens_seen": 111411200 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0006781208624929071, |
|
"loss": 3.41, |
|
"theoretical_loss": 4.68326082423593, |
|
"tokens_seen": 114688000 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0006772933610743332, |
|
"loss": 3.3813, |
|
"theoretical_loss": 4.665391572426282, |
|
"tokens_seen": 117964800 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0006764658596557593, |
|
"loss": 3.3451, |
|
"theoretical_loss": 4.648146638719739, |
|
"tokens_seen": 121241600 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0006756383582371856, |
|
"loss": 3.3597, |
|
"theoretical_loss": 4.631488200339643, |
|
"tokens_seen": 124518400 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0006748108568186117, |
|
"loss": 3.3776, |
|
"theoretical_loss": 4.615381645715717, |
|
"tokens_seen": 127795200 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0006739833554000378, |
|
"loss": 3.3091, |
|
"theoretical_loss": 4.599795227690505, |
|
"tokens_seen": 131072000 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.000673155853981464, |
|
"loss": 3.3312, |
|
"theoretical_loss": 4.584699761792674, |
|
"tokens_seen": 134348800 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0006723283525628902, |
|
"loss": 3.2886, |
|
"theoretical_loss": 4.570068362778516, |
|
"tokens_seen": 137625600 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0006715008511443163, |
|
"loss": 3.3051, |
|
"theoretical_loss": 4.555876213804037, |
|
"tokens_seen": 140902400 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0006706733497257424, |
|
"loss": 3.2901, |
|
"theoretical_loss": 4.542100363530799, |
|
"tokens_seen": 144179200 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0006698458483071685, |
|
"loss": 3.2702, |
|
"theoretical_loss": 4.528719547234816, |
|
"tokens_seen": 147456000 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0006690183468885946, |
|
"loss": 3.2275, |
|
"theoretical_loss": 4.515714028614996, |
|
"tokens_seen": 150732800 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0006681908454700209, |
|
"loss": 3.2427, |
|
"theoretical_loss": 4.503065459513339, |
|
"tokens_seen": 154009600 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.000667363344051447, |
|
"loss": 3.2513, |
|
"theoretical_loss": 4.4907567551852665, |
|
"tokens_seen": 157286400 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0006665358426328731, |
|
"loss": 3.2329, |
|
"theoretical_loss": 4.478771983111967, |
|
"tokens_seen": 160563200 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"objective/train/avg_token_score": 0.027871694415807724, |
|
"objective/train/avg_weight": 0.977715790271759, |
|
"objective/train/docs_used": 104000, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 3.3409407138824463, |
|
"objective/train/std_weight": 0.06310887634754181, |
|
"objective/train/theoretical_loss": 4.467096263641219, |
|
"objective/train/tokens_used": 184300000, |
|
"theoretical_loss": 4.467096263641219, |
|
"tokens_seen": 163840000 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0006657083412142992, |
|
"loss": 3.2359, |
|
"theoretical_loss": 4.467096263641219, |
|
"tokens_seen": 163840000 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0006648808397957253, |
|
"loss": 3.3137, |
|
"theoretical_loss": 4.455715680989545, |
|
"tokens_seen": 167116800 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0006640533383771514, |
|
"loss": 3.2122, |
|
"theoretical_loss": 4.44461720334543, |
|
"tokens_seen": 170393600 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0006632258369585777, |
|
"loss": 3.1455, |
|
"theoretical_loss": 4.433788610987646, |
|
"tokens_seen": 173670400 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0006623983355400038, |
|
"loss": 3.198, |
|
"theoretical_loss": 4.42321843148016, |
|
"tokens_seen": 176947200 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0006615708341214299, |
|
"loss": 3.1453, |
|
"theoretical_loss": 4.412895881130142, |
|
"tokens_seen": 180224000 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.000660743332702856, |
|
"loss": 3.1834, |
|
"theoretical_loss": 4.4028108120020795, |
|
"tokens_seen": 183500800 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0006599158312842821, |
|
"loss": 3.1772, |
|
"theoretical_loss": 4.392953663871862, |
|
"tokens_seen": 186777600 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0006590883298657083, |
|
"loss": 3.1628, |
|
"theoretical_loss": 4.383315420582533, |
|
"tokens_seen": 190054400 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0006582608284471345, |
|
"loss": 3.1954, |
|
"theoretical_loss": 4.373887570330275, |
|
"tokens_seen": 193331200 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0006574333270285606, |
|
"loss": 3.1504, |
|
"theoretical_loss": 4.364662069466704, |
|
"tokens_seen": 196608000 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0006566058256099867, |
|
"loss": 3.1496, |
|
"theoretical_loss": 4.355631309453283, |
|
"tokens_seen": 199884800 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0006557783241914128, |
|
"loss": 3.1045, |
|
"theoretical_loss": 4.346788086646671, |
|
"tokens_seen": 203161600 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0006549508227728391, |
|
"loss": 3.1352, |
|
"theoretical_loss": 4.33812557463116, |
|
"tokens_seen": 206438400 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0006541233213542652, |
|
"loss": 3.1432, |
|
"theoretical_loss": 4.329637298846812, |
|
"tokens_seen": 209715200 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0006532958199356913, |
|
"loss": 3.106, |
|
"theoretical_loss": 4.321317113290252, |
|
"tokens_seen": 212992000 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0006524683185171174, |
|
"loss": 3.0747, |
|
"theoretical_loss": 4.3131591790897925, |
|
"tokens_seen": 216268800 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0006516408170985437, |
|
"loss": 3.1173, |
|
"theoretical_loss": 4.305157944778228, |
|
"tokens_seen": 219545600 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0006508133156799698, |
|
"loss": 3.1451, |
|
"theoretical_loss": 4.297308128105687, |
|
"tokens_seen": 222822400 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0006499858142613959, |
|
"loss": 3.1329, |
|
"theoretical_loss": 4.2896046992515995, |
|
"tokens_seen": 226099200 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.000649158312842822, |
|
"loss": 3.1677, |
|
"theoretical_loss": 4.282042865309616, |
|
"tokens_seen": 229376000 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0006483308114242481, |
|
"loss": 3.1376, |
|
"theoretical_loss": 4.274618055932298, |
|
"tokens_seen": 232652800 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0006475033100056744, |
|
"loss": 3.1095, |
|
"theoretical_loss": 4.267325910033897, |
|
"tokens_seen": 235929600 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0006466758085871005, |
|
"loss": 3.0555, |
|
"theoretical_loss": 4.260162263459744, |
|
"tokens_seen": 239206400 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0006458483071685266, |
|
"loss": 3.083, |
|
"theoretical_loss": 4.253123137539814, |
|
"tokens_seen": 242483200 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0006450208057499527, |
|
"loss": 3.1657, |
|
"theoretical_loss": 4.246204728452055, |
|
"tokens_seen": 245760000 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0006441933043313788, |
|
"loss": 3.1313, |
|
"theoretical_loss": 4.239403397328261, |
|
"tokens_seen": 249036800 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0006433658029128049, |
|
"loss": 3.0927, |
|
"theoretical_loss": 4.232715661041632, |
|
"tokens_seen": 252313600 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0006425383014942312, |
|
"loss": 3.1078, |
|
"theoretical_loss": 4.226138183620867, |
|
"tokens_seen": 255590400 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0006417108000756573, |
|
"loss": 3.0412, |
|
"theoretical_loss": 4.219667768240775, |
|
"tokens_seen": 258867200 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0006408832986570834, |
|
"loss": 3.045, |
|
"theoretical_loss": 4.213301349743924, |
|
"tokens_seen": 262144000 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0006400557972385095, |
|
"loss": 2.9994, |
|
"theoretical_loss": 4.20703598765197, |
|
"tokens_seen": 265420800 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0006392282958199356, |
|
"loss": 2.9626, |
|
"theoretical_loss": 4.2008688596290025, |
|
"tokens_seen": 268697600 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0006384007944013618, |
|
"loss": 2.9181, |
|
"theoretical_loss": 4.194797255362549, |
|
"tokens_seen": 271974400 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.000637573292982788, |
|
"loss": 3.0047, |
|
"theoretical_loss": 4.188818570830883, |
|
"tokens_seen": 275251200 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0006367457915642141, |
|
"loss": 3.0216, |
|
"theoretical_loss": 4.182930302927963, |
|
"tokens_seen": 278528000 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0006359182901456402, |
|
"loss": 2.9805, |
|
"theoretical_loss": 4.17713004441978, |
|
"tokens_seen": 281804800 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0006350907887270663, |
|
"loss": 3.0196, |
|
"theoretical_loss": 4.1714154792080915, |
|
"tokens_seen": 285081600 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0006342632873084925, |
|
"loss": 3.0324, |
|
"theoretical_loss": 4.165784377879517, |
|
"tokens_seen": 288358400 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0006334357858899187, |
|
"loss": 2.9929, |
|
"theoretical_loss": 4.160234593519768, |
|
"tokens_seen": 291635200 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0006326082844713448, |
|
"loss": 3.0177, |
|
"theoretical_loss": 4.15476405777444, |
|
"tokens_seen": 294912000 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0006317807830527709, |
|
"loss": 2.9921, |
|
"theoretical_loss": 4.149370777139286, |
|
"tokens_seen": 298188800 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0006309532816341972, |
|
"loss": 2.9847, |
|
"theoretical_loss": 4.144052829464249, |
|
"tokens_seen": 301465600 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0006301257802156233, |
|
"loss": 2.9369, |
|
"theoretical_loss": 4.138808360656742, |
|
"tokens_seen": 304742400 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0006292982787970494, |
|
"loss": 2.9483, |
|
"theoretical_loss": 4.133635581570836, |
|
"tokens_seen": 308019200 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0006284707773784755, |
|
"loss": 2.9245, |
|
"theoretical_loss": 4.128532765070004, |
|
"tokens_seen": 311296000 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0006276432759599016, |
|
"loss": 2.9521, |
|
"theoretical_loss": 4.123498243252032, |
|
"tokens_seen": 314572800 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0006268157745413279, |
|
"loss": 2.9375, |
|
"theoretical_loss": 4.118530404825556, |
|
"tokens_seen": 317849600 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.000625988273122754, |
|
"loss": 2.9599, |
|
"theoretical_loss": 4.113627692628464, |
|
"tokens_seen": 321126400 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0006251607717041801, |
|
"loss": 2.9643, |
|
"theoretical_loss": 4.108788601279149, |
|
"tokens_seen": 324403200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.5365128506817183, |
|
"debugging/distinct-1-grams": 0.7612814402327299, |
|
"debugging/distinct-2-grams": 0.9694583753853511, |
|
"debugging/entropy-1-grams": 6.003629944255698, |
|
"debugging/entropy-2-grams": 7.054987089269872, |
|
"debugging/length": 495.25, |
|
"debugging/num_segments": 16, |
|
"epoch": 0.12, |
|
"objective/train/avg_token_score": 0.04385810345411301, |
|
"objective/train/avg_weight": 0.9649326205253601, |
|
"objective/train/docs_used": 197327, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.908684015274048, |
|
"objective/train/std_weight": 0.12546230852603912, |
|
"objective/train/theoretical_loss": 4.10401167495222, |
|
"objective/train/tokens_used": 348140000, |
|
"theoretical_loss": 4.10401167495222, |
|
"tokens_seen": 327680000 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0006243332702856062, |
|
"loss": 2.9932, |
|
"theoretical_loss": 4.10401167495222, |
|
"tokens_seen": 327680000 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0006235057688670323, |
|
"loss": 2.9925, |
|
"theoretical_loss": 4.099295505270921, |
|
"tokens_seen": 330956800 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0006226782674484584, |
|
"loss": 2.9372, |
|
"theoretical_loss": 4.094638729309031, |
|
"tokens_seen": 334233600 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0006218507660298847, |
|
"loss": 2.9488, |
|
"theoretical_loss": 4.090040027695556, |
|
"tokens_seen": 337510400 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0006210232646113108, |
|
"loss": 2.906, |
|
"theoretical_loss": 4.085498122815992, |
|
"tokens_seen": 340787200 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0006201957631927369, |
|
"loss": 2.9313, |
|
"theoretical_loss": 4.081011777104333, |
|
"tokens_seen": 344064000 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.000619368261774163, |
|
"loss": 2.9368, |
|
"theoretical_loss": 4.076579791420469, |
|
"tokens_seen": 347340800 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.0006185407603555891, |
|
"loss": 2.9504, |
|
"theoretical_loss": 4.0722010035079155, |
|
"tokens_seen": 350617600 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.0006177132589370153, |
|
"loss": 2.9416, |
|
"theoretical_loss": 4.067874286527197, |
|
"tokens_seen": 353894400 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.0006168857575184414, |
|
"loss": 2.9402, |
|
"theoretical_loss": 4.063598547660519, |
|
"tokens_seen": 357171200 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.0006160582560998676, |
|
"loss": 2.9692, |
|
"theoretical_loss": 4.05937272678363, |
|
"tokens_seen": 360448000 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.0006152307546812937, |
|
"loss": 2.957, |
|
"theoretical_loss": 4.055195795201069, |
|
"tokens_seen": 363724800 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.0006144032532627198, |
|
"loss": 2.9066, |
|
"theoretical_loss": 4.051066754441235, |
|
"tokens_seen": 367001600 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.000613575751844146, |
|
"loss": 2.9128, |
|
"theoretical_loss": 4.04698463510794, |
|
"tokens_seen": 370278400 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.0006127482504255721, |
|
"loss": 2.9181, |
|
"theoretical_loss": 4.042948495785312, |
|
"tokens_seen": 373555200 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.0006119207490069983, |
|
"loss": 2.8869, |
|
"theoretical_loss": 4.038957421993153, |
|
"tokens_seen": 376832000 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0006110932475884244, |
|
"loss": 2.9055, |
|
"theoretical_loss": 4.035010525189982, |
|
"tokens_seen": 380108800 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0006102657461698505, |
|
"loss": 2.953, |
|
"theoretical_loss": 4.031106941821218, |
|
"tokens_seen": 383385600 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0006094382447512768, |
|
"loss": 2.9048, |
|
"theoretical_loss": 4.027245832410079, |
|
"tokens_seen": 386662400 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0006086107433327029, |
|
"loss": 2.8699, |
|
"theoretical_loss": 4.023426380688943, |
|
"tokens_seen": 389939200 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.000607783241914129, |
|
"loss": 2.9089, |
|
"theoretical_loss": 4.019647792769048, |
|
"tokens_seen": 393216000 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0006069722905239265, |
|
"loss": 2.9903, |
|
"theoretical_loss": 4.015909296346521, |
|
"tokens_seen": 396492800 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0006061447891053528, |
|
"loss": 2.9199, |
|
"theoretical_loss": 4.012210139942894, |
|
"tokens_seen": 399769600 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0006053172876867789, |
|
"loss": 2.9731, |
|
"theoretical_loss": 4.008549592178291, |
|
"tokens_seen": 403046400 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.000604489786268205, |
|
"loss": 2.9404, |
|
"theoretical_loss": 4.004926941075674, |
|
"tokens_seen": 406323200 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0006036622848496311, |
|
"loss": 2.9768, |
|
"theoretical_loss": 4.001341493394558, |
|
"tokens_seen": 409600000 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0006028347834310573, |
|
"loss": 2.9789, |
|
"theoretical_loss": 3.997792573992726, |
|
"tokens_seen": 412876800 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0006020072820124835, |
|
"loss": 2.9275, |
|
"theoretical_loss": 3.994279525214554, |
|
"tokens_seen": 416153600 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0006011797805939096, |
|
"loss": 2.9751, |
|
"theoretical_loss": 3.990801706304647, |
|
"tokens_seen": 419430400 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0006003522791753358, |
|
"loss": 2.9311, |
|
"theoretical_loss": 3.987358492845532, |
|
"tokens_seen": 422707200 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0005995247777567619, |
|
"loss": 2.9599, |
|
"theoretical_loss": 3.9839492762182647, |
|
"tokens_seen": 425984000 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.000598697276338188, |
|
"loss": 2.9319, |
|
"theoretical_loss": 3.9805734630848306, |
|
"tokens_seen": 429260800 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0005978697749196142, |
|
"loss": 2.9141, |
|
"theoretical_loss": 3.9772304748913054, |
|
"tokens_seen": 432537600 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.0005970422735010403, |
|
"loss": 2.9076, |
|
"theoretical_loss": 3.973919747390801, |
|
"tokens_seen": 435814400 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.0005962147720824664, |
|
"loss": 2.918, |
|
"theoretical_loss": 3.9706407301852487, |
|
"tokens_seen": 439091200 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.0005953872706638926, |
|
"loss": 2.8962, |
|
"theoretical_loss": 3.9673928862851655, |
|
"tokens_seen": 442368000 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.0005945597692453187, |
|
"loss": 2.887, |
|
"theoretical_loss": 3.9641756916865463, |
|
"tokens_seen": 445644800 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.0005937322678267449, |
|
"loss": 2.8944, |
|
"theoretical_loss": 3.960988634964113, |
|
"tokens_seen": 448921600 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.000592904766408171, |
|
"loss": 2.8689, |
|
"theoretical_loss": 3.9578312168801597, |
|
"tokens_seen": 452198400 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.0005920772649895971, |
|
"loss": 2.8671, |
|
"theoretical_loss": 3.954702950008308, |
|
"tokens_seen": 455475200 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.0005912497635710232, |
|
"loss": 2.8527, |
|
"theoretical_loss": 3.9516033583714734, |
|
"tokens_seen": 458752000 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0005904222621524494, |
|
"loss": 2.8787, |
|
"theoretical_loss": 3.9485319770934355, |
|
"tokens_seen": 462028800 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0005895947607338756, |
|
"loss": 2.8612, |
|
"theoretical_loss": 3.945488352063391, |
|
"tokens_seen": 465305600 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0005887672593153017, |
|
"loss": 2.8394, |
|
"theoretical_loss": 3.942472039612926, |
|
"tokens_seen": 468582400 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0005879397578967278, |
|
"loss": 2.8409, |
|
"theoretical_loss": 3.939482606204863, |
|
"tokens_seen": 471859200 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0005871122564781539, |
|
"loss": 2.8556, |
|
"theoretical_loss": 3.936519628133466, |
|
"tokens_seen": 475136000 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.00058628475505958, |
|
"loss": 2.8919, |
|
"theoretical_loss": 3.9335826912355114, |
|
"tokens_seen": 478412800 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0005854572536410061, |
|
"loss": 2.9328, |
|
"theoretical_loss": 3.93067139061177, |
|
"tokens_seen": 481689600 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0005846297522224324, |
|
"loss": 2.859, |
|
"theoretical_loss": 3.927785330358441, |
|
"tokens_seen": 484966400 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0005838022508038585, |
|
"loss": 2.8503, |
|
"theoretical_loss": 3.9249241233081333, |
|
"tokens_seen": 488243200 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"objective/train/avg_token_score": 0.004644907079637051, |
|
"objective/train/avg_weight": 0.9962868094444275, |
|
"objective/train/docs_used": 287192, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 3.003862142562866, |
|
"objective/train/std_weight": 0.018475370481610298, |
|
"objective/train/theoretical_loss": 3.92208739077998, |
|
"objective/train/tokens_used": 511980000, |
|
"theoretical_loss": 3.92208739077998, |
|
"tokens_seen": 491520000 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0005829747493852846, |
|
"loss": 2.8251, |
|
"theoretical_loss": 3.92208739077998, |
|
"tokens_seen": 491520000 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0005821472479667107, |
|
"loss": 2.8393, |
|
"theoretical_loss": 3.919274762338519, |
|
"tokens_seen": 494796800 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.000581319746548137, |
|
"loss": 2.8629, |
|
"theoretical_loss": 3.9164858755609613, |
|
"tokens_seen": 498073600 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0005804922451295631, |
|
"loss": 2.8465, |
|
"theoretical_loss": 3.9137203758125176, |
|
"tokens_seen": 501350400 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0005796647437109892, |
|
"loss": 2.8088, |
|
"theoretical_loss": 3.910977916029439, |
|
"tokens_seen": 504627200 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0005788372422924154, |
|
"loss": 2.8193, |
|
"theoretical_loss": 3.908258156509472, |
|
"tokens_seen": 507904000 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0005780097408738415, |
|
"loss": 2.8096, |
|
"theoretical_loss": 3.905560764709417, |
|
"tokens_seen": 511180800 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0005771822394552677, |
|
"loss": 2.7896, |
|
"theoretical_loss": 3.9028854150495143, |
|
"tokens_seen": 514457600 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0005763547380366938, |
|
"loss": 2.8133, |
|
"theoretical_loss": 3.9002317887243834, |
|
"tokens_seen": 517734400 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0005755272366181199, |
|
"loss": 2.8119, |
|
"theoretical_loss": 3.897599573520247, |
|
"tokens_seen": 521011200 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0005746997351995461, |
|
"loss": 2.8456, |
|
"theoretical_loss": 3.8949884636382106, |
|
"tokens_seen": 524288000 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0005738722337809722, |
|
"loss": 2.8278, |
|
"theoretical_loss": 3.892398159523345, |
|
"tokens_seen": 527564800 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0005730447323623984, |
|
"loss": 2.8807, |
|
"theoretical_loss": 3.889828367699349, |
|
"tokens_seen": 530841600 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0005722172309438245, |
|
"loss": 2.8198, |
|
"theoretical_loss": 3.8872788006085894, |
|
"tokens_seen": 534118400 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0005713897295252506, |
|
"loss": 2.8464, |
|
"theoretical_loss": 3.8847491764572926, |
|
"tokens_seen": 537395200 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0005705622281066767, |
|
"loss": 2.8284, |
|
"theoretical_loss": 3.882239219065708, |
|
"tokens_seen": 540672000 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0005697347266881029, |
|
"loss": 2.8548, |
|
"theoretical_loss": 3.879748657723039, |
|
"tokens_seen": 543948800 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0005689072252695291, |
|
"loss": 2.8829, |
|
"theoretical_loss": 3.8772772270469824, |
|
"tokens_seen": 547225600 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0005680797238509552, |
|
"loss": 2.8451, |
|
"theoretical_loss": 3.8748246668476827, |
|
"tokens_seen": 550502400 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0005672522224323813, |
|
"loss": 2.8627, |
|
"theoretical_loss": 3.8723907219959486, |
|
"tokens_seen": 553779200 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0005664247210138074, |
|
"loss": 2.8789, |
|
"theoretical_loss": 3.869975142295573, |
|
"tokens_seen": 557056000 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0005655972195952335, |
|
"loss": 2.9283, |
|
"theoretical_loss": 3.8675776823595998, |
|
"tokens_seen": 560332800 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0005647697181766596, |
|
"loss": 2.8956, |
|
"theoretical_loss": 3.8651981014904027, |
|
"tokens_seen": 563609600 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0005639422167580859, |
|
"loss": 2.8693, |
|
"theoretical_loss": 3.8628361635634265, |
|
"tokens_seen": 566886400 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.000563114715339512, |
|
"loss": 2.8919, |
|
"theoretical_loss": 3.8604916369144666, |
|
"tokens_seen": 570163200 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0005622872139209381, |
|
"loss": 2.8761, |
|
"theoretical_loss": 3.858164294230354, |
|
"tokens_seen": 573440000 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0005614597125023642, |
|
"loss": 2.8493, |
|
"theoretical_loss": 3.85585391244293, |
|
"tokens_seen": 576716800 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0005606322110837905, |
|
"loss": 2.8528, |
|
"theoretical_loss": 3.8535602726261864, |
|
"tokens_seen": 579993600 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0005598047096652166, |
|
"loss": 2.8398, |
|
"theoretical_loss": 3.851283159896468, |
|
"tokens_seen": 583270400 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0005589772082466427, |
|
"loss": 2.8225, |
|
"theoretical_loss": 3.8490223633156173, |
|
"tokens_seen": 586547200 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0005581497068280688, |
|
"loss": 2.8074, |
|
"theoretical_loss": 3.846777675796974, |
|
"tokens_seen": 589824000 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.000557322205409495, |
|
"loss": 2.8092, |
|
"theoretical_loss": 3.844548894014116, |
|
"tokens_seen": 593100800 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0005564947039909212, |
|
"loss": 2.8185, |
|
"theoretical_loss": 3.8423358183122582, |
|
"tokens_seen": 596377600 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0005556672025723473, |
|
"loss": 2.7918, |
|
"theoretical_loss": 3.840138252622208, |
|
"tokens_seen": 599654400 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0005548397011537734, |
|
"loss": 2.8229, |
|
"theoretical_loss": 3.837956004376799, |
|
"tokens_seen": 602931200 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0005540121997351996, |
|
"loss": 2.7877, |
|
"theoretical_loss": 3.8357888844297094, |
|
"tokens_seen": 606208000 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0005531846983166257, |
|
"loss": 2.8112, |
|
"theoretical_loss": 3.8336367069765958, |
|
"tokens_seen": 609484800 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0005523571968980519, |
|
"loss": 2.7626, |
|
"theoretical_loss": 3.8314992894784536, |
|
"tokens_seen": 612761600 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.000551529695479478, |
|
"loss": 2.8083, |
|
"theoretical_loss": 3.829376452587134, |
|
"tokens_seen": 616038400 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0005507021940609041, |
|
"loss": 2.8228, |
|
"theoretical_loss": 3.827268020072948, |
|
"tokens_seen": 619315200 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0005498746926423302, |
|
"loss": 2.8554, |
|
"theoretical_loss": 3.8251738187542843, |
|
"tokens_seen": 622592000 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0005490471912237563, |
|
"loss": 2.8388, |
|
"theoretical_loss": 3.8230936784291787, |
|
"tokens_seen": 625868800 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0005482196898051826, |
|
"loss": 2.8705, |
|
"theoretical_loss": 3.8210274318087656, |
|
"tokens_seen": 629145600 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0005473921883866087, |
|
"loss": 2.8195, |
|
"theoretical_loss": 3.818974914452557, |
|
"tokens_seen": 632422400 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0005465646869680348, |
|
"loss": 2.8152, |
|
"theoretical_loss": 3.8169359647054835, |
|
"tokens_seen": 635699200 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0005457371855494609, |
|
"loss": 2.7996, |
|
"theoretical_loss": 3.8149104236366433, |
|
"tokens_seen": 638976000 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.000544909684130887, |
|
"loss": 2.7787, |
|
"theoretical_loss": 3.8128981349797098, |
|
"tokens_seen": 642252800 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0005440821827123131, |
|
"loss": 2.8041, |
|
"theoretical_loss": 3.8108989450749293, |
|
"tokens_seen": 645529600 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0005432546812937394, |
|
"loss": 2.7924, |
|
"theoretical_loss": 3.8089127028126764, |
|
"tokens_seen": 648806400 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0005424271798751655, |
|
"loss": 2.8324, |
|
"theoretical_loss": 3.8069392595785083, |
|
"tokens_seen": 652083200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.5265375629586004, |
|
"debugging/distinct-1-grams": 0.7435820408094715, |
|
"debugging/distinct-2-grams": 0.9558103821233092, |
|
"debugging/entropy-1-grams": 5.931434510687563, |
|
"debugging/entropy-2-grams": 6.886416755326388, |
|
"debugging/length": 521.9230769230769, |
|
"debugging/num_segments": 13, |
|
"epoch": 0.23, |
|
"objective/train/avg_token_score": 0.022742915898561478, |
|
"objective/train/avg_weight": 0.9818012714385986, |
|
"objective/train/docs_used": 379091, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 3.068922758102417, |
|
"objective/train/std_weight": 0.06274868547916412, |
|
"objective/train/theoretical_loss": 3.804978469199669, |
|
"objective/train/tokens_used": 675820000, |
|
"theoretical_loss": 3.804978469199669, |
|
"tokens_seen": 655360000 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0005415996784565916, |
|
"loss": 2.8223, |
|
"theoretical_loss": 3.804978469199669, |
|
"tokens_seen": 655360000 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0005407721770380177, |
|
"loss": 2.837, |
|
"theoretical_loss": 3.803030187893005, |
|
"tokens_seen": 658636800 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0005399612256478154, |
|
"loss": 2.8345, |
|
"theoretical_loss": 3.8010942742142415, |
|
"tokens_seen": 661913600 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0005391337242292415, |
|
"loss": 2.8643, |
|
"theoretical_loss": 3.799170589008585, |
|
"tokens_seen": 665190400 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0005383062228106677, |
|
"loss": 2.8481, |
|
"theoretical_loss": 3.7972589953626006, |
|
"tokens_seen": 668467200 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0005374787213920938, |
|
"loss": 2.8602, |
|
"theoretical_loss": 3.795359358557337, |
|
"tokens_seen": 671744000 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0005366512199735199, |
|
"loss": 2.8011, |
|
"theoretical_loss": 3.79347154602265, |
|
"tokens_seen": 675020800 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0005358237185549461, |
|
"loss": 2.8665, |
|
"theoretical_loss": 3.7915954272926955, |
|
"tokens_seen": 678297600 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0005350127671647437, |
|
"loss": 2.8685, |
|
"theoretical_loss": 3.789730873962557, |
|
"tokens_seen": 681574400 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0005341852657461698, |
|
"loss": 2.7516, |
|
"theoretical_loss": 3.787877759645963, |
|
"tokens_seen": 684851200 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.000533357764327596, |
|
"loss": 2.7768, |
|
"theoretical_loss": 3.7860359599340776, |
|
"tokens_seen": 688128000 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0005325302629090222, |
|
"loss": 2.8109, |
|
"theoretical_loss": 3.784205352355321, |
|
"tokens_seen": 691404800 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0005317027614904483, |
|
"loss": 2.8397, |
|
"theoretical_loss": 3.782385816336189, |
|
"tokens_seen": 694681600 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0005308752600718744, |
|
"loss": 2.7955, |
|
"theoretical_loss": 3.7805772331630516, |
|
"tokens_seen": 697958400 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0005300477586533005, |
|
"loss": 2.7725, |
|
"theoretical_loss": 3.7787794859448898, |
|
"tokens_seen": 701235200 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0005292202572347266, |
|
"loss": 2.754, |
|
"theoretical_loss": 3.7769924595769546, |
|
"tokens_seen": 704512000 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0005283927558161528, |
|
"loss": 2.7445, |
|
"theoretical_loss": 3.7752160407053115, |
|
"tokens_seen": 707788800 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.000527565254397579, |
|
"loss": 2.7262, |
|
"theoretical_loss": 3.7734501176922493, |
|
"tokens_seen": 711065600 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0005267377529790051, |
|
"loss": 2.7704, |
|
"theoretical_loss": 3.7716945805825337, |
|
"tokens_seen": 714342400 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0005259102515604312, |
|
"loss": 2.8113, |
|
"theoretical_loss": 3.7699493210704667, |
|
"tokens_seen": 717619200 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0005250827501418573, |
|
"loss": 2.7852, |
|
"theoretical_loss": 3.7682142324677455, |
|
"tokens_seen": 720896000 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0005242552487232835, |
|
"loss": 2.8514, |
|
"theoretical_loss": 3.7664892096720886, |
|
"tokens_seen": 724172800 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0005234277473047097, |
|
"loss": 2.7829, |
|
"theoretical_loss": 3.7647741491366067, |
|
"tokens_seen": 727449600 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0005226002458861358, |
|
"loss": 2.7945, |
|
"theoretical_loss": 3.7630689488399027, |
|
"tokens_seen": 730726400 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0005217727444675619, |
|
"loss": 2.7599, |
|
"theoretical_loss": 3.7613735082568764, |
|
"tokens_seen": 734003200 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0005209452430489881, |
|
"loss": 2.7937, |
|
"theoretical_loss": 3.759687728330217, |
|
"tokens_seen": 737280000 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0005201177416304143, |
|
"loss": 2.7877, |
|
"theoretical_loss": 3.75801151144256, |
|
"tokens_seen": 740556800 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0005192902402118404, |
|
"loss": 2.7713, |
|
"theoretical_loss": 3.756344761389295, |
|
"tokens_seen": 743833600 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0005184627387932665, |
|
"loss": 2.7429, |
|
"theoretical_loss": 3.754687383352003, |
|
"tokens_seen": 747110400 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0005176352373746927, |
|
"loss": 2.747, |
|
"theoretical_loss": 3.7530392838725097, |
|
"tokens_seen": 750387200 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0005168077359561188, |
|
"loss": 2.7654, |
|
"theoretical_loss": 3.751400370827529, |
|
"tokens_seen": 753664000 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.000515980234537545, |
|
"loss": 2.7878, |
|
"theoretical_loss": 3.749770553403895, |
|
"tokens_seen": 756940800 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0005151527331189711, |
|
"loss": 2.7522, |
|
"theoretical_loss": 3.748149742074355, |
|
"tokens_seen": 760217600 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0005143252317003972, |
|
"loss": 2.782, |
|
"theoretical_loss": 3.746537848573908, |
|
"tokens_seen": 763494400 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0005134977302818233, |
|
"loss": 2.7967, |
|
"theoretical_loss": 3.744934785876686, |
|
"tokens_seen": 766771200 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0005126702288632494, |
|
"loss": 2.802, |
|
"theoretical_loss": 3.7433404681733475, |
|
"tokens_seen": 770048000 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0005118427274446757, |
|
"loss": 2.8081, |
|
"theoretical_loss": 3.7417548108489846, |
|
"tokens_seen": 773324800 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0005110152260261018, |
|
"loss": 2.781, |
|
"theoretical_loss": 3.740177730461517, |
|
"tokens_seen": 776601600 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0005101877246075279, |
|
"loss": 2.8304, |
|
"theoretical_loss": 3.73860914472057, |
|
"tokens_seen": 779878400 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.000509360223188954, |
|
"loss": 2.7936, |
|
"theoretical_loss": 3.7370489724668197, |
|
"tokens_seen": 783155200 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0005085327217703801, |
|
"loss": 2.8074, |
|
"theoretical_loss": 3.735497133651788, |
|
"tokens_seen": 786432000 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0005077052203518063, |
|
"loss": 2.7948, |
|
"theoretical_loss": 3.733953549318091, |
|
"tokens_seen": 789708800 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0005068777189332325, |
|
"loss": 2.8081, |
|
"theoretical_loss": 3.7324181415801094, |
|
"tokens_seen": 792985600 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0005060502175146586, |
|
"loss": 2.8048, |
|
"theoretical_loss": 3.7308908336050814, |
|
"tokens_seen": 796262400 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0005052227160960847, |
|
"loss": 2.7886, |
|
"theoretical_loss": 3.729371549594614, |
|
"tokens_seen": 799539200 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0005043952146775108, |
|
"loss": 2.8075, |
|
"theoretical_loss": 3.7278602147665776, |
|
"tokens_seen": 802816000 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.000503567713258937, |
|
"loss": 2.7927, |
|
"theoretical_loss": 3.726356755337407, |
|
"tokens_seen": 806092800 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0005027402118403631, |
|
"loss": 2.7687, |
|
"theoretical_loss": 3.724861098504767, |
|
"tokens_seen": 809369600 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0005019127104217893, |
|
"loss": 2.748, |
|
"theoretical_loss": 3.7233731724305974, |
|
"tokens_seen": 812646400 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0005010852090032154, |
|
"loss": 2.7755, |
|
"theoretical_loss": 3.7218929062245105, |
|
"tokens_seen": 815923200 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"objective/train/avg_token_score": 0.009068925864994526, |
|
"objective/train/avg_weight": 0.9927405714988708, |
|
"objective/train/docs_used": 471128, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.671570301055908, |
|
"objective/train/std_weight": 0.04655551165342331, |
|
"objective/train/theoretical_loss": 3.7204202299275475, |
|
"objective/train/tokens_used": 839660000, |
|
"theoretical_loss": 3.7204202299275475, |
|
"tokens_seen": 819200000 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0005002577075846415, |
|
"loss": 2.7207, |
|
"theoretical_loss": 3.7204202299275475, |
|
"tokens_seen": 819200000 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0004994302061660678, |
|
"loss": 2.744, |
|
"theoretical_loss": 3.7189550744962707, |
|
"tokens_seen": 822476800 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0004986027047474939, |
|
"loss": 2.7245, |
|
"theoretical_loss": 3.717497371787192, |
|
"tokens_seen": 825753600 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.00049777520332892, |
|
"loss": 2.7221, |
|
"theoretical_loss": 3.7160470545415274, |
|
"tokens_seen": 829030400 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0004969477019103461, |
|
"loss": 2.721, |
|
"theoretical_loss": 3.714604056370267, |
|
"tokens_seen": 832307200 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0004961202004917723, |
|
"loss": 2.7335, |
|
"theoretical_loss": 3.713168311739558, |
|
"tokens_seen": 835584000 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0004952926990731985, |
|
"loss": 2.7169, |
|
"theoretical_loss": 3.7117397559563843, |
|
"tokens_seen": 838860800 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0004944651976546246, |
|
"loss": 2.7024, |
|
"theoretical_loss": 3.710318325154545, |
|
"tokens_seen": 842137600 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0004936376962360507, |
|
"loss": 2.7488, |
|
"theoretical_loss": 3.7089039562809223, |
|
"tokens_seen": 845414400 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0004928101948174768, |
|
"loss": 2.7216, |
|
"theoretical_loss": 3.7074965870820193, |
|
"tokens_seen": 848691200 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0004919826933989029, |
|
"loss": 2.7227, |
|
"theoretical_loss": 3.7060961560907857, |
|
"tokens_seen": 851968000 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0004911551919803292, |
|
"loss": 2.78, |
|
"theoretical_loss": 3.7047026026137, |
|
"tokens_seen": 855244800 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0004903276905617553, |
|
"loss": 2.7405, |
|
"theoretical_loss": 3.7033158667181154, |
|
"tokens_seen": 858521600 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0004895001891431814, |
|
"loss": 2.7717, |
|
"theoretical_loss": 3.701935889219863, |
|
"tokens_seen": 861798400 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0004886726877246075, |
|
"loss": 2.7393, |
|
"theoretical_loss": 3.7005626116710966, |
|
"tokens_seen": 865075200 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00048784518630603363, |
|
"loss": 2.7463, |
|
"theoretical_loss": 3.69919597634839, |
|
"tokens_seen": 868352000 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00048701768488745975, |
|
"loss": 2.7256, |
|
"theoretical_loss": 3.6978359262410603, |
|
"tokens_seen": 871628800 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.000486190183468886, |
|
"loss": 2.7512, |
|
"theoretical_loss": 3.6964824050397276, |
|
"tokens_seen": 874905600 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0004853626820503121, |
|
"loss": 2.6954, |
|
"theoretical_loss": 3.6951353571251015, |
|
"tokens_seen": 878182400 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0004845351806317382, |
|
"loss": 2.695, |
|
"theoretical_loss": 3.693794727556988, |
|
"tokens_seen": 881459200 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0004837076792131644, |
|
"loss": 2.699, |
|
"theoretical_loss": 3.692460462063506, |
|
"tokens_seen": 884736000 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0004828801777945905, |
|
"loss": 2.7523, |
|
"theoretical_loss": 3.691132507030521, |
|
"tokens_seen": 888012800 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0004820526763760166, |
|
"loss": 2.7663, |
|
"theoretical_loss": 3.6898108094912816, |
|
"tokens_seen": 891289600 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.00048122517495744274, |
|
"loss": 2.7245, |
|
"theoretical_loss": 3.6884953171162556, |
|
"tokens_seen": 894566400 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.00048039767353886897, |
|
"loss": 2.7305, |
|
"theoretical_loss": 3.6871859782031624, |
|
"tokens_seen": 897843200 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0004795701721202951, |
|
"loss": 2.7417, |
|
"theoretical_loss": 3.685882741667202, |
|
"tokens_seen": 901120000 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0004787426707017212, |
|
"loss": 2.744, |
|
"theoretical_loss": 3.684585557031461, |
|
"tokens_seen": 904396800 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0004779151692831473, |
|
"loss": 2.7572, |
|
"theoretical_loss": 3.6832943744175126, |
|
"tokens_seen": 907673600 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00047708766786457344, |
|
"loss": 2.7303, |
|
"theoretical_loss": 3.682009144536188, |
|
"tokens_seen": 910950400 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00047626016644599956, |
|
"loss": 2.7572, |
|
"theoretical_loss": 3.680729818678526, |
|
"tokens_seen": 914227200 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.0004754326650274258, |
|
"loss": 2.7362, |
|
"theoretical_loss": 3.6794563487068936, |
|
"tokens_seen": 917504000 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.0004746051636088519, |
|
"loss": 2.7454, |
|
"theoretical_loss": 3.6781886870462692, |
|
"tokens_seen": 920780800 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.000473777662190278, |
|
"loss": 2.7626, |
|
"theoretical_loss": 3.676926786675698, |
|
"tokens_seen": 924057600 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.0004729501607717042, |
|
"loss": 2.7011, |
|
"theoretical_loss": 3.6756706011198963, |
|
"tokens_seen": 927334400 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.0004721226593531303, |
|
"loss": 2.6682, |
|
"theoretical_loss": 3.6744200844410217, |
|
"tokens_seen": 930611200 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00047129515793455643, |
|
"loss": 2.6695, |
|
"theoretical_loss": 3.6731751912305914, |
|
"tokens_seen": 933888000 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00047046765651598266, |
|
"loss": 2.7057, |
|
"theoretical_loss": 3.671935876601547, |
|
"tokens_seen": 937164800 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0004696401550974088, |
|
"loss": 2.6439, |
|
"theoretical_loss": 3.6707020961804715, |
|
"tokens_seen": 940441600 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0004688126536788349, |
|
"loss": 2.6978, |
|
"theoretical_loss": 3.6694738060999468, |
|
"tokens_seen": 943718400 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.000467985152260261, |
|
"loss": 2.7145, |
|
"theoretical_loss": 3.668250962991049, |
|
"tokens_seen": 946995200 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.00046715765084168713, |
|
"loss": 2.7161, |
|
"theoretical_loss": 3.667033523975983, |
|
"tokens_seen": 950272000 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.00046633014942311325, |
|
"loss": 2.6986, |
|
"theoretical_loss": 3.66582144666085, |
|
"tokens_seen": 953548800 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0004655026480045394, |
|
"loss": 2.7021, |
|
"theoretical_loss": 3.664614689128546, |
|
"tokens_seen": 956825600 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0004646751465859656, |
|
"loss": 2.6985, |
|
"theoretical_loss": 3.6634132099317886, |
|
"tokens_seen": 960102400 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0004638476451673917, |
|
"loss": 2.7076, |
|
"theoretical_loss": 3.662216968086267, |
|
"tokens_seen": 963379200 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00046302014374881783, |
|
"loss": 2.7369, |
|
"theoretical_loss": 3.6610259230639217, |
|
"tokens_seen": 966656000 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.000462192642330244, |
|
"loss": 2.7711, |
|
"theoretical_loss": 3.659840034786333, |
|
"tokens_seen": 969932800 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0004613651409116701, |
|
"loss": 2.8314, |
|
"theoretical_loss": 3.6586592636182376, |
|
"tokens_seen": 973209600 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0004605541895214677, |
|
"loss": 2.8239, |
|
"theoretical_loss": 3.6574835703611566, |
|
"tokens_seen": 976486400 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00045972668810289393, |
|
"loss": 2.8097, |
|
"theoretical_loss": 3.6563129162471313, |
|
"tokens_seen": 979763200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.4286046663919377, |
|
"debugging/distinct-1-grams": 0.8147567798871364, |
|
"debugging/distinct-2-grams": 0.9823269374342457, |
|
"debugging/entropy-1-grams": 6.1671920556004824, |
|
"debugging/entropy-2-grams": 6.947028138756313, |
|
"debugging/length": 477.53333333333336, |
|
"debugging/num_segments": 15, |
|
"epoch": 0.35, |
|
"objective/train/avg_token_score": 0.020611366257071495, |
|
"objective/train/avg_weight": 0.9834998250007629, |
|
"objective/train/docs_used": 560408, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.9286718368530273, |
|
"objective/train/std_weight": 0.0680035874247551, |
|
"objective/train/theoretical_loss": 3.6551472629325787, |
|
"objective/train/tokens_used": 1003500000, |
|
"theoretical_loss": 3.6551472629325787, |
|
"tokens_seen": 983040000 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00045889918668432005, |
|
"loss": 2.8094, |
|
"theoretical_loss": 3.6551472629325787, |
|
"tokens_seen": 983040000 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00045807168526574616, |
|
"loss": 2.8104, |
|
"theoretical_loss": 3.653986572492247, |
|
"tokens_seen": 986316800 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0004572441838471723, |
|
"loss": 2.7787, |
|
"theoretical_loss": 3.65283080741328, |
|
"tokens_seen": 989593600 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0004564166824285984, |
|
"loss": 2.7906, |
|
"theoretical_loss": 3.6516799305893866, |
|
"tokens_seen": 992870400 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0004555891810100246, |
|
"loss": 2.7836, |
|
"theoretical_loss": 3.6505339053151076, |
|
"tokens_seen": 996147200 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.00045476167959145075, |
|
"loss": 2.7921, |
|
"theoretical_loss": 3.649392695280186, |
|
"tokens_seen": 999424000 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.00045393417817287686, |
|
"loss": 2.7558, |
|
"theoretical_loss": 3.6482562645640337, |
|
"tokens_seen": 1002700800 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.00045310667675430304, |
|
"loss": 2.761, |
|
"theoretical_loss": 3.6471245776302883, |
|
"tokens_seen": 1005977600 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.00045227917533572916, |
|
"loss": 2.7962, |
|
"theoretical_loss": 3.6459975993214724, |
|
"tokens_seen": 1009254400 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0004514516739171553, |
|
"loss": 2.7592, |
|
"theoretical_loss": 3.6448752948537377, |
|
"tokens_seen": 1012531200 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0004506241724985814, |
|
"loss": 2.7952, |
|
"theoretical_loss": 3.6437576298116996, |
|
"tokens_seen": 1015808000 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0004497966710800076, |
|
"loss": 2.7877, |
|
"theoretical_loss": 3.6426445701433607, |
|
"tokens_seen": 1019084800 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00044896916966143374, |
|
"loss": 2.8108, |
|
"theoretical_loss": 3.6415360821551226, |
|
"tokens_seen": 1022361600 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00044814166824285986, |
|
"loss": 2.7629, |
|
"theoretical_loss": 3.6404321325068754, |
|
"tokens_seen": 1025638400 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.000447314166824286, |
|
"loss": 2.7842, |
|
"theoretical_loss": 3.639332688207178, |
|
"tokens_seen": 1028915200 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0004464866654057121, |
|
"loss": 2.7649, |
|
"theoretical_loss": 3.6382377166085096, |
|
"tokens_seen": 1032192000 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0004456591639871382, |
|
"loss": 2.7244, |
|
"theoretical_loss": 3.6371471854026147, |
|
"tokens_seen": 1035468800 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0004448316625685644, |
|
"loss": 2.7313, |
|
"theoretical_loss": 3.6360610626159087, |
|
"tokens_seen": 1038745600 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00044400416114999055, |
|
"loss": 2.6993, |
|
"theoretical_loss": 3.634979316604973, |
|
"tokens_seen": 1042022400 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0004431766597314167, |
|
"loss": 2.714, |
|
"theoretical_loss": 3.6339019160521198, |
|
"tokens_seen": 1045299200 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00044234915831284285, |
|
"loss": 2.7069, |
|
"theoretical_loss": 3.632828829961029, |
|
"tokens_seen": 1048576000 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.00044152165689426896, |
|
"loss": 2.7232, |
|
"theoretical_loss": 3.631760027652461, |
|
"tokens_seen": 1051852800 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0004406941554756951, |
|
"loss": 2.7506, |
|
"theoretical_loss": 3.630695478760034, |
|
"tokens_seen": 1055129600 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0004398666540571212, |
|
"loss": 2.7249, |
|
"theoretical_loss": 3.6296351532260767, |
|
"tokens_seen": 1058406400 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0004390391526385474, |
|
"loss": 2.7194, |
|
"theoretical_loss": 3.6285790212975435, |
|
"tokens_seen": 1061683200 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.00043821165121997355, |
|
"loss": 2.6962, |
|
"theoretical_loss": 3.6275270535220008, |
|
"tokens_seen": 1064960000 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.00043738414980139966, |
|
"loss": 2.7203, |
|
"theoretical_loss": 3.626479220743673, |
|
"tokens_seen": 1068236800 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0004365566483828258, |
|
"loss": 2.7303, |
|
"theoretical_loss": 3.6254354940995586, |
|
"tokens_seen": 1071513600 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0004357291469642519, |
|
"loss": 2.7215, |
|
"theoretical_loss": 3.624395845015602, |
|
"tokens_seen": 1074790400 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.00043490164554567807, |
|
"loss": 2.6763, |
|
"theoretical_loss": 3.6233602452029348, |
|
"tokens_seen": 1078067200 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.00043407414412710424, |
|
"loss": 2.6939, |
|
"theoretical_loss": 3.6223286666541683, |
|
"tokens_seen": 1081344000 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.00043324664270853036, |
|
"loss": 2.7216, |
|
"theoretical_loss": 3.621301081639753, |
|
"tokens_seen": 1084620800 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0004324191412899565, |
|
"loss": 2.7593, |
|
"theoretical_loss": 3.6202774627043923, |
|
"tokens_seen": 1087897600 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.00043159163987138265, |
|
"loss": 2.7155, |
|
"theoretical_loss": 3.619257782663513, |
|
"tokens_seen": 1091174400 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.00043078068848118023, |
|
"loss": 2.7244, |
|
"theoretical_loss": 3.618242014599793, |
|
"tokens_seen": 1094451200 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.00042995318706260635, |
|
"loss": 2.7124, |
|
"theoretical_loss": 3.617230131859743, |
|
"tokens_seen": 1097728000 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.00042912568564403247, |
|
"loss": 2.6959, |
|
"theoretical_loss": 3.6162221080503416, |
|
"tokens_seen": 1101004800 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0004282981842254587, |
|
"loss": 2.6816, |
|
"theoretical_loss": 3.615217917035726, |
|
"tokens_seen": 1104281600 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0004274706828068848, |
|
"loss": 2.7135, |
|
"theoretical_loss": 3.614217532933929, |
|
"tokens_seen": 1107558400 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00042664318138831093, |
|
"loss": 2.7048, |
|
"theoretical_loss": 3.6132209301136715, |
|
"tokens_seen": 1110835200 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00042581567996973705, |
|
"loss": 2.6877, |
|
"theoretical_loss": 3.612228083191205, |
|
"tokens_seen": 1114112000 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0004249881785511632, |
|
"loss": 2.7443, |
|
"theoretical_loss": 3.611238967027199, |
|
"tokens_seen": 1117388800 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00042416067713258934, |
|
"loss": 2.7844, |
|
"theoretical_loss": 3.610253556723679, |
|
"tokens_seen": 1120665600 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0004233331757140155, |
|
"loss": 2.7489, |
|
"theoretical_loss": 3.609271827621014, |
|
"tokens_seen": 1123942400 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0004225056742954417, |
|
"loss": 2.7523, |
|
"theoretical_loss": 3.6082937552949463, |
|
"tokens_seen": 1127219200 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0004216781728768678, |
|
"loss": 2.7191, |
|
"theoretical_loss": 3.607319315553669, |
|
"tokens_seen": 1130496000 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0004208506714582939, |
|
"loss": 2.6832, |
|
"theoretical_loss": 3.6063484844349456, |
|
"tokens_seen": 1133772800 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00042002317003972004, |
|
"loss": 2.7085, |
|
"theoretical_loss": 3.605381238203279, |
|
"tokens_seen": 1137049600 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00041919566862114616, |
|
"loss": 2.7165, |
|
"theoretical_loss": 3.604417553347117, |
|
"tokens_seen": 1140326400 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.0004183681672025724, |
|
"loss": 2.7073, |
|
"theoretical_loss": 3.603457406576106, |
|
"tokens_seen": 1143603200 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"objective/train/avg_token_score": 0.022877871990203857, |
|
"objective/train/avg_weight": 0.9816967248916626, |
|
"objective/train/docs_used": 649861, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.648519992828369, |
|
"objective/train/std_weight": 0.07637551426887512, |
|
"objective/train/theoretical_loss": 3.602500774818379, |
|
"objective/train/tokens_used": 1167340000, |
|
"theoretical_loss": 3.602500774818379, |
|
"tokens_seen": 1146880000 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.0004175406657839985, |
|
"loss": 2.6952, |
|
"theoretical_loss": 3.602500774818379, |
|
"tokens_seen": 1146880000 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.0004167131643654246, |
|
"loss": 2.7354, |
|
"theoretical_loss": 3.601547635217892, |
|
"tokens_seen": 1150156800 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00041588566294685074, |
|
"loss": 2.7348, |
|
"theoretical_loss": 3.6005979651317976, |
|
"tokens_seen": 1153433600 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00041505816152827686, |
|
"loss": 2.7431, |
|
"theoretical_loss": 3.599651742127855, |
|
"tokens_seen": 1156710400 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00041423066010970303, |
|
"loss": 2.6952, |
|
"theoretical_loss": 3.5987089439818805, |
|
"tokens_seen": 1159987200 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00041340315869112915, |
|
"loss": 2.6743, |
|
"theoretical_loss": 3.5977695486752426, |
|
"tokens_seen": 1163264000 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0004125756572725553, |
|
"loss": 2.709, |
|
"theoretical_loss": 3.596833534392379, |
|
"tokens_seen": 1166540800 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0004117481558539815, |
|
"loss": 2.7274, |
|
"theoretical_loss": 3.595900879518368, |
|
"tokens_seen": 1169817600 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0004109206544354076, |
|
"loss": 2.7434, |
|
"theoretical_loss": 3.594971562636521, |
|
"tokens_seen": 1173094400 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00041009315301683373, |
|
"loss": 2.6976, |
|
"theoretical_loss": 3.5940455625260226, |
|
"tokens_seen": 1176371200 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00040926565159825985, |
|
"loss": 2.7078, |
|
"theoretical_loss": 3.5931228581595938, |
|
"tokens_seen": 1179648000 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00040843815017968597, |
|
"loss": 2.7067, |
|
"theoretical_loss": 3.5922034287011995, |
|
"tokens_seen": 1182924800 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0004076106487611122, |
|
"loss": 2.6717, |
|
"theoretical_loss": 3.5912872535037828, |
|
"tokens_seen": 1186201600 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0004067831473425383, |
|
"loss": 2.7545, |
|
"theoretical_loss": 3.590374312107035, |
|
"tokens_seen": 1189478400 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.00040595564592396443, |
|
"loss": 2.7398, |
|
"theoretical_loss": 3.5894645842351993, |
|
"tokens_seen": 1192755200 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.00040512814450539055, |
|
"loss": 2.7241, |
|
"theoretical_loss": 3.588558049794902, |
|
"tokens_seen": 1196032000 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.0004043006430868167, |
|
"loss": 2.7036, |
|
"theoretical_loss": 3.5876546888730187, |
|
"tokens_seen": 1199308800 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.00040347314166824284, |
|
"loss": 2.7239, |
|
"theoretical_loss": 3.5867544817345713, |
|
"tokens_seen": 1202585600 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.000402645640249669, |
|
"loss": 2.7503, |
|
"theoretical_loss": 3.585857408820652, |
|
"tokens_seen": 1205862400 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.00040181813883109513, |
|
"loss": 2.7573, |
|
"theoretical_loss": 3.58496345074638, |
|
"tokens_seen": 1209139200 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.0004009906374125213, |
|
"loss": 2.7716, |
|
"theoretical_loss": 3.5840725882988873, |
|
"tokens_seen": 1212416000 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.0004001631359939474, |
|
"loss": 2.7395, |
|
"theoretical_loss": 3.5831848024353317, |
|
"tokens_seen": 1215692800 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00039933563457537354, |
|
"loss": 2.7634, |
|
"theoretical_loss": 3.5823000742809374, |
|
"tokens_seen": 1218969600 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00039850813315679966, |
|
"loss": 2.7348, |
|
"theoretical_loss": 3.5814183851270673, |
|
"tokens_seen": 1222246400 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0003976806317382258, |
|
"loss": 2.7239, |
|
"theoretical_loss": 3.5805397164293167, |
|
"tokens_seen": 1225523200 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.000396853130319652, |
|
"loss": 2.7059, |
|
"theoretical_loss": 3.5796640498056407, |
|
"tokens_seen": 1228800000 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0003960256289010781, |
|
"loss": 2.7119, |
|
"theoretical_loss": 3.5787913670345013, |
|
"tokens_seen": 1232076800 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00039519812748250424, |
|
"loss": 2.6905, |
|
"theoretical_loss": 3.577921650053045, |
|
"tokens_seen": 1235353600 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00039437062606393036, |
|
"loss": 2.687, |
|
"theoretical_loss": 3.577054880955303, |
|
"tokens_seen": 1238630400 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00039354312464535653, |
|
"loss": 2.6705, |
|
"theoretical_loss": 3.5761910419904193, |
|
"tokens_seen": 1241907200 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00039271562322678265, |
|
"loss": 2.7329, |
|
"theoretical_loss": 3.5753301155609014, |
|
"tokens_seen": 1245184000 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.0003918881218082088, |
|
"loss": 2.6739, |
|
"theoretical_loss": 3.574472084220896, |
|
"tokens_seen": 1248460800 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00039106062038963494, |
|
"loss": 2.6639, |
|
"theoretical_loss": 3.5736169306744885, |
|
"tokens_seen": 1251737600 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.0003902331189710611, |
|
"loss": 2.6337, |
|
"theoretical_loss": 3.572764637774024, |
|
"tokens_seen": 1255014400 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00038940561755248723, |
|
"loss": 2.6435, |
|
"theoretical_loss": 3.571915188518457, |
|
"tokens_seen": 1258291200 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00038857811613391335, |
|
"loss": 2.662, |
|
"theoretical_loss": 3.571068566051716, |
|
"tokens_seen": 1261568000 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00038775061471533947, |
|
"loss": 2.704, |
|
"theoretical_loss": 3.5702247536610976, |
|
"tokens_seen": 1264844800 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.0003869231132967657, |
|
"loss": 2.6831, |
|
"theoretical_loss": 3.5693837347756783, |
|
"tokens_seen": 1268121600 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.0003860956118781918, |
|
"loss": 2.6552, |
|
"theoretical_loss": 3.5685454929647475, |
|
"tokens_seen": 1271398400 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00038526811045961793, |
|
"loss": 2.6671, |
|
"theoretical_loss": 3.5677100119362675, |
|
"tokens_seen": 1274675200 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00038444060904104405, |
|
"loss": 2.6461, |
|
"theoretical_loss": 3.566877275535345, |
|
"tokens_seen": 1277952000 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00038361310762247017, |
|
"loss": 2.6451, |
|
"theoretical_loss": 3.566047267742733, |
|
"tokens_seen": 1281228800 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00038278560620389634, |
|
"loss": 2.6221, |
|
"theoretical_loss": 3.5652199726733453, |
|
"tokens_seen": 1284505600 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0003819581047853225, |
|
"loss": 2.6172, |
|
"theoretical_loss": 3.564395374574796, |
|
"tokens_seen": 1287782400 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00038113060336674863, |
|
"loss": 2.6767, |
|
"theoretical_loss": 3.5635734578259557, |
|
"tokens_seen": 1291059200 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0003803031019481748, |
|
"loss": 2.6908, |
|
"theoretical_loss": 3.5627542069355282, |
|
"tokens_seen": 1294336000 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0003794756005296009, |
|
"loss": 2.6578, |
|
"theoretical_loss": 3.5619376065406474, |
|
"tokens_seen": 1297612800 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00037864809911102704, |
|
"loss": 2.6722, |
|
"theoretical_loss": 3.5611236414054868, |
|
"tokens_seen": 1300889600 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00037782059769245316, |
|
"loss": 2.6776, |
|
"theoretical_loss": 3.560312296419899, |
|
"tokens_seen": 1304166400 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.0003769930962738793, |
|
"loss": 2.6728, |
|
"theoretical_loss": 3.55950355659806, |
|
"tokens_seen": 1307443200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.49020908264157476, |
|
"debugging/distinct-1-grams": 0.768901113497886, |
|
"debugging/distinct-2-grams": 0.9428782333551957, |
|
"debugging/entropy-1-grams": 6.085999550681761, |
|
"debugging/entropy-2-grams": 7.0033060167714964, |
|
"debugging/length": 490.2352941176471, |
|
"debugging/num_segments": 17, |
|
"epoch": 0.47, |
|
"objective/train/avg_token_score": 0.02056093141436577, |
|
"objective/train/avg_weight": 0.983538031578064, |
|
"objective/train/docs_used": 741674, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.6528468132019043, |
|
"objective/train/std_weight": 0.0878894254565239, |
|
"objective/train/theoretical_loss": 3.558697407077142, |
|
"objective/train/tokens_used": 1331180000, |
|
"theoretical_loss": 3.558697407077142, |
|
"tokens_seen": 1310720000 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.0003761655948553055, |
|
"loss": 2.6652, |
|
"theoretical_loss": 3.558697407077142, |
|
"tokens_seen": 1310720000 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.0003753380934367316, |
|
"loss": 2.6785, |
|
"theoretical_loss": 3.5578938331159975, |
|
"tokens_seen": 1313996800 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00037451059201815774, |
|
"loss": 2.6523, |
|
"theoretical_loss": 3.557092820093863, |
|
"tokens_seen": 1317273600 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00037368309059958386, |
|
"loss": 2.6731, |
|
"theoretical_loss": 3.556294353509079, |
|
"tokens_seen": 1320550400 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00037285558918101, |
|
"loss": 2.6639, |
|
"theoretical_loss": 3.555498418977828, |
|
"tokens_seen": 1323827200 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00037202808776243615, |
|
"loss": 2.6579, |
|
"theoretical_loss": 3.5547050022328874, |
|
"tokens_seen": 1327104000 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.0003712005863438623, |
|
"loss": 2.6554, |
|
"theoretical_loss": 3.553914089122399, |
|
"tokens_seen": 1330380800 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00037037308492528844, |
|
"loss": 2.7062, |
|
"theoretical_loss": 3.553125665608655, |
|
"tokens_seen": 1333657600 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.0003695455835067146, |
|
"loss": 2.6814, |
|
"theoretical_loss": 3.5523397177669005, |
|
"tokens_seen": 1336934400 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00036871808208814073, |
|
"loss": 2.691, |
|
"theoretical_loss": 3.551556231784149, |
|
"tokens_seen": 1340211200 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00036789058066956685, |
|
"loss": 2.6452, |
|
"theoretical_loss": 3.5507751939580148, |
|
"tokens_seen": 1343488000 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00036706307925099297, |
|
"loss": 2.646, |
|
"theoretical_loss": 3.5499965906955606, |
|
"tokens_seen": 1346764800 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.0003662355778324192, |
|
"loss": 2.6952, |
|
"theoretical_loss": 3.549220408512161, |
|
"tokens_seen": 1350041600 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.0003654080764138453, |
|
"loss": 2.6716, |
|
"theoretical_loss": 3.5484466340303755, |
|
"tokens_seen": 1353318400 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00036458057499527143, |
|
"loss": 2.6566, |
|
"theoretical_loss": 3.547675253978843, |
|
"tokens_seen": 1356595200 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00036375307357669755, |
|
"loss": 2.6947, |
|
"theoretical_loss": 3.5469062551911854, |
|
"tokens_seen": 1359872000 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00036292557215812367, |
|
"loss": 2.7012, |
|
"theoretical_loss": 3.5461396246049244, |
|
"tokens_seen": 1363148800 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.0003620980707395498, |
|
"loss": 2.6675, |
|
"theoretical_loss": 3.545375349260419, |
|
"tokens_seen": 1366425600 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00036127056932097596, |
|
"loss": 2.7088, |
|
"theoretical_loss": 3.544613416299808, |
|
"tokens_seen": 1369702400 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00036044306790240213, |
|
"loss": 2.6847, |
|
"theoretical_loss": 3.5438538129659687, |
|
"tokens_seen": 1372979200 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00035961556648382825, |
|
"loss": 2.7015, |
|
"theoretical_loss": 3.5430965266014933, |
|
"tokens_seen": 1376256000 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.0003587880650652544, |
|
"loss": 2.7051, |
|
"theoretical_loss": 3.5423415446476705, |
|
"tokens_seen": 1379532800 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00035796056364668054, |
|
"loss": 2.6407, |
|
"theoretical_loss": 3.541588854643487, |
|
"tokens_seen": 1382809600 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.00035713306222810666, |
|
"loss": 2.6672, |
|
"theoretical_loss": 3.5408384442246343, |
|
"tokens_seen": 1386086400 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.0003563055608095328, |
|
"loss": 2.623, |
|
"theoretical_loss": 3.540090301122535, |
|
"tokens_seen": 1389363200 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.000355478059390959, |
|
"loss": 2.6465, |
|
"theoretical_loss": 3.5393444131633762, |
|
"tokens_seen": 1392640000 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.0003546505579723851, |
|
"loss": 2.696, |
|
"theoretical_loss": 3.5386007682671576, |
|
"tokens_seen": 1395916800 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.00035382305655381124, |
|
"loss": 2.6751, |
|
"theoretical_loss": 3.5378593544467494, |
|
"tokens_seen": 1399193600 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.00035299555513523736, |
|
"loss": 2.6564, |
|
"theoretical_loss": 3.537120159806962, |
|
"tokens_seen": 1402470400 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.0003521680537166635, |
|
"loss": 2.6455, |
|
"theoretical_loss": 3.5363831725436317, |
|
"tokens_seen": 1405747200 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.00035134055229808965, |
|
"loss": 2.6179, |
|
"theoretical_loss": 3.5356483809427104, |
|
"tokens_seen": 1409024000 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.0003505130508795158, |
|
"loss": 2.6164, |
|
"theoretical_loss": 3.53491577337937, |
|
"tokens_seen": 1412300800 |
|
}, |
|
{ |
|
"epoch": 0.51, |
|
"learning_rate": 0.00034968554946094194, |
|
"loss": 2.6266, |
|
"theoretical_loss": 3.5341853383171205, |
|
"tokens_seen": 1415577600 |
|
}, |
|
{ |
|
"epoch": 0.51, |
|
"learning_rate": 0.00034885804804236806, |
|
"loss": 2.6593, |
|
"theoretical_loss": 3.5334570643069325, |
|
"tokens_seen": 1418854400 |
|
}, |
|
{ |
|
"epoch": 0.51, |
|
"learning_rate": 0.00034803054662379423, |
|
"loss": 2.6381, |
|
"theoretical_loss": 3.5327309399863767, |
|
"tokens_seen": 1422131200 |
|
}, |
|
{ |
|
"epoch": 0.51, |
|
"learning_rate": 0.00034720304520522035, |
|
"loss": 2.6604, |
|
"theoretical_loss": 3.5320069540787706, |
|
"tokens_seen": 1425408000 |
|
}, |
|
{ |
|
"epoch": 0.51, |
|
"learning_rate": 0.0003463755437866465, |
|
"loss": 2.6229, |
|
"theoretical_loss": 3.531285095392335, |
|
"tokens_seen": 1428684800 |
|
}, |
|
{ |
|
"epoch": 0.51, |
|
"learning_rate": 0.00034554804236807264, |
|
"loss": 2.641, |
|
"theoretical_loss": 3.5305653528193623, |
|
"tokens_seen": 1431961600 |
|
}, |
|
{ |
|
"epoch": 0.51, |
|
"learning_rate": 0.00034472054094949876, |
|
"loss": 2.6146, |
|
"theoretical_loss": 3.5298477153353955, |
|
"tokens_seen": 1435238400 |
|
}, |
|
{ |
|
"epoch": 0.51, |
|
"learning_rate": 0.00034389303953092493, |
|
"loss": 2.5814, |
|
"theoretical_loss": 3.5291321719984134, |
|
"tokens_seen": 1438515200 |
|
}, |
|
{ |
|
"epoch": 0.51, |
|
"learning_rate": 0.0003430820881407225, |
|
"loss": 2.5997, |
|
"theoretical_loss": 3.5284187119480315, |
|
"tokens_seen": 1441792000 |
|
}, |
|
{ |
|
"epoch": 0.52, |
|
"learning_rate": 0.00034225458672214863, |
|
"loss": 2.5815, |
|
"theoretical_loss": 3.527707324404704, |
|
"tokens_seen": 1445068800 |
|
}, |
|
{ |
|
"epoch": 0.52, |
|
"learning_rate": 0.0003414270853035748, |
|
"loss": 2.6071, |
|
"theoretical_loss": 3.5269979986689464, |
|
"tokens_seen": 1448345600 |
|
}, |
|
{ |
|
"epoch": 0.52, |
|
"learning_rate": 0.000340599583885001, |
|
"loss": 2.6161, |
|
"theoretical_loss": 3.5262907241205568, |
|
"tokens_seen": 1451622400 |
|
}, |
|
{ |
|
"epoch": 0.52, |
|
"learning_rate": 0.0003397720824664271, |
|
"loss": 2.5943, |
|
"theoretical_loss": 3.5255854902178534, |
|
"tokens_seen": 1454899200 |
|
}, |
|
{ |
|
"epoch": 0.52, |
|
"learning_rate": 0.00033894458104785326, |
|
"loss": 2.5878, |
|
"theoretical_loss": 3.524882286496917, |
|
"tokens_seen": 1458176000 |
|
}, |
|
{ |
|
"epoch": 0.52, |
|
"learning_rate": 0.0003381170796292794, |
|
"loss": 2.6206, |
|
"theoretical_loss": 3.524181102570846, |
|
"tokens_seen": 1461452800 |
|
}, |
|
{ |
|
"epoch": 0.52, |
|
"learning_rate": 0.0003372895782107055, |
|
"loss": 2.6115, |
|
"theoretical_loss": 3.5234819281290166, |
|
"tokens_seen": 1464729600 |
|
}, |
|
{ |
|
"epoch": 0.52, |
|
"learning_rate": 0.0003364620767921316, |
|
"loss": 2.6357, |
|
"theoretical_loss": 3.5227847529363547, |
|
"tokens_seen": 1468006400 |
|
}, |
|
{ |
|
"epoch": 0.53, |
|
"learning_rate": 0.0003356345753735578, |
|
"loss": 2.6296, |
|
"theoretical_loss": 3.522089566832615, |
|
"tokens_seen": 1471283200 |
|
}, |
|
{ |
|
"epoch": 0.53, |
|
"objective/train/avg_token_score": 0.03047107346355915, |
|
"objective/train/avg_weight": 0.9756120443344116, |
|
"objective/train/docs_used": 835915, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.7272753715515137, |
|
"objective/train/std_weight": 0.09095926582813263, |
|
"objective/train/theoretical_loss": 3.521396359731668, |
|
"objective/train/tokens_used": 1495020000, |
|
"theoretical_loss": 3.521396359731668, |
|
"tokens_seen": 1474560000 |
|
}, |
|
{ |
|
"epoch": 0.53, |
|
"learning_rate": 0.0003348070739549839, |
|
"loss": 2.6236, |
|
"theoretical_loss": 3.521396359731668, |
|
"tokens_seen": 1474560000 |
|
}, |
|
{ |
|
"epoch": 0.53, |
|
"learning_rate": 0.00033397957253641003, |
|
"loss": 2.6548, |
|
"theoretical_loss": 3.520705121620797, |
|
"tokens_seen": 1477836800 |
|
}, |
|
{ |
|
"epoch": 0.53, |
|
"learning_rate": 0.0003331520711178362, |
|
"loss": 2.6266, |
|
"theoretical_loss": 3.520015842560001, |
|
"tokens_seen": 1481113600 |
|
}, |
|
{ |
|
"epoch": 0.53, |
|
"learning_rate": 0.0003323245696992623, |
|
"loss": 2.6482, |
|
"theoretical_loss": 3.5193285126813105, |
|
"tokens_seen": 1484390400 |
|
}, |
|
{ |
|
"epoch": 0.53, |
|
"learning_rate": 0.0003314970682806885, |
|
"loss": 2.6722, |
|
"theoretical_loss": 3.5186431221881023, |
|
"tokens_seen": 1487667200 |
|
}, |
|
{ |
|
"epoch": 0.53, |
|
"learning_rate": 0.0003306695668621146, |
|
"loss": 2.6795, |
|
"theoretical_loss": 3.517959661354433, |
|
"tokens_seen": 1490944000 |
|
}, |
|
{ |
|
"epoch": 0.53, |
|
"learning_rate": 0.0003298420654435408, |
|
"loss": 2.6729, |
|
"theoretical_loss": 3.517278120524372, |
|
"tokens_seen": 1494220800 |
|
}, |
|
{ |
|
"epoch": 0.53, |
|
"learning_rate": 0.0003290145640249669, |
|
"loss": 2.6819, |
|
"theoretical_loss": 3.516598490111348, |
|
"tokens_seen": 1497497600 |
|
}, |
|
{ |
|
"epoch": 0.54, |
|
"learning_rate": 0.00032818706260639307, |
|
"loss": 2.6962, |
|
"theoretical_loss": 3.515920760597498, |
|
"tokens_seen": 1500774400 |
|
}, |
|
{ |
|
"epoch": 0.54, |
|
"learning_rate": 0.0003273595611878192, |
|
"loss": 2.6508, |
|
"theoretical_loss": 3.515244922533025, |
|
"tokens_seen": 1504051200 |
|
}, |
|
{ |
|
"epoch": 0.54, |
|
"learning_rate": 0.0003265320597692453, |
|
"loss": 2.6405, |
|
"theoretical_loss": 3.514570966535569, |
|
"tokens_seen": 1507328000 |
|
}, |
|
{ |
|
"epoch": 0.54, |
|
"learning_rate": 0.0003257045583506715, |
|
"loss": 2.683, |
|
"theoretical_loss": 3.513898883289575, |
|
"tokens_seen": 1510604800 |
|
}, |
|
{ |
|
"epoch": 0.54, |
|
"learning_rate": 0.0003248770569320976, |
|
"loss": 2.6592, |
|
"theoretical_loss": 3.513228663545678, |
|
"tokens_seen": 1513881600 |
|
}, |
|
{ |
|
"epoch": 0.54, |
|
"learning_rate": 0.0003240495555135237, |
|
"loss": 2.6774, |
|
"theoretical_loss": 3.5125602981200874, |
|
"tokens_seen": 1517158400 |
|
}, |
|
{ |
|
"epoch": 0.54, |
|
"learning_rate": 0.0003232220540949499, |
|
"loss": 2.6356, |
|
"theoretical_loss": 3.5118937778939836, |
|
"tokens_seen": 1520435200 |
|
}, |
|
{ |
|
"epoch": 0.54, |
|
"learning_rate": 0.000322394552676376, |
|
"loss": 2.6391, |
|
"theoretical_loss": 3.5112290938129176, |
|
"tokens_seen": 1523712000 |
|
}, |
|
{ |
|
"epoch": 0.55, |
|
"learning_rate": 0.0003215670512578021, |
|
"loss": 2.6629, |
|
"theoretical_loss": 3.5105662368862234, |
|
"tokens_seen": 1526988800 |
|
}, |
|
{ |
|
"epoch": 0.55, |
|
"learning_rate": 0.0003207395498392283, |
|
"loss": 2.634, |
|
"theoretical_loss": 3.5099051981864275, |
|
"tokens_seen": 1530265600 |
|
}, |
|
{ |
|
"epoch": 0.55, |
|
"learning_rate": 0.0003199120484206544, |
|
"loss": 2.6155, |
|
"theoretical_loss": 3.509245968848674, |
|
"tokens_seen": 1533542400 |
|
}, |
|
{ |
|
"epoch": 0.55, |
|
"learning_rate": 0.0003190845470020806, |
|
"loss": 2.6151, |
|
"theoretical_loss": 3.5085885400701535, |
|
"tokens_seen": 1536819200 |
|
}, |
|
{ |
|
"epoch": 0.55, |
|
"learning_rate": 0.0003182570455835067, |
|
"loss": 2.5978, |
|
"theoretical_loss": 3.507932903109534, |
|
"tokens_seen": 1540096000 |
|
}, |
|
{ |
|
"epoch": 0.55, |
|
"learning_rate": 0.0003174295441649329, |
|
"loss": 2.609, |
|
"theoretical_loss": 3.5072790492864057, |
|
"tokens_seen": 1543372800 |
|
}, |
|
{ |
|
"epoch": 0.55, |
|
"learning_rate": 0.000316602042746359, |
|
"loss": 2.631, |
|
"theoretical_loss": 3.506626969980724, |
|
"tokens_seen": 1546649600 |
|
}, |
|
{ |
|
"epoch": 0.55, |
|
"learning_rate": 0.0003157745413277851, |
|
"loss": 2.6608, |
|
"theoretical_loss": 3.505976656632269, |
|
"tokens_seen": 1549926400 |
|
}, |
|
{ |
|
"epoch": 0.55, |
|
"learning_rate": 0.0003149470399092113, |
|
"loss": 2.6371, |
|
"theoretical_loss": 3.505328100740097, |
|
"tokens_seen": 1553203200 |
|
}, |
|
{ |
|
"epoch": 0.56, |
|
"learning_rate": 0.0003141195384906374, |
|
"loss": 2.6324, |
|
"theoretical_loss": 3.5046812938620135, |
|
"tokens_seen": 1556480000 |
|
}, |
|
{ |
|
"epoch": 0.56, |
|
"learning_rate": 0.0003132920370720635, |
|
"loss": 2.6286, |
|
"theoretical_loss": 3.5040362276140393, |
|
"tokens_seen": 1559756800 |
|
}, |
|
{ |
|
"epoch": 0.56, |
|
"learning_rate": 0.0003124645356534897, |
|
"loss": 2.6088, |
|
"theoretical_loss": 3.5033928936698904, |
|
"tokens_seen": 1563033600 |
|
}, |
|
{ |
|
"epoch": 0.56, |
|
"learning_rate": 0.0003116370342349158, |
|
"loss": 2.6365, |
|
"theoretical_loss": 3.50275128376046, |
|
"tokens_seen": 1566310400 |
|
}, |
|
{ |
|
"epoch": 0.56, |
|
"learning_rate": 0.00031080953281634194, |
|
"loss": 2.5913, |
|
"theoretical_loss": 3.5021113896733076, |
|
"tokens_seen": 1569587200 |
|
}, |
|
{ |
|
"epoch": 0.56, |
|
"learning_rate": 0.0003099820313977681, |
|
"loss": 2.6035, |
|
"theoretical_loss": 3.501473203252153, |
|
"tokens_seen": 1572864000 |
|
}, |
|
{ |
|
"epoch": 0.56, |
|
"learning_rate": 0.0003091545299791942, |
|
"loss": 2.6589, |
|
"theoretical_loss": 3.500836716396377, |
|
"tokens_seen": 1576140800 |
|
}, |
|
{ |
|
"epoch": 0.56, |
|
"learning_rate": 0.0003083270285606204, |
|
"loss": 2.6469, |
|
"theoretical_loss": 3.5002019210605253, |
|
"tokens_seen": 1579417600 |
|
}, |
|
{ |
|
"epoch": 0.57, |
|
"learning_rate": 0.0003074995271420465, |
|
"loss": 2.6533, |
|
"theoretical_loss": 3.4995688092538213, |
|
"tokens_seen": 1582694400 |
|
}, |
|
{ |
|
"epoch": 0.57, |
|
"learning_rate": 0.0003066720257234727, |
|
"loss": 2.6015, |
|
"theoretical_loss": 3.49893737303968, |
|
"tokens_seen": 1585971200 |
|
}, |
|
{ |
|
"epoch": 0.57, |
|
"learning_rate": 0.0003058445243048988, |
|
"loss": 2.584, |
|
"theoretical_loss": 3.4983076045352317, |
|
"tokens_seen": 1589248000 |
|
}, |
|
{ |
|
"epoch": 0.57, |
|
"learning_rate": 0.0003050170228863249, |
|
"loss": 2.5704, |
|
"theoretical_loss": 3.4976794959108464, |
|
"tokens_seen": 1592524800 |
|
}, |
|
{ |
|
"epoch": 0.57, |
|
"learning_rate": 0.0003041895214677511, |
|
"loss": 2.6315, |
|
"theoretical_loss": 3.4970530393896673, |
|
"tokens_seen": 1595801600 |
|
}, |
|
{ |
|
"epoch": 0.57, |
|
"learning_rate": 0.0003033620200491772, |
|
"loss": 2.6007, |
|
"theoretical_loss": 3.4964282272471467, |
|
"tokens_seen": 1599078400 |
|
}, |
|
{ |
|
"epoch": 0.57, |
|
"learning_rate": 0.00030253451863060333, |
|
"loss": 2.5865, |
|
"theoretical_loss": 3.495805051810588, |
|
"tokens_seen": 1602355200 |
|
}, |
|
{ |
|
"epoch": 0.57, |
|
"learning_rate": 0.0003017070172120295, |
|
"loss": 2.6131, |
|
"theoretical_loss": 3.4951835054586926, |
|
"tokens_seen": 1605632000 |
|
}, |
|
{ |
|
"epoch": 0.57, |
|
"learning_rate": 0.0003008795157934556, |
|
"loss": 2.5888, |
|
"theoretical_loss": 3.4945635806211115, |
|
"tokens_seen": 1608908800 |
|
}, |
|
{ |
|
"epoch": 0.58, |
|
"learning_rate": 0.00030005201437488174, |
|
"loss": 2.6327, |
|
"theoretical_loss": 3.493945269778002, |
|
"tokens_seen": 1612185600 |
|
}, |
|
{ |
|
"epoch": 0.58, |
|
"learning_rate": 0.0002992245129563079, |
|
"loss": 2.6377, |
|
"theoretical_loss": 3.4933285654595867, |
|
"tokens_seen": 1615462400 |
|
}, |
|
{ |
|
"epoch": 0.58, |
|
"learning_rate": 0.00029839701153773403, |
|
"loss": 2.6268, |
|
"theoretical_loss": 3.4927134602457244, |
|
"tokens_seen": 1618739200 |
|
}, |
|
{ |
|
"epoch": 0.58, |
|
"learning_rate": 0.0002975695101191602, |
|
"loss": 2.632, |
|
"theoretical_loss": 3.4920999467654754, |
|
"tokens_seen": 1622016000 |
|
}, |
|
{ |
|
"epoch": 0.58, |
|
"learning_rate": 0.0002967420087005864, |
|
"loss": 2.6142, |
|
"theoretical_loss": 3.49148801769668, |
|
"tokens_seen": 1625292800 |
|
}, |
|
{ |
|
"epoch": 0.58, |
|
"learning_rate": 0.0002959145072820125, |
|
"loss": 2.6468, |
|
"theoretical_loss": 3.4908776657655354, |
|
"tokens_seen": 1628569600 |
|
}, |
|
{ |
|
"epoch": 0.58, |
|
"learning_rate": 0.0002950870058634386, |
|
"loss": 2.5539, |
|
"theoretical_loss": 3.4902688837461824, |
|
"tokens_seen": 1631846400 |
|
}, |
|
{ |
|
"epoch": 0.58, |
|
"learning_rate": 0.0002942595044448648, |
|
"loss": 2.574, |
|
"theoretical_loss": 3.489661664460292, |
|
"tokens_seen": 1635123200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.4862537472725578, |
|
"debugging/distinct-1-grams": 0.7771784690107355, |
|
"debugging/distinct-2-grams": 0.9668669456652452, |
|
"debugging/entropy-1-grams": 6.202714981817353, |
|
"debugging/entropy-2-grams": 7.211883548596637, |
|
"debugging/length": 471.0, |
|
"debugging/num_segments": 21, |
|
"epoch": 0.59, |
|
"objective/train/avg_token_score": 0.013972360640764236, |
|
"objective/train/avg_weight": 0.9888162612915039, |
|
"objective/train/docs_used": 927402, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.8989129066467285, |
|
"objective/train/std_weight": 0.05004078894853592, |
|
"objective/train/theoretical_loss": 3.4890560007766607, |
|
"objective/train/tokens_used": 1658860000, |
|
"theoretical_loss": 3.4890560007766607, |
|
"tokens_seen": 1638400000 |
|
}, |
|
{ |
|
"epoch": 0.59, |
|
"learning_rate": 0.0002934320030262909, |
|
"loss": 2.5711, |
|
"theoretical_loss": 3.4890560007766607, |
|
"tokens_seen": 1638400000 |
|
}, |
|
{ |
|
"epoch": 0.59, |
|
"learning_rate": 0.000292604501607717, |
|
"loss": 2.5866, |
|
"theoretical_loss": 3.4884518856108038, |
|
"tokens_seen": 1641676800 |
|
}, |
|
{ |
|
"epoch": 0.59, |
|
"learning_rate": 0.0002917770001891432, |
|
"loss": 2.5923, |
|
"theoretical_loss": 3.4878493119245606, |
|
"tokens_seen": 1644953600 |
|
}, |
|
{ |
|
"epoch": 0.59, |
|
"learning_rate": 0.0002909494987705693, |
|
"loss": 2.5891, |
|
"theoretical_loss": 3.4872482727256964, |
|
"tokens_seen": 1648230400 |
|
}, |
|
{ |
|
"epoch": 0.59, |
|
"learning_rate": 0.00029012199735199543, |
|
"loss": 2.6069, |
|
"theoretical_loss": 3.4866487610675163, |
|
"tokens_seen": 1651507200 |
|
}, |
|
{ |
|
"epoch": 0.59, |
|
"learning_rate": 0.0002892944959334216, |
|
"loss": 2.5879, |
|
"theoretical_loss": 3.4860507700484757, |
|
"tokens_seen": 1654784000 |
|
}, |
|
{ |
|
"epoch": 0.59, |
|
"learning_rate": 0.0002884669945148477, |
|
"loss": 2.545, |
|
"theoretical_loss": 3.4854542928117986, |
|
"tokens_seen": 1658060800 |
|
}, |
|
{ |
|
"epoch": 0.59, |
|
"learning_rate": 0.0002876394930962739, |
|
"loss": 2.5565, |
|
"theoretical_loss": 3.484859322545101, |
|
"tokens_seen": 1661337600 |
|
}, |
|
{ |
|
"epoch": 0.59, |
|
"learning_rate": 0.0002868119916777, |
|
"loss": 2.5784, |
|
"theoretical_loss": 3.484265852480015, |
|
"tokens_seen": 1664614400 |
|
}, |
|
{ |
|
"epoch": 0.6, |
|
"learning_rate": 0.0002859844902591262, |
|
"loss": 2.5979, |
|
"theoretical_loss": 3.48367387589182, |
|
"tokens_seen": 1667891200 |
|
}, |
|
{ |
|
"epoch": 0.6, |
|
"learning_rate": 0.0002851569888405523, |
|
"loss": 2.6108, |
|
"theoretical_loss": 3.4830833860990733, |
|
"tokens_seen": 1671168000 |
|
}, |
|
{ |
|
"epoch": 0.6, |
|
"learning_rate": 0.0002843294874219784, |
|
"loss": 2.6149, |
|
"theoretical_loss": 3.4824943764632503, |
|
"tokens_seen": 1674444800 |
|
}, |
|
{ |
|
"epoch": 0.6, |
|
"learning_rate": 0.0002835019860034046, |
|
"loss": 2.6237, |
|
"theoretical_loss": 3.4819068403883855, |
|
"tokens_seen": 1677721600 |
|
}, |
|
{ |
|
"epoch": 0.6, |
|
"learning_rate": 0.0002826744845848307, |
|
"loss": 2.6082, |
|
"theoretical_loss": 3.481320771320714, |
|
"tokens_seen": 1680998400 |
|
}, |
|
{ |
|
"epoch": 0.6, |
|
"learning_rate": 0.00028184698316625683, |
|
"loss": 2.5942, |
|
"theoretical_loss": 3.4807361627483235, |
|
"tokens_seen": 1684275200 |
|
}, |
|
{ |
|
"epoch": 0.6, |
|
"learning_rate": 0.000281019481747683, |
|
"loss": 2.652, |
|
"theoretical_loss": 3.480153008200804, |
|
"tokens_seen": 1687552000 |
|
}, |
|
{ |
|
"epoch": 0.6, |
|
"learning_rate": 0.0002801919803291091, |
|
"loss": 2.6077, |
|
"theoretical_loss": 3.4795713012489045, |
|
"tokens_seen": 1690828800 |
|
}, |
|
{ |
|
"epoch": 0.61, |
|
"learning_rate": 0.00027936447891053524, |
|
"loss": 2.5755, |
|
"theoretical_loss": 3.478991035504191, |
|
"tokens_seen": 1694105600 |
|
}, |
|
{ |
|
"epoch": 0.61, |
|
"learning_rate": 0.0002785369774919614, |
|
"loss": 2.6501, |
|
"theoretical_loss": 3.478412204618709, |
|
"tokens_seen": 1697382400 |
|
}, |
|
{ |
|
"epoch": 0.61, |
|
"learning_rate": 0.00027770947607338753, |
|
"loss": 2.615, |
|
"theoretical_loss": 3.4778348022846517, |
|
"tokens_seen": 1700659200 |
|
}, |
|
{ |
|
"epoch": 0.61, |
|
"learning_rate": 0.0002768819746548137, |
|
"loss": 2.6458, |
|
"theoretical_loss": 3.477258822234026, |
|
"tokens_seen": 1703936000 |
|
}, |
|
{ |
|
"epoch": 0.61, |
|
"learning_rate": 0.0002760544732362398, |
|
"loss": 2.5928, |
|
"theoretical_loss": 3.4766842582383277, |
|
"tokens_seen": 1707212800 |
|
}, |
|
{ |
|
"epoch": 0.61, |
|
"learning_rate": 0.000275226971817666, |
|
"loss": 2.5872, |
|
"theoretical_loss": 3.476111104108216, |
|
"tokens_seen": 1710489600 |
|
}, |
|
{ |
|
"epoch": 0.61, |
|
"learning_rate": 0.0002743994703990921, |
|
"loss": 2.5986, |
|
"theoretical_loss": 3.4755393536931942, |
|
"tokens_seen": 1713766400 |
|
}, |
|
{ |
|
"epoch": 0.61, |
|
"learning_rate": 0.0002735719689805183, |
|
"loss": 2.6081, |
|
"theoretical_loss": 3.4749690008812903, |
|
"tokens_seen": 1717043200 |
|
}, |
|
{ |
|
"epoch": 0.61, |
|
"learning_rate": 0.0002727444675619444, |
|
"loss": 2.6058, |
|
"theoretical_loss": 3.474400039598745, |
|
"tokens_seen": 1720320000 |
|
}, |
|
{ |
|
"epoch": 0.62, |
|
"learning_rate": 0.0002719169661433705, |
|
"loss": 2.6357, |
|
"theoretical_loss": 3.473832463809698, |
|
"tokens_seen": 1723596800 |
|
}, |
|
{ |
|
"epoch": 0.62, |
|
"learning_rate": 0.00027108946472479664, |
|
"loss": 2.6347, |
|
"theoretical_loss": 3.473266267515883, |
|
"tokens_seen": 1726873600 |
|
}, |
|
{ |
|
"epoch": 0.62, |
|
"learning_rate": 0.0002702619633062228, |
|
"loss": 2.6571, |
|
"theoretical_loss": 3.4727014447563196, |
|
"tokens_seen": 1730150400 |
|
}, |
|
{ |
|
"epoch": 0.62, |
|
"learning_rate": 0.00026943446188764893, |
|
"loss": 2.6456, |
|
"theoretical_loss": 3.4721379896070133, |
|
"tokens_seen": 1733427200 |
|
}, |
|
{ |
|
"epoch": 0.62, |
|
"learning_rate": 0.00026860696046907505, |
|
"loss": 2.6561, |
|
"theoretical_loss": 3.4715758961806573, |
|
"tokens_seen": 1736704000 |
|
}, |
|
{ |
|
"epoch": 0.62, |
|
"learning_rate": 0.0002677794590505012, |
|
"loss": 2.6359, |
|
"theoretical_loss": 3.471015158626335, |
|
"tokens_seen": 1739980800 |
|
}, |
|
{ |
|
"epoch": 0.62, |
|
"learning_rate": 0.00026695195763192734, |
|
"loss": 2.6374, |
|
"theoretical_loss": 3.4704557711292265, |
|
"tokens_seen": 1743257600 |
|
}, |
|
{ |
|
"epoch": 0.62, |
|
"learning_rate": 0.0002661244562133535, |
|
"loss": 2.6482, |
|
"theoretical_loss": 3.469897727910321, |
|
"tokens_seen": 1746534400 |
|
}, |
|
{ |
|
"epoch": 0.62, |
|
"learning_rate": 0.00026529695479477963, |
|
"loss": 2.6676, |
|
"theoretical_loss": 3.4693410232261277, |
|
"tokens_seen": 1749811200 |
|
}, |
|
{ |
|
"epoch": 0.63, |
|
"learning_rate": 0.0002644694533762058, |
|
"loss": 2.6847, |
|
"theoretical_loss": 3.4687856513683917, |
|
"tokens_seen": 1753088000 |
|
}, |
|
{ |
|
"epoch": 0.63, |
|
"learning_rate": 0.0002636419519576319, |
|
"loss": 2.6594, |
|
"theoretical_loss": 3.468231606663813, |
|
"tokens_seen": 1756364800 |
|
}, |
|
{ |
|
"epoch": 0.63, |
|
"learning_rate": 0.0002628144505390581, |
|
"loss": 2.6502, |
|
"theoretical_loss": 3.4676788834737677, |
|
"tokens_seen": 1759641600 |
|
}, |
|
{ |
|
"epoch": 0.63, |
|
"learning_rate": 0.0002619869491204842, |
|
"loss": 2.6297, |
|
"theoretical_loss": 3.4671274761940314, |
|
"tokens_seen": 1762918400 |
|
}, |
|
{ |
|
"epoch": 0.63, |
|
"learning_rate": 0.00026115944770191033, |
|
"loss": 2.6097, |
|
"theoretical_loss": 3.466577379254506, |
|
"tokens_seen": 1766195200 |
|
}, |
|
{ |
|
"epoch": 0.63, |
|
"learning_rate": 0.0002603319462833365, |
|
"loss": 2.617, |
|
"theoretical_loss": 3.4660285871189496, |
|
"tokens_seen": 1769472000 |
|
}, |
|
{ |
|
"epoch": 0.63, |
|
"learning_rate": 0.0002595044448647626, |
|
"loss": 2.6377, |
|
"theoretical_loss": 3.465481094284707, |
|
"tokens_seen": 1772748800 |
|
}, |
|
{ |
|
"epoch": 0.63, |
|
"learning_rate": 0.00025867694344618874, |
|
"loss": 2.6237, |
|
"theoretical_loss": 3.4649348952824477, |
|
"tokens_seen": 1776025600 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"learning_rate": 0.0002578494420276149, |
|
"loss": 2.6087, |
|
"theoretical_loss": 3.4643899846758983, |
|
"tokens_seen": 1779302400 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"learning_rate": 0.00025702194060904103, |
|
"loss": 2.5569, |
|
"theoretical_loss": 3.4638463570615854, |
|
"tokens_seen": 1782579200 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"learning_rate": 0.00025619443919046715, |
|
"loss": 2.612, |
|
"theoretical_loss": 3.4633040070685777, |
|
"tokens_seen": 1785856000 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"learning_rate": 0.0002553669377718933, |
|
"loss": 2.6094, |
|
"theoretical_loss": 3.46276292935823, |
|
"tokens_seen": 1789132800 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"learning_rate": 0.00025453943635331944, |
|
"loss": 2.5892, |
|
"theoretical_loss": 3.4622231186239296, |
|
"tokens_seen": 1792409600 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"learning_rate": 0.0002537119349347456, |
|
"loss": 2.5919, |
|
"theoretical_loss": 3.461684569590848, |
|
"tokens_seen": 1795686400 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"learning_rate": 0.00025288443351617173, |
|
"loss": 2.5741, |
|
"theoretical_loss": 3.4611472770156917, |
|
"tokens_seen": 1798963200 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"objective/train/avg_token_score": 0.013695425353944302, |
|
"objective/train/avg_weight": 0.9890386462211609, |
|
"objective/train/docs_used": 1018281, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.694667339324951, |
|
"objective/train/std_weight": 0.05179031938314438, |
|
"objective/train/theoretical_loss": 3.460611235686456, |
|
"objective/train/tokens_used": 1822700000, |
|
"theoretical_loss": 3.460611235686456, |
|
"tokens_seen": 1802240000 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"learning_rate": 0.0002520569320975979, |
|
"loss": 2.5751, |
|
"theoretical_loss": 3.460611235686456, |
|
"tokens_seen": 1802240000 |
|
}, |
|
{ |
|
"epoch": 0.64, |
|
"learning_rate": 0.000251229430679024, |
|
"loss": 2.5591, |
|
"theoretical_loss": 3.4600764404221835, |
|
"tokens_seen": 1805516800 |
|
}, |
|
{ |
|
"epoch": 0.65, |
|
"learning_rate": 0.00025040192926045014, |
|
"loss": 2.5516, |
|
"theoretical_loss": 3.4595428860727218, |
|
"tokens_seen": 1808793600 |
|
}, |
|
{ |
|
"epoch": 0.65, |
|
"learning_rate": 0.0002495744278418763, |
|
"loss": 2.5381, |
|
"theoretical_loss": 3.4590105675184866, |
|
"tokens_seen": 1812070400 |
|
}, |
|
{ |
|
"epoch": 0.65, |
|
"learning_rate": 0.00024874692642330243, |
|
"loss": 2.5791, |
|
"theoretical_loss": 3.4584794796702223, |
|
"tokens_seen": 1815347200 |
|
}, |
|
{ |
|
"epoch": 0.65, |
|
"learning_rate": 0.00024791942500472855, |
|
"loss": 2.5841, |
|
"theoretical_loss": 3.457949617468771, |
|
"tokens_seen": 1818624000 |
|
}, |
|
{ |
|
"epoch": 0.65, |
|
"learning_rate": 0.0002470919235861547, |
|
"loss": 2.6289, |
|
"theoretical_loss": 3.4574209758848395, |
|
"tokens_seen": 1821900800 |
|
}, |
|
{ |
|
"epoch": 0.65, |
|
"learning_rate": 0.00024626442216758084, |
|
"loss": 2.5694, |
|
"theoretical_loss": 3.4568935499187696, |
|
"tokens_seen": 1825177600 |
|
}, |
|
{ |
|
"epoch": 0.65, |
|
"learning_rate": 0.00024543692074900696, |
|
"loss": 2.5867, |
|
"theoretical_loss": 3.456367334600311, |
|
"tokens_seen": 1828454400 |
|
}, |
|
{ |
|
"epoch": 0.65, |
|
"learning_rate": 0.00024460941933043313, |
|
"loss": 2.5827, |
|
"theoretical_loss": 3.455842324988396, |
|
"tokens_seen": 1831731200 |
|
}, |
|
{ |
|
"epoch": 0.66, |
|
"learning_rate": 0.00024379846794023077, |
|
"loss": 2.581, |
|
"theoretical_loss": 3.455318516170914, |
|
"tokens_seen": 1835008000 |
|
}, |
|
{ |
|
"epoch": 0.66, |
|
"learning_rate": 0.00024297096652165688, |
|
"loss": 2.5736, |
|
"theoretical_loss": 3.454795903264496, |
|
"tokens_seen": 1838284800 |
|
}, |
|
{ |
|
"epoch": 0.66, |
|
"learning_rate": 0.00024214346510308306, |
|
"loss": 2.5878, |
|
"theoretical_loss": 3.4542744814142896, |
|
"tokens_seen": 1841561600 |
|
}, |
|
{ |
|
"epoch": 0.66, |
|
"learning_rate": 0.00024131596368450918, |
|
"loss": 2.6106, |
|
"theoretical_loss": 3.4537542457937453, |
|
"tokens_seen": 1844838400 |
|
}, |
|
{ |
|
"epoch": 0.66, |
|
"learning_rate": 0.0002404884622659353, |
|
"loss": 2.6384, |
|
"theoretical_loss": 3.4532351916044, |
|
"tokens_seen": 1848115200 |
|
}, |
|
{ |
|
"epoch": 0.66, |
|
"learning_rate": 0.00023966096084736147, |
|
"loss": 2.6424, |
|
"theoretical_loss": 3.4527173140756666, |
|
"tokens_seen": 1851392000 |
|
}, |
|
{ |
|
"epoch": 0.66, |
|
"learning_rate": 0.00023883345942878758, |
|
"loss": 2.653, |
|
"theoretical_loss": 3.4522006084646204, |
|
"tokens_seen": 1854668800 |
|
}, |
|
{ |
|
"epoch": 0.66, |
|
"learning_rate": 0.00023800595801021373, |
|
"loss": 2.6569, |
|
"theoretical_loss": 3.4516850700557917, |
|
"tokens_seen": 1857945600 |
|
}, |
|
{ |
|
"epoch": 0.66, |
|
"learning_rate": 0.00023717845659163985, |
|
"loss": 2.6964, |
|
"theoretical_loss": 3.4511706941609566, |
|
"tokens_seen": 1861222400 |
|
}, |
|
{ |
|
"epoch": 0.67, |
|
"learning_rate": 0.00023635095517306602, |
|
"loss": 2.6305, |
|
"theoretical_loss": 3.450657476118936, |
|
"tokens_seen": 1864499200 |
|
}, |
|
{ |
|
"epoch": 0.67, |
|
"learning_rate": 0.00023552345375449214, |
|
"loss": 2.6041, |
|
"theoretical_loss": 3.450145411295389, |
|
"tokens_seen": 1867776000 |
|
}, |
|
{ |
|
"epoch": 0.67, |
|
"learning_rate": 0.00023469595233591828, |
|
"loss": 2.594, |
|
"theoretical_loss": 3.4496344950826114, |
|
"tokens_seen": 1871052800 |
|
}, |
|
{ |
|
"epoch": 0.67, |
|
"learning_rate": 0.00023386845091734443, |
|
"loss": 2.5873, |
|
"theoretical_loss": 3.449124722899339, |
|
"tokens_seen": 1874329600 |
|
}, |
|
{ |
|
"epoch": 0.67, |
|
"learning_rate": 0.00023304094949877057, |
|
"loss": 2.5695, |
|
"theoretical_loss": 3.4486160901905487, |
|
"tokens_seen": 1877606400 |
|
}, |
|
{ |
|
"epoch": 0.67, |
|
"learning_rate": 0.0002322134480801967, |
|
"loss": 2.5621, |
|
"theoretical_loss": 3.448108592427261, |
|
"tokens_seen": 1880883200 |
|
}, |
|
{ |
|
"epoch": 0.67, |
|
"learning_rate": 0.00023138594666162287, |
|
"loss": 2.5457, |
|
"theoretical_loss": 3.4476022251063485, |
|
"tokens_seen": 1884160000 |
|
}, |
|
{ |
|
"epoch": 0.67, |
|
"learning_rate": 0.00023055844524304898, |
|
"loss": 2.5388, |
|
"theoretical_loss": 3.447096983750342, |
|
"tokens_seen": 1887436800 |
|
}, |
|
{ |
|
"epoch": 0.68, |
|
"learning_rate": 0.0002297309438244751, |
|
"loss": 2.5436, |
|
"theoretical_loss": 3.4465928639072416, |
|
"tokens_seen": 1890713600 |
|
}, |
|
{ |
|
"epoch": 0.68, |
|
"learning_rate": 0.00022890344240590127, |
|
"loss": 2.5626, |
|
"theoretical_loss": 3.446089861150326, |
|
"tokens_seen": 1893990400 |
|
}, |
|
{ |
|
"epoch": 0.68, |
|
"learning_rate": 0.0002280759409873274, |
|
"loss": 2.5964, |
|
"theoretical_loss": 3.445587971077968, |
|
"tokens_seen": 1897267200 |
|
}, |
|
{ |
|
"epoch": 0.68, |
|
"learning_rate": 0.00022724843956875354, |
|
"loss": 2.5563, |
|
"theoretical_loss": 3.4450871893134467, |
|
"tokens_seen": 1900544000 |
|
}, |
|
{ |
|
"epoch": 0.68, |
|
"learning_rate": 0.0002264209381501797, |
|
"loss": 2.5558, |
|
"theoretical_loss": 3.4445875115047655, |
|
"tokens_seen": 1903820800 |
|
}, |
|
{ |
|
"epoch": 0.68, |
|
"learning_rate": 0.00022559343673160583, |
|
"loss": 2.5288, |
|
"theoretical_loss": 3.4440889333244695, |
|
"tokens_seen": 1907097600 |
|
}, |
|
{ |
|
"epoch": 0.68, |
|
"learning_rate": 0.00022478248534140344, |
|
"loss": 2.5053, |
|
"theoretical_loss": 3.443591450469466, |
|
"tokens_seen": 1910374400 |
|
}, |
|
{ |
|
"epoch": 0.68, |
|
"learning_rate": 0.0002239549839228296, |
|
"loss": 2.5519, |
|
"theoretical_loss": 3.4430950586608438, |
|
"tokens_seen": 1913651200 |
|
}, |
|
{ |
|
"epoch": 0.68, |
|
"learning_rate": 0.00022312748250425573, |
|
"loss": 2.553, |
|
"theoretical_loss": 3.4425997536437, |
|
"tokens_seen": 1916928000 |
|
}, |
|
{ |
|
"epoch": 0.69, |
|
"learning_rate": 0.00022229998108568185, |
|
"loss": 2.5718, |
|
"theoretical_loss": 3.4421055311869604, |
|
"tokens_seen": 1920204800 |
|
}, |
|
{ |
|
"epoch": 0.69, |
|
"learning_rate": 0.00022147247966710802, |
|
"loss": 2.6013, |
|
"theoretical_loss": 3.441612387083208, |
|
"tokens_seen": 1923481600 |
|
}, |
|
{ |
|
"epoch": 0.69, |
|
"learning_rate": 0.00022064497824853414, |
|
"loss": 2.628, |
|
"theoretical_loss": 3.44112031714851, |
|
"tokens_seen": 1926758400 |
|
}, |
|
{ |
|
"epoch": 0.69, |
|
"learning_rate": 0.00021981747682996028, |
|
"loss": 2.6616, |
|
"theoretical_loss": 3.4406293172222466, |
|
"tokens_seen": 1930035200 |
|
}, |
|
{ |
|
"epoch": 0.69, |
|
"learning_rate": 0.0002189899754113864, |
|
"loss": 2.6224, |
|
"theoretical_loss": 3.4401393831669447, |
|
"tokens_seen": 1933312000 |
|
}, |
|
{ |
|
"epoch": 0.69, |
|
"learning_rate": 0.00021816247399281257, |
|
"loss": 2.5962, |
|
"theoretical_loss": 3.4396505108681046, |
|
"tokens_seen": 1936588800 |
|
}, |
|
{ |
|
"epoch": 0.69, |
|
"learning_rate": 0.00021738462265935313, |
|
"loss": 2.6001, |
|
"theoretical_loss": 3.4391626962340394, |
|
"tokens_seen": 1939865600 |
|
}, |
|
{ |
|
"epoch": 0.69, |
|
"learning_rate": 0.00021655712124077928, |
|
"loss": 2.6397, |
|
"theoretical_loss": 3.4386759351957066, |
|
"tokens_seen": 1943142400 |
|
}, |
|
{ |
|
"epoch": 0.7, |
|
"learning_rate": 0.00021572961982220542, |
|
"loss": 2.6024, |
|
"theoretical_loss": 3.438190223706548, |
|
"tokens_seen": 1946419200 |
|
}, |
|
{ |
|
"epoch": 0.7, |
|
"learning_rate": 0.00021490211840363154, |
|
"loss": 2.6123, |
|
"theoretical_loss": 3.437705557742323, |
|
"tokens_seen": 1949696000 |
|
}, |
|
{ |
|
"epoch": 0.7, |
|
"learning_rate": 0.00021407461698505766, |
|
"loss": 2.5588, |
|
"theoretical_loss": 3.4372219333009535, |
|
"tokens_seen": 1952972800 |
|
}, |
|
{ |
|
"epoch": 0.7, |
|
"learning_rate": 0.00021324711556648383, |
|
"loss": 2.6149, |
|
"theoretical_loss": 3.436739346402362, |
|
"tokens_seen": 1956249600 |
|
}, |
|
{ |
|
"epoch": 0.7, |
|
"learning_rate": 0.00021241961414790995, |
|
"loss": 2.6347, |
|
"theoretical_loss": 3.4362577930883154, |
|
"tokens_seen": 1959526400 |
|
}, |
|
{ |
|
"epoch": 0.7, |
|
"learning_rate": 0.0002115921127293361, |
|
"loss": 2.6124, |
|
"theoretical_loss": 3.435777269422266, |
|
"tokens_seen": 1962803200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.4796244782337672, |
|
"debugging/distinct-1-grams": 0.7874466763773558, |
|
"debugging/distinct-2-grams": 0.9537592626603371, |
|
"debugging/entropy-1-grams": 6.225892551400493, |
|
"debugging/entropy-2-grams": 7.143796200176747, |
|
"debugging/length": 450.05, |
|
"debugging/num_segments": 20, |
|
"epoch": 0.7, |
|
"objective/train/avg_token_score": 0.028914527967572212, |
|
"objective/train/avg_weight": 0.9768664240837097, |
|
"objective/train/docs_used": 1112341, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.6348719596862793, |
|
"objective/train/std_weight": 0.07542026787996292, |
|
"objective/train/theoretical_loss": 3.4352977714892017, |
|
"objective/train/tokens_used": 1986540000, |
|
"theoretical_loss": 3.4352977714892017, |
|
"tokens_seen": 1966080000 |
|
}, |
|
{ |
|
"epoch": 0.7, |
|
"learning_rate": 0.00021076461131076224, |
|
"loss": 2.6115, |
|
"theoretical_loss": 3.4352977714892017, |
|
"tokens_seen": 1966080000 |
|
}, |
|
{ |
|
"epoch": 0.7, |
|
"learning_rate": 0.00020993710989218839, |
|
"loss": 2.5605, |
|
"theoretical_loss": 3.434819295395487, |
|
"tokens_seen": 1969356800 |
|
}, |
|
{ |
|
"epoch": 0.7, |
|
"learning_rate": 0.0002091096084736145, |
|
"loss": 2.5862, |
|
"theoretical_loss": 3.4343418372687156, |
|
"tokens_seen": 1972633600 |
|
}, |
|
{ |
|
"epoch": 0.71, |
|
"learning_rate": 0.00020828210705504068, |
|
"loss": 2.5877, |
|
"theoretical_loss": 3.433865393257557, |
|
"tokens_seen": 1975910400 |
|
}, |
|
{ |
|
"epoch": 0.71, |
|
"learning_rate": 0.0002074546056364668, |
|
"loss": 2.5431, |
|
"theoretical_loss": 3.433389959531607, |
|
"tokens_seen": 1979187200 |
|
}, |
|
{ |
|
"epoch": 0.71, |
|
"learning_rate": 0.00020662710421789294, |
|
"loss": 2.5604, |
|
"theoretical_loss": 3.4329155322812417, |
|
"tokens_seen": 1982464000 |
|
}, |
|
{ |
|
"epoch": 0.71, |
|
"learning_rate": 0.00020579960279931909, |
|
"loss": 2.5535, |
|
"theoretical_loss": 3.432442107717467, |
|
"tokens_seen": 1985740800 |
|
}, |
|
{ |
|
"epoch": 0.71, |
|
"learning_rate": 0.00020497210138074523, |
|
"loss": 2.5895, |
|
"theoretical_loss": 3.4319696820717778, |
|
"tokens_seen": 1989017600 |
|
}, |
|
{ |
|
"epoch": 0.71, |
|
"learning_rate": 0.00020414459996217135, |
|
"loss": 2.6229, |
|
"theoretical_loss": 3.4314982515960093, |
|
"tokens_seen": 1992294400 |
|
}, |
|
{ |
|
"epoch": 0.71, |
|
"learning_rate": 0.00020331709854359752, |
|
"loss": 2.6362, |
|
"theoretical_loss": 3.4310278125621965, |
|
"tokens_seen": 1995571200 |
|
}, |
|
{ |
|
"epoch": 0.71, |
|
"learning_rate": 0.00020248959712502364, |
|
"loss": 2.6353, |
|
"theoretical_loss": 3.430558361262433, |
|
"tokens_seen": 1998848000 |
|
}, |
|
{ |
|
"epoch": 0.72, |
|
"learning_rate": 0.00020166209570644976, |
|
"loss": 2.606, |
|
"theoretical_loss": 3.4300898940087254, |
|
"tokens_seen": 2002124800 |
|
}, |
|
{ |
|
"epoch": 0.72, |
|
"learning_rate": 0.00020083459428787593, |
|
"loss": 2.626, |
|
"theoretical_loss": 3.4296224071328623, |
|
"tokens_seen": 2005401600 |
|
}, |
|
{ |
|
"epoch": 0.72, |
|
"learning_rate": 0.00020000709286930208, |
|
"loss": 2.6057, |
|
"theoretical_loss": 3.429155896986268, |
|
"tokens_seen": 2008678400 |
|
}, |
|
{ |
|
"epoch": 0.72, |
|
"learning_rate": 0.0001991795914507282, |
|
"loss": 2.5761, |
|
"theoretical_loss": 3.4286903599398704, |
|
"tokens_seen": 2011955200 |
|
}, |
|
{ |
|
"epoch": 0.72, |
|
"learning_rate": 0.00019835209003215437, |
|
"loss": 2.6151, |
|
"theoretical_loss": 3.428225792383964, |
|
"tokens_seen": 2015232000 |
|
}, |
|
{ |
|
"epoch": 0.72, |
|
"learning_rate": 0.00019752458861358049, |
|
"loss": 2.607, |
|
"theoretical_loss": 3.4277621907280746, |
|
"tokens_seen": 2018508800 |
|
}, |
|
{ |
|
"epoch": 0.72, |
|
"learning_rate": 0.0001966970871950066, |
|
"loss": 2.5647, |
|
"theoretical_loss": 3.427299551400826, |
|
"tokens_seen": 2021785600 |
|
}, |
|
{ |
|
"epoch": 0.72, |
|
"learning_rate": 0.00019586958577643275, |
|
"loss": 2.5789, |
|
"theoretical_loss": 3.4268378708498086, |
|
"tokens_seen": 2025062400 |
|
}, |
|
{ |
|
"epoch": 0.72, |
|
"learning_rate": 0.0001950420843578589, |
|
"loss": 2.5631, |
|
"theoretical_loss": 3.426377145541448, |
|
"tokens_seen": 2028339200 |
|
}, |
|
{ |
|
"epoch": 0.73, |
|
"learning_rate": 0.00019421458293928504, |
|
"loss": 2.5439, |
|
"theoretical_loss": 3.425917371960872, |
|
"tokens_seen": 2031616000 |
|
}, |
|
{ |
|
"epoch": 0.73, |
|
"learning_rate": 0.00019338708152071116, |
|
"loss": 2.5114, |
|
"theoretical_loss": 3.425458546611787, |
|
"tokens_seen": 2034892800 |
|
}, |
|
{ |
|
"epoch": 0.73, |
|
"learning_rate": 0.00019255958010213733, |
|
"loss": 2.5113, |
|
"theoretical_loss": 3.425000666016346, |
|
"tokens_seen": 2038169600 |
|
}, |
|
{ |
|
"epoch": 0.73, |
|
"learning_rate": 0.00019173207868356345, |
|
"loss": 2.5199, |
|
"theoretical_loss": 3.4245437267150214, |
|
"tokens_seen": 2041446400 |
|
}, |
|
{ |
|
"epoch": 0.73, |
|
"learning_rate": 0.0001909045772649896, |
|
"loss": 2.5649, |
|
"theoretical_loss": 3.4240877252664834, |
|
"tokens_seen": 2044723200 |
|
}, |
|
{ |
|
"epoch": 0.73, |
|
"learning_rate": 0.00019007707584641574, |
|
"loss": 2.5396, |
|
"theoretical_loss": 3.4236326582474716, |
|
"tokens_seen": 2048000000 |
|
}, |
|
{ |
|
"epoch": 0.73, |
|
"learning_rate": 0.00018924957442784189, |
|
"loss": 2.5308, |
|
"theoretical_loss": 3.423178522252672, |
|
"tokens_seen": 2051276800 |
|
}, |
|
{ |
|
"epoch": 0.73, |
|
"learning_rate": 0.000188422073009268, |
|
"loss": 2.5336, |
|
"theoretical_loss": 3.4227253138945946, |
|
"tokens_seen": 2054553600 |
|
}, |
|
{ |
|
"epoch": 0.73, |
|
"learning_rate": 0.00018759457159069418, |
|
"loss": 2.5377, |
|
"theoretical_loss": 3.4222730298034545, |
|
"tokens_seen": 2057830400 |
|
}, |
|
{ |
|
"epoch": 0.74, |
|
"learning_rate": 0.0001867670701721203, |
|
"loss": 2.534, |
|
"theoretical_loss": 3.421821666627046, |
|
"tokens_seen": 2061107200 |
|
}, |
|
{ |
|
"epoch": 0.74, |
|
"learning_rate": 0.0001859395687535464, |
|
"loss": 2.5507, |
|
"theoretical_loss": 3.4213712210306286, |
|
"tokens_seen": 2064384000 |
|
}, |
|
{ |
|
"epoch": 0.74, |
|
"learning_rate": 0.00018511206733497258, |
|
"loss": 2.5617, |
|
"theoretical_loss": 3.4209216896968044, |
|
"tokens_seen": 2067660800 |
|
}, |
|
{ |
|
"epoch": 0.74, |
|
"learning_rate": 0.0001842845659163987, |
|
"loss": 2.5099, |
|
"theoretical_loss": 3.4204730693254035, |
|
"tokens_seen": 2070937600 |
|
}, |
|
{ |
|
"epoch": 0.74, |
|
"learning_rate": 0.00018345706449782485, |
|
"loss": 2.5817, |
|
"theoretical_loss": 3.4200253566333645, |
|
"tokens_seen": 2074214400 |
|
}, |
|
{ |
|
"epoch": 0.74, |
|
"learning_rate": 0.00018262956307925102, |
|
"loss": 2.6119, |
|
"theoretical_loss": 3.4195785483546226, |
|
"tokens_seen": 2077491200 |
|
}, |
|
{ |
|
"epoch": 0.74, |
|
"learning_rate": 0.00018180206166067714, |
|
"loss": 2.6099, |
|
"theoretical_loss": 3.419132641239991, |
|
"tokens_seen": 2080768000 |
|
}, |
|
{ |
|
"epoch": 0.74, |
|
"learning_rate": 0.00018097456024210326, |
|
"loss": 2.5597, |
|
"theoretical_loss": 3.4186876320570505, |
|
"tokens_seen": 2084044800 |
|
}, |
|
{ |
|
"epoch": 0.75, |
|
"learning_rate": 0.0001801470588235294, |
|
"loss": 2.5872, |
|
"theoretical_loss": 3.418243517590036, |
|
"tokens_seen": 2087321600 |
|
}, |
|
{ |
|
"epoch": 0.75, |
|
"learning_rate": 0.00017931955740495555, |
|
"loss": 2.5631, |
|
"theoretical_loss": 3.4178002946397217, |
|
"tokens_seen": 2090598400 |
|
}, |
|
{ |
|
"epoch": 0.75, |
|
"learning_rate": 0.0001784920559863817, |
|
"loss": 2.5988, |
|
"theoretical_loss": 3.4173579600233155, |
|
"tokens_seen": 2093875200 |
|
}, |
|
{ |
|
"epoch": 0.75, |
|
"learning_rate": 0.0001776645545678078, |
|
"loss": 2.5737, |
|
"theoretical_loss": 3.416916510574345, |
|
"tokens_seen": 2097152000 |
|
}, |
|
{ |
|
"epoch": 0.75, |
|
"learning_rate": 0.00017683705314923398, |
|
"loss": 2.5573, |
|
"theoretical_loss": 3.4164759431425495, |
|
"tokens_seen": 2100428800 |
|
}, |
|
{ |
|
"epoch": 0.75, |
|
"learning_rate": 0.0001760095517306601, |
|
"loss": 2.5844, |
|
"theoretical_loss": 3.4160362545937732, |
|
"tokens_seen": 2103705600 |
|
}, |
|
{ |
|
"epoch": 0.75, |
|
"learning_rate": 0.00017518205031208622, |
|
"loss": 2.5791, |
|
"theoretical_loss": 3.4155974418098562, |
|
"tokens_seen": 2106982400 |
|
}, |
|
{ |
|
"epoch": 0.75, |
|
"learning_rate": 0.0001743545488935124, |
|
"loss": 2.5757, |
|
"theoretical_loss": 3.415159501688528, |
|
"tokens_seen": 2110259200 |
|
}, |
|
{ |
|
"epoch": 0.75, |
|
"learning_rate": 0.00017352704747493854, |
|
"loss": 2.6096, |
|
"theoretical_loss": 3.4147224311433053, |
|
"tokens_seen": 2113536000 |
|
}, |
|
{ |
|
"epoch": 0.76, |
|
"learning_rate": 0.00017269954605636468, |
|
"loss": 2.6193, |
|
"theoretical_loss": 3.414286227103381, |
|
"tokens_seen": 2116812800 |
|
}, |
|
{ |
|
"epoch": 0.76, |
|
"learning_rate": 0.0001718720446377908, |
|
"loss": 2.5949, |
|
"theoretical_loss": 3.413850886513529, |
|
"tokens_seen": 2120089600 |
|
}, |
|
{ |
|
"epoch": 0.76, |
|
"learning_rate": 0.00017104454321921695, |
|
"loss": 2.5676, |
|
"theoretical_loss": 3.413416406333992, |
|
"tokens_seen": 2123366400 |
|
}, |
|
{ |
|
"epoch": 0.76, |
|
"learning_rate": 0.0001702170418006431, |
|
"loss": 2.58, |
|
"theoretical_loss": 3.412982783540387, |
|
"tokens_seen": 2126643200 |
|
}, |
|
{ |
|
"epoch": 0.76, |
|
"objective/train/avg_token_score": 0.011589955538511276, |
|
"objective/train/avg_weight": 0.9907302856445312, |
|
"objective/train/docs_used": 1206282, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.4739208221435547, |
|
"objective/train/std_weight": 0.03995203971862793, |
|
"objective/train/theoretical_loss": 3.4125500151236, |
|
"objective/train/tokens_used": 2150380000, |
|
"theoretical_loss": 3.4125500151236, |
|
"tokens_seen": 2129920000 |
|
}, |
|
{ |
|
"epoch": 0.76, |
|
"learning_rate": 0.0001693895403820692, |
|
"loss": 2.529, |
|
"theoretical_loss": 3.4125500151236, |
|
"tokens_seen": 2129920000 |
|
}, |
|
{ |
|
"epoch": 0.76, |
|
"learning_rate": 0.00016856203896349536, |
|
"loss": 2.5521, |
|
"theoretical_loss": 3.412118098089686, |
|
"tokens_seen": 2133196800 |
|
}, |
|
{ |
|
"epoch": 0.76, |
|
"learning_rate": 0.0001677345375449215, |
|
"loss": 2.5706, |
|
"theoretical_loss": 3.411687029459771, |
|
"tokens_seen": 2136473600 |
|
}, |
|
{ |
|
"epoch": 0.76, |
|
"learning_rate": 0.00016690703612634765, |
|
"loss": 2.5397, |
|
"theoretical_loss": 3.411256806269951, |
|
"tokens_seen": 2139750400 |
|
}, |
|
{ |
|
"epoch": 0.77, |
|
"learning_rate": 0.0001660795347077738, |
|
"loss": 2.5421, |
|
"theoretical_loss": 3.4108274255711954, |
|
"tokens_seen": 2143027200 |
|
}, |
|
{ |
|
"epoch": 0.77, |
|
"learning_rate": 0.0001652520332891999, |
|
"loss": 2.5618, |
|
"theoretical_loss": 3.4103988844292497, |
|
"tokens_seen": 2146304000 |
|
}, |
|
{ |
|
"epoch": 0.77, |
|
"learning_rate": 0.00016442453187062606, |
|
"loss": 2.5877, |
|
"theoretical_loss": 3.4099711799245385, |
|
"tokens_seen": 2149580800 |
|
}, |
|
{ |
|
"epoch": 0.77, |
|
"learning_rate": 0.0001635970304520522, |
|
"loss": 2.5964, |
|
"theoretical_loss": 3.4095443091520705, |
|
"tokens_seen": 2152857600 |
|
}, |
|
{ |
|
"epoch": 0.77, |
|
"learning_rate": 0.00016276952903347835, |
|
"loss": 2.5631, |
|
"theoretical_loss": 3.4091182692213415, |
|
"tokens_seen": 2156134400 |
|
}, |
|
{ |
|
"epoch": 0.77, |
|
"learning_rate": 0.0001619420276149045, |
|
"loss": 2.5217, |
|
"theoretical_loss": 3.4086930572562446, |
|
"tokens_seen": 2159411200 |
|
}, |
|
{ |
|
"epoch": 0.77, |
|
"learning_rate": 0.00016111452619633064, |
|
"loss": 2.5495, |
|
"theoretical_loss": 3.4082686703949716, |
|
"tokens_seen": 2162688000 |
|
}, |
|
{ |
|
"epoch": 0.77, |
|
"learning_rate": 0.00016028702477775676, |
|
"loss": 2.5851, |
|
"theoretical_loss": 3.4078451057899244, |
|
"tokens_seen": 2165964800 |
|
}, |
|
{ |
|
"epoch": 0.77, |
|
"learning_rate": 0.0001594595233591829, |
|
"loss": 2.6082, |
|
"theoretical_loss": 3.4074223606076206, |
|
"tokens_seen": 2169241600 |
|
}, |
|
{ |
|
"epoch": 0.78, |
|
"learning_rate": 0.00015863202194060902, |
|
"loss": 2.6043, |
|
"theoretical_loss": 3.4070004320286036, |
|
"tokens_seen": 2172518400 |
|
}, |
|
{ |
|
"epoch": 0.78, |
|
"learning_rate": 0.00015780452052203517, |
|
"loss": 2.5623, |
|
"theoretical_loss": 3.4065793172473526, |
|
"tokens_seen": 2175795200 |
|
}, |
|
{ |
|
"epoch": 0.78, |
|
"learning_rate": 0.00015697701910346134, |
|
"loss": 2.5446, |
|
"theoretical_loss": 3.40615901347219, |
|
"tokens_seen": 2179072000 |
|
}, |
|
{ |
|
"epoch": 0.78, |
|
"learning_rate": 0.00015614951768488746, |
|
"loss": 2.5797, |
|
"theoretical_loss": 3.405739517925197, |
|
"tokens_seen": 2182348800 |
|
}, |
|
{ |
|
"epoch": 0.78, |
|
"learning_rate": 0.0001553220162663136, |
|
"loss": 2.5793, |
|
"theoretical_loss": 3.4053208278421208, |
|
"tokens_seen": 2185625600 |
|
}, |
|
{ |
|
"epoch": 0.78, |
|
"learning_rate": 0.00015449451484773975, |
|
"loss": 2.5536, |
|
"theoretical_loss": 3.4049029404722897, |
|
"tokens_seen": 2188902400 |
|
}, |
|
{ |
|
"epoch": 0.78, |
|
"learning_rate": 0.00015366701342916586, |
|
"loss": 2.5341, |
|
"theoretical_loss": 3.4044858530785254, |
|
"tokens_seen": 2192179200 |
|
}, |
|
{ |
|
"epoch": 0.78, |
|
"learning_rate": 0.000152839512010592, |
|
"loss": 2.5888, |
|
"theoretical_loss": 3.4040695629370563, |
|
"tokens_seen": 2195456000 |
|
}, |
|
{ |
|
"epoch": 0.79, |
|
"learning_rate": 0.00015201201059201816, |
|
"loss": 2.5843, |
|
"theoretical_loss": 3.403654067337433, |
|
"tokens_seen": 2198732800 |
|
}, |
|
{ |
|
"epoch": 0.79, |
|
"learning_rate": 0.0001511845091734443, |
|
"loss": 2.5466, |
|
"theoretical_loss": 3.4032393635824416, |
|
"tokens_seen": 2202009600 |
|
}, |
|
{ |
|
"epoch": 0.79, |
|
"learning_rate": 0.00015035700775487045, |
|
"loss": 2.5983, |
|
"theoretical_loss": 3.4028254489880227, |
|
"tokens_seen": 2205286400 |
|
}, |
|
{ |
|
"epoch": 0.79, |
|
"learning_rate": 0.00014952950633629656, |
|
"loss": 2.7334, |
|
"theoretical_loss": 3.4024123208831836, |
|
"tokens_seen": 2208563200 |
|
}, |
|
{ |
|
"epoch": 0.79, |
|
"learning_rate": 0.0001487020049177227, |
|
"loss": 2.7895, |
|
"theoretical_loss": 3.4019999766099187, |
|
"tokens_seen": 2211840000 |
|
}, |
|
{ |
|
"epoch": 0.79, |
|
"learning_rate": 0.00014787450349914886, |
|
"loss": 2.834, |
|
"theoretical_loss": 3.4015884135231262, |
|
"tokens_seen": 2215116800 |
|
}, |
|
{ |
|
"epoch": 0.79, |
|
"learning_rate": 0.000147047002080575, |
|
"loss": 2.7539, |
|
"theoretical_loss": 3.401177628990526, |
|
"tokens_seen": 2218393600 |
|
}, |
|
{ |
|
"epoch": 0.79, |
|
"learning_rate": 0.00014621950066200115, |
|
"loss": 2.7577, |
|
"theoretical_loss": 3.400767620392579, |
|
"tokens_seen": 2221670400 |
|
}, |
|
{ |
|
"epoch": 0.79, |
|
"learning_rate": 0.0001453919992434273, |
|
"loss": 2.7237, |
|
"theoretical_loss": 3.4003583851224075, |
|
"tokens_seen": 2224947200 |
|
}, |
|
{ |
|
"epoch": 0.8, |
|
"learning_rate": 0.0001445644978248534, |
|
"loss": 2.7093, |
|
"theoretical_loss": 3.3999499205857138, |
|
"tokens_seen": 2228224000 |
|
}, |
|
{ |
|
"epoch": 0.8, |
|
"learning_rate": 0.00014373699640627956, |
|
"loss": 2.6425, |
|
"theoretical_loss": 3.3995422242007036, |
|
"tokens_seen": 2231500800 |
|
}, |
|
{ |
|
"epoch": 0.8, |
|
"learning_rate": 0.00014290949498770567, |
|
"loss": 2.6649, |
|
"theoretical_loss": 3.399135293398004, |
|
"tokens_seen": 2234777600 |
|
}, |
|
{ |
|
"epoch": 0.8, |
|
"learning_rate": 0.00014208199356913182, |
|
"loss": 2.6231, |
|
"theoretical_loss": 3.39872912562059, |
|
"tokens_seen": 2238054400 |
|
}, |
|
{ |
|
"epoch": 0.8, |
|
"learning_rate": 0.00014125449215055796, |
|
"loss": 2.613, |
|
"theoretical_loss": 3.3983237183237023, |
|
"tokens_seen": 2241331200 |
|
}, |
|
{ |
|
"epoch": 0.8, |
|
"learning_rate": 0.0001404269907319841, |
|
"loss": 2.5821, |
|
"theoretical_loss": 3.397919068974775, |
|
"tokens_seen": 2244608000 |
|
}, |
|
{ |
|
"epoch": 0.8, |
|
"learning_rate": 0.00013959948931341025, |
|
"loss": 2.5606, |
|
"theoretical_loss": 3.3975151750533574, |
|
"tokens_seen": 2247884800 |
|
}, |
|
{ |
|
"epoch": 0.8, |
|
"learning_rate": 0.0001387719878948364, |
|
"loss": 2.5323, |
|
"theoretical_loss": 3.3971120340510375, |
|
"tokens_seen": 2251161600 |
|
}, |
|
{ |
|
"epoch": 0.81, |
|
"learning_rate": 0.00013794448647626252, |
|
"loss": 2.5517, |
|
"theoretical_loss": 3.39670964347137, |
|
"tokens_seen": 2254438400 |
|
}, |
|
{ |
|
"epoch": 0.81, |
|
"learning_rate": 0.00013711698505768866, |
|
"loss": 2.5579, |
|
"theoretical_loss": 3.396308000829799, |
|
"tokens_seen": 2257715200 |
|
}, |
|
{ |
|
"epoch": 0.81, |
|
"learning_rate": 0.0001362894836391148, |
|
"loss": 2.5327, |
|
"theoretical_loss": 3.395907103653585, |
|
"tokens_seen": 2260992000 |
|
}, |
|
{ |
|
"epoch": 0.81, |
|
"learning_rate": 0.00013546198222054095, |
|
"loss": 2.5313, |
|
"theoretical_loss": 3.395506949481734, |
|
"tokens_seen": 2264268800 |
|
}, |
|
{ |
|
"epoch": 0.81, |
|
"learning_rate": 0.0001346344808019671, |
|
"loss": 2.5387, |
|
"theoretical_loss": 3.3951075358649208, |
|
"tokens_seen": 2267545600 |
|
}, |
|
{ |
|
"epoch": 0.81, |
|
"learning_rate": 0.00013380697938339322, |
|
"loss": 2.5161, |
|
"theoretical_loss": 3.3947088603654203, |
|
"tokens_seen": 2270822400 |
|
}, |
|
{ |
|
"epoch": 0.81, |
|
"learning_rate": 0.00013297947796481936, |
|
"loss": 2.5693, |
|
"theoretical_loss": 3.394310920557034, |
|
"tokens_seen": 2274099200 |
|
}, |
|
{ |
|
"epoch": 0.81, |
|
"learning_rate": 0.0001321519765462455, |
|
"loss": 2.5449, |
|
"theoretical_loss": 3.393913714025021, |
|
"tokens_seen": 2277376000 |
|
}, |
|
{ |
|
"epoch": 0.81, |
|
"learning_rate": 0.00013132447512767163, |
|
"loss": 2.5575, |
|
"theoretical_loss": 3.393517238366023, |
|
"tokens_seen": 2280652800 |
|
}, |
|
{ |
|
"epoch": 0.82, |
|
"learning_rate": 0.0001304969737090978, |
|
"loss": 2.5835, |
|
"theoretical_loss": 3.3931214911880008, |
|
"tokens_seen": 2283929600 |
|
}, |
|
{ |
|
"epoch": 0.82, |
|
"learning_rate": 0.00012966947229052394, |
|
"loss": 2.5613, |
|
"theoretical_loss": 3.3927264701101594, |
|
"tokens_seen": 2287206400 |
|
}, |
|
{ |
|
"epoch": 0.82, |
|
"learning_rate": 0.00012884197087195006, |
|
"loss": 2.5945, |
|
"theoretical_loss": 3.392332172762883, |
|
"tokens_seen": 2290483200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.515096219338814, |
|
"debugging/distinct-1-grams": 0.7923118197529954, |
|
"debugging/distinct-2-grams": 0.9542743672238616, |
|
"debugging/entropy-1-grams": 6.147062465574793, |
|
"debugging/entropy-2-grams": 7.107820582543805, |
|
"debugging/length": 504.1666666666667, |
|
"debugging/num_segments": 18, |
|
"epoch": 0.82, |
|
"objective/train/avg_token_score": 0.012544393539428711, |
|
"objective/train/avg_weight": 0.989965558052063, |
|
"objective/train/docs_used": 1292636, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.5035319328308105, |
|
"objective/train/std_weight": 0.0364028625190258, |
|
"objective/train/theoretical_loss": 3.3919385967876625, |
|
"objective/train/tokens_used": 2314220000, |
|
"theoretical_loss": 3.3919385967876625, |
|
"tokens_seen": 2293760000 |
|
}, |
|
{ |
|
"epoch": 0.82, |
|
"learning_rate": 0.0001280144694533762, |
|
"loss": 2.57, |
|
"theoretical_loss": 3.3919385967876625, |
|
"tokens_seen": 2293760000 |
|
}, |
|
{ |
|
"epoch": 0.82, |
|
"learning_rate": 0.00012718696803480233, |
|
"loss": 2.5982, |
|
"theoretical_loss": 3.3915457398370323, |
|
"tokens_seen": 2297036800 |
|
}, |
|
{ |
|
"epoch": 0.82, |
|
"learning_rate": 0.00012635946661622847, |
|
"loss": 2.6667, |
|
"theoretical_loss": 3.3911535995744995, |
|
"tokens_seen": 2300313600 |
|
}, |
|
{ |
|
"epoch": 0.82, |
|
"learning_rate": 0.00012553196519765462, |
|
"loss": 2.7193, |
|
"theoretical_loss": 3.390762173674479, |
|
"tokens_seen": 2303590400 |
|
}, |
|
{ |
|
"epoch": 0.82, |
|
"learning_rate": 0.00012470446377908076, |
|
"loss": 2.7049, |
|
"theoretical_loss": 3.3903714598222243, |
|
"tokens_seen": 2306867200 |
|
}, |
|
{ |
|
"epoch": 0.83, |
|
"learning_rate": 0.0001238769623605069, |
|
"loss": 2.6534, |
|
"theoretical_loss": 3.389981455713767, |
|
"tokens_seen": 2310144000 |
|
}, |
|
{ |
|
"epoch": 0.83, |
|
"learning_rate": 0.00012304946094193305, |
|
"loss": 2.6746, |
|
"theoretical_loss": 3.389592159055845, |
|
"tokens_seen": 2313420800 |
|
}, |
|
{ |
|
"epoch": 0.83, |
|
"learning_rate": 0.00012222195952335917, |
|
"loss": 2.6331, |
|
"theoretical_loss": 3.389203567565843, |
|
"tokens_seen": 2316697600 |
|
}, |
|
{ |
|
"epoch": 0.83, |
|
"learning_rate": 0.00012139445810478532, |
|
"loss": 2.5633, |
|
"theoretical_loss": 3.3888156789717243, |
|
"tokens_seen": 2319974400 |
|
}, |
|
{ |
|
"epoch": 0.83, |
|
"learning_rate": 0.00012056695668621148, |
|
"loss": 2.625, |
|
"theoretical_loss": 3.3884284910119704, |
|
"tokens_seen": 2323251200 |
|
}, |
|
{ |
|
"epoch": 0.83, |
|
"learning_rate": 0.0001197394552676376, |
|
"loss": 2.5941, |
|
"theoretical_loss": 3.3880420014355135, |
|
"tokens_seen": 2326528000 |
|
}, |
|
{ |
|
"epoch": 0.83, |
|
"learning_rate": 0.00011891195384906374, |
|
"loss": 2.572, |
|
"theoretical_loss": 3.387656208001678, |
|
"tokens_seen": 2329804800 |
|
}, |
|
{ |
|
"epoch": 0.83, |
|
"learning_rate": 0.00011808445243048987, |
|
"loss": 2.5808, |
|
"theoretical_loss": 3.387271108480113, |
|
"tokens_seen": 2333081600 |
|
}, |
|
{ |
|
"epoch": 0.83, |
|
"learning_rate": 0.00011725695101191602, |
|
"loss": 2.5701, |
|
"theoretical_loss": 3.386886700650736, |
|
"tokens_seen": 2336358400 |
|
}, |
|
{ |
|
"epoch": 0.84, |
|
"learning_rate": 0.00011642944959334216, |
|
"loss": 2.5821, |
|
"theoretical_loss": 3.386502982303667, |
|
"tokens_seen": 2339635200 |
|
}, |
|
{ |
|
"epoch": 0.84, |
|
"learning_rate": 0.0001156019481747683, |
|
"loss": 2.5398, |
|
"theoretical_loss": 3.3861199512391673, |
|
"tokens_seen": 2342912000 |
|
}, |
|
{ |
|
"epoch": 0.84, |
|
"learning_rate": 0.00011477444675619444, |
|
"loss": 2.6034, |
|
"theoretical_loss": 3.385737605267583, |
|
"tokens_seen": 2346188800 |
|
}, |
|
{ |
|
"epoch": 0.84, |
|
"learning_rate": 0.00011394694533762059, |
|
"loss": 2.5721, |
|
"theoretical_loss": 3.38535594220928, |
|
"tokens_seen": 2349465600 |
|
}, |
|
{ |
|
"epoch": 0.84, |
|
"learning_rate": 0.00011311944391904672, |
|
"loss": 2.5623, |
|
"theoretical_loss": 3.3849749598945875, |
|
"tokens_seen": 2352742400 |
|
}, |
|
{ |
|
"epoch": 0.84, |
|
"learning_rate": 0.00011229194250047286, |
|
"loss": 2.5541, |
|
"theoretical_loss": 3.3845946561637366, |
|
"tokens_seen": 2356019200 |
|
}, |
|
{ |
|
"epoch": 0.84, |
|
"learning_rate": 0.00011146444108189901, |
|
"loss": 2.6458, |
|
"theoretical_loss": 3.384215028866803, |
|
"tokens_seen": 2359296000 |
|
}, |
|
{ |
|
"epoch": 0.84, |
|
"learning_rate": 0.00011063693966332514, |
|
"loss": 2.6661, |
|
"theoretical_loss": 3.3838360758636474, |
|
"tokens_seen": 2362572800 |
|
}, |
|
{ |
|
"epoch": 0.84, |
|
"learning_rate": 0.00010980943824475128, |
|
"loss": 2.602, |
|
"theoretical_loss": 3.3834577950238582, |
|
"tokens_seen": 2365849600 |
|
}, |
|
{ |
|
"epoch": 0.85, |
|
"learning_rate": 0.0001089819368261774, |
|
"loss": 2.615, |
|
"theoretical_loss": 3.383080184226693, |
|
"tokens_seen": 2369126400 |
|
}, |
|
{ |
|
"epoch": 0.85, |
|
"learning_rate": 0.00010815443540760355, |
|
"loss": 2.6303, |
|
"theoretical_loss": 3.3827032413610225, |
|
"tokens_seen": 2372403200 |
|
}, |
|
{ |
|
"epoch": 0.85, |
|
"learning_rate": 0.00010732693398902971, |
|
"loss": 2.6031, |
|
"theoretical_loss": 3.382326964325273, |
|
"tokens_seen": 2375680000 |
|
}, |
|
{ |
|
"epoch": 0.85, |
|
"learning_rate": 0.00010649943257045583, |
|
"loss": 2.5776, |
|
"theoretical_loss": 3.3819513510273698, |
|
"tokens_seen": 2378956800 |
|
}, |
|
{ |
|
"epoch": 0.85, |
|
"learning_rate": 0.00010567193115188197, |
|
"loss": 2.5819, |
|
"theoretical_loss": 3.381576399384682, |
|
"tokens_seen": 2382233600 |
|
}, |
|
{ |
|
"epoch": 0.85, |
|
"learning_rate": 0.00010484442973330812, |
|
"loss": 2.6222, |
|
"theoretical_loss": 3.3812021073239675, |
|
"tokens_seen": 2385510400 |
|
}, |
|
{ |
|
"epoch": 0.85, |
|
"learning_rate": 0.00010401692831473425, |
|
"loss": 2.6193, |
|
"theoretical_loss": 3.380828472781316, |
|
"tokens_seen": 2388787200 |
|
}, |
|
{ |
|
"epoch": 0.85, |
|
"learning_rate": 0.0001031894268961604, |
|
"loss": 2.6218, |
|
"theoretical_loss": 3.380455493702095, |
|
"tokens_seen": 2392064000 |
|
}, |
|
{ |
|
"epoch": 0.86, |
|
"learning_rate": 0.00010236192547758653, |
|
"loss": 2.6051, |
|
"theoretical_loss": 3.3800831680408963, |
|
"tokens_seen": 2395340800 |
|
}, |
|
{ |
|
"epoch": 0.86, |
|
"learning_rate": 0.00010153442405901267, |
|
"loss": 2.5811, |
|
"theoretical_loss": 3.3797114937614814, |
|
"tokens_seen": 2398617600 |
|
}, |
|
{ |
|
"epoch": 0.86, |
|
"learning_rate": 0.00010070692264043882, |
|
"loss": 2.5999, |
|
"theoretical_loss": 3.3793404688367277, |
|
"tokens_seen": 2401894400 |
|
}, |
|
{ |
|
"epoch": 0.86, |
|
"learning_rate": 9.987942122186495e-05, |
|
"loss": 2.5834, |
|
"theoretical_loss": 3.378970091248576, |
|
"tokens_seen": 2405171200 |
|
}, |
|
{ |
|
"epoch": 0.86, |
|
"learning_rate": 9.90519198032911e-05, |
|
"loss": 2.5701, |
|
"theoretical_loss": 3.3786003589879763, |
|
"tokens_seen": 2408448000 |
|
}, |
|
{ |
|
"epoch": 0.86, |
|
"learning_rate": 9.822441838471724e-05, |
|
"loss": 2.5315, |
|
"theoretical_loss": 3.3782312700548376, |
|
"tokens_seen": 2411724800 |
|
}, |
|
{ |
|
"epoch": 0.86, |
|
"learning_rate": 9.739691696614337e-05, |
|
"loss": 2.5271, |
|
"theoretical_loss": 3.377862822457974, |
|
"tokens_seen": 2415001600 |
|
}, |
|
{ |
|
"epoch": 0.86, |
|
"learning_rate": 9.656941554756952e-05, |
|
"loss": 2.5361, |
|
"theoretical_loss": 3.3774950142150537, |
|
"tokens_seen": 2418278400 |
|
}, |
|
{ |
|
"epoch": 0.86, |
|
"learning_rate": 9.574191412899566e-05, |
|
"loss": 2.5455, |
|
"theoretical_loss": 3.3771278433525476, |
|
"tokens_seen": 2421555200 |
|
}, |
|
{ |
|
"epoch": 0.87, |
|
"learning_rate": 9.491441271042178e-05, |
|
"loss": 2.5594, |
|
"theoretical_loss": 3.376761307905678, |
|
"tokens_seen": 2424832000 |
|
}, |
|
{ |
|
"epoch": 0.87, |
|
"learning_rate": 9.408691129184794e-05, |
|
"loss": 2.5663, |
|
"theoretical_loss": 3.376395405918369, |
|
"tokens_seen": 2428108800 |
|
}, |
|
{ |
|
"epoch": 0.87, |
|
"learning_rate": 9.325940987327406e-05, |
|
"loss": 2.5476, |
|
"theoretical_loss": 3.3760301354431963, |
|
"tokens_seen": 2431385600 |
|
}, |
|
{ |
|
"epoch": 0.87, |
|
"learning_rate": 9.24319084547002e-05, |
|
"loss": 2.5528, |
|
"theoretical_loss": 3.375665494541335, |
|
"tokens_seen": 2434662400 |
|
}, |
|
{ |
|
"epoch": 0.87, |
|
"learning_rate": 9.160440703612635e-05, |
|
"loss": 2.5069, |
|
"theoretical_loss": 3.3753014812825133, |
|
"tokens_seen": 2437939200 |
|
}, |
|
{ |
|
"epoch": 0.87, |
|
"learning_rate": 9.077690561755248e-05, |
|
"loss": 2.4946, |
|
"theoretical_loss": 3.374938093744962, |
|
"tokens_seen": 2441216000 |
|
}, |
|
{ |
|
"epoch": 0.87, |
|
"learning_rate": 8.994940419897862e-05, |
|
"loss": 2.5248, |
|
"theoretical_loss": 3.3745753300153636, |
|
"tokens_seen": 2444492800 |
|
}, |
|
{ |
|
"epoch": 0.87, |
|
"learning_rate": 8.912190278040477e-05, |
|
"loss": 2.5206, |
|
"theoretical_loss": 3.374213188188808, |
|
"tokens_seen": 2447769600 |
|
}, |
|
{ |
|
"epoch": 0.88, |
|
"learning_rate": 8.82944013618309e-05, |
|
"loss": 2.5247, |
|
"theoretical_loss": 3.373851666368742, |
|
"tokens_seen": 2451046400 |
|
}, |
|
{ |
|
"epoch": 0.88, |
|
"learning_rate": 8.746689994325705e-05, |
|
"loss": 2.5211, |
|
"theoretical_loss": 3.373490762666921, |
|
"tokens_seen": 2454323200 |
|
}, |
|
{ |
|
"epoch": 0.88, |
|
"objective/train/avg_token_score": 0.009017497301101685, |
|
"objective/train/avg_weight": 0.9927802085876465, |
|
"objective/train/docs_used": 1380428, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.533566951751709, |
|
"objective/train/std_weight": 0.05582087114453316, |
|
"objective/train/theoretical_loss": 3.3731304752033617, |
|
"objective/train/tokens_used": 2478060000, |
|
"theoretical_loss": 3.3731304752033617, |
|
"tokens_seen": 2457600000 |
|
}, |
|
{ |
|
"epoch": 0.88, |
|
"learning_rate": 8.663939852468318e-05, |
|
"loss": 2.5607, |
|
"theoretical_loss": 3.3731304752033617, |
|
"tokens_seen": 2457600000 |
|
}, |
|
{ |
|
"epoch": 0.88, |
|
"learning_rate": 8.581189710610932e-05, |
|
"loss": 2.5554, |
|
"theoretical_loss": 3.372770802106297, |
|
"tokens_seen": 2460876800 |
|
}, |
|
{ |
|
"epoch": 0.88, |
|
"learning_rate": 8.498439568753547e-05, |
|
"loss": 2.5424, |
|
"theoretical_loss": 3.3724117415121273, |
|
"tokens_seen": 2464153600 |
|
}, |
|
{ |
|
"epoch": 0.88, |
|
"learning_rate": 8.41568942689616e-05, |
|
"loss": 2.5477, |
|
"theoretical_loss": 3.372053291565374, |
|
"tokens_seen": 2467430400 |
|
}, |
|
{ |
|
"epoch": 0.88, |
|
"learning_rate": 8.332939285038775e-05, |
|
"loss": 2.5676, |
|
"theoretical_loss": 3.3716954504186347, |
|
"tokens_seen": 2470707200 |
|
}, |
|
{ |
|
"epoch": 0.88, |
|
"learning_rate": 8.250189143181388e-05, |
|
"loss": 2.5319, |
|
"theoretical_loss": 3.371338216232536, |
|
"tokens_seen": 2473984000 |
|
}, |
|
{ |
|
"epoch": 0.88, |
|
"learning_rate": 8.167439001324002e-05, |
|
"loss": 2.5425, |
|
"theoretical_loss": 3.370981587175689, |
|
"tokens_seen": 2477260800 |
|
}, |
|
{ |
|
"epoch": 0.89, |
|
"learning_rate": 8.084688859466617e-05, |
|
"loss": 2.5249, |
|
"theoretical_loss": 3.370625561424644, |
|
"tokens_seen": 2480537600 |
|
}, |
|
{ |
|
"epoch": 0.89, |
|
"learning_rate": 8.00193871760923e-05, |
|
"loss": 2.4889, |
|
"theoretical_loss": 3.3702701371638453, |
|
"tokens_seen": 2483814400 |
|
}, |
|
{ |
|
"epoch": 0.89, |
|
"learning_rate": 7.919188575751843e-05, |
|
"loss": 2.4904, |
|
"theoretical_loss": 3.369915312585586, |
|
"tokens_seen": 2487091200 |
|
}, |
|
{ |
|
"epoch": 0.89, |
|
"learning_rate": 7.836438433894458e-05, |
|
"loss": 2.491, |
|
"theoretical_loss": 3.3695610858899663, |
|
"tokens_seen": 2490368000 |
|
}, |
|
{ |
|
"epoch": 0.89, |
|
"learning_rate": 7.753688292037072e-05, |
|
"loss": 2.4622, |
|
"theoretical_loss": 3.3692074552848457, |
|
"tokens_seen": 2493644800 |
|
}, |
|
{ |
|
"epoch": 0.89, |
|
"learning_rate": 7.670938150179686e-05, |
|
"loss": 2.5122, |
|
"theoretical_loss": 3.368854418985804, |
|
"tokens_seen": 2496921600 |
|
}, |
|
{ |
|
"epoch": 0.89, |
|
"learning_rate": 7.5881880083223e-05, |
|
"loss": 2.4713, |
|
"theoretical_loss": 3.368501975216093, |
|
"tokens_seen": 2500198400 |
|
}, |
|
{ |
|
"epoch": 0.89, |
|
"learning_rate": 7.505437866464915e-05, |
|
"loss": 2.4991, |
|
"theoretical_loss": 3.3681501222065977, |
|
"tokens_seen": 2503475200 |
|
}, |
|
{ |
|
"epoch": 0.9, |
|
"learning_rate": 7.422687724607528e-05, |
|
"loss": 2.4945, |
|
"theoretical_loss": 3.367798858195792, |
|
"tokens_seen": 2506752000 |
|
}, |
|
{ |
|
"epoch": 0.9, |
|
"learning_rate": 7.339937582750141e-05, |
|
"loss": 2.5301, |
|
"theoretical_loss": 3.367448181429695, |
|
"tokens_seen": 2510028800 |
|
}, |
|
{ |
|
"epoch": 0.9, |
|
"learning_rate": 7.257187440892756e-05, |
|
"loss": 2.5284, |
|
"theoretical_loss": 3.36709809016183, |
|
"tokens_seen": 2513305600 |
|
}, |
|
{ |
|
"epoch": 0.9, |
|
"learning_rate": 7.17443729903537e-05, |
|
"loss": 2.5485, |
|
"theoretical_loss": 3.3667485826531847, |
|
"tokens_seen": 2516582400 |
|
}, |
|
{ |
|
"epoch": 0.9, |
|
"learning_rate": 7.091687157177983e-05, |
|
"loss": 2.5142, |
|
"theoretical_loss": 3.366399657172166, |
|
"tokens_seen": 2519859200 |
|
}, |
|
{ |
|
"epoch": 0.9, |
|
"learning_rate": 7.010592018157745e-05, |
|
"loss": 2.4943, |
|
"theoretical_loss": 3.36605131199456, |
|
"tokens_seen": 2523136000 |
|
}, |
|
{ |
|
"epoch": 0.9, |
|
"learning_rate": 6.92784187630036e-05, |
|
"loss": 2.4464, |
|
"theoretical_loss": 3.365703545403493, |
|
"tokens_seen": 2526412800 |
|
}, |
|
{ |
|
"epoch": 0.9, |
|
"learning_rate": 6.845091734442973e-05, |
|
"loss": 2.4318, |
|
"theoretical_loss": 3.3653563556893875, |
|
"tokens_seen": 2529689600 |
|
}, |
|
{ |
|
"epoch": 0.9, |
|
"learning_rate": 6.762341592585588e-05, |
|
"loss": 2.4614, |
|
"theoretical_loss": 3.365009741149925, |
|
"tokens_seen": 2532966400 |
|
}, |
|
{ |
|
"epoch": 0.91, |
|
"learning_rate": 6.679591450728202e-05, |
|
"loss": 2.4944, |
|
"theoretical_loss": 3.364663700090004, |
|
"tokens_seen": 2536243200 |
|
}, |
|
{ |
|
"epoch": 0.91, |
|
"learning_rate": 6.596841308870815e-05, |
|
"loss": 2.4663, |
|
"theoretical_loss": 3.3643182308216986, |
|
"tokens_seen": 2539520000 |
|
}, |
|
{ |
|
"epoch": 0.91, |
|
"learning_rate": 6.514091167013429e-05, |
|
"loss": 2.4534, |
|
"theoretical_loss": 3.3639733316642233, |
|
"tokens_seen": 2542796800 |
|
}, |
|
{ |
|
"epoch": 0.91, |
|
"learning_rate": 6.431341025156043e-05, |
|
"loss": 2.467, |
|
"theoretical_loss": 3.36362900094389, |
|
"tokens_seen": 2546073600 |
|
}, |
|
{ |
|
"epoch": 0.91, |
|
"learning_rate": 6.348590883298658e-05, |
|
"loss": 2.4816, |
|
"theoretical_loss": 3.3632852369940696, |
|
"tokens_seen": 2549350400 |
|
}, |
|
{ |
|
"epoch": 0.91, |
|
"learning_rate": 6.265840741441271e-05, |
|
"loss": 2.4938, |
|
"theoretical_loss": 3.362942038155154, |
|
"tokens_seen": 2552627200 |
|
}, |
|
{ |
|
"epoch": 0.91, |
|
"learning_rate": 6.183090599583885e-05, |
|
"loss": 2.4269, |
|
"theoretical_loss": 3.362599402774518, |
|
"tokens_seen": 2555904000 |
|
}, |
|
{ |
|
"epoch": 0.91, |
|
"learning_rate": 6.1003404577264985e-05, |
|
"loss": 2.4912, |
|
"theoretical_loss": 3.362257329206479, |
|
"tokens_seen": 2559180800 |
|
}, |
|
{ |
|
"epoch": 0.92, |
|
"learning_rate": 6.017590315869113e-05, |
|
"loss": 2.5047, |
|
"theoretical_loss": 3.36191581581226, |
|
"tokens_seen": 2562457600 |
|
}, |
|
{ |
|
"epoch": 0.92, |
|
"learning_rate": 5.934840174011727e-05, |
|
"loss": 2.4899, |
|
"theoretical_loss": 3.3615748609599545, |
|
"tokens_seen": 2565734400 |
|
}, |
|
{ |
|
"epoch": 0.92, |
|
"learning_rate": 5.852090032154341e-05, |
|
"loss": 2.4645, |
|
"theoretical_loss": 3.361234463024483, |
|
"tokens_seen": 2569011200 |
|
}, |
|
{ |
|
"epoch": 0.92, |
|
"learning_rate": 5.769339890296954e-05, |
|
"loss": 2.4517, |
|
"theoretical_loss": 3.3608946203875614, |
|
"tokens_seen": 2572288000 |
|
}, |
|
{ |
|
"epoch": 0.92, |
|
"learning_rate": 5.686589748439569e-05, |
|
"loss": 2.4807, |
|
"theoretical_loss": 3.360555331437662, |
|
"tokens_seen": 2575564800 |
|
}, |
|
{ |
|
"epoch": 0.92, |
|
"learning_rate": 5.6038396065821823e-05, |
|
"loss": 2.5191, |
|
"theoretical_loss": 3.360216594569975, |
|
"tokens_seen": 2578841600 |
|
}, |
|
{ |
|
"epoch": 0.92, |
|
"learning_rate": 5.521089464724796e-05, |
|
"loss": 2.4516, |
|
"theoretical_loss": 3.3598784081863755, |
|
"tokens_seen": 2582118400 |
|
}, |
|
{ |
|
"epoch": 0.92, |
|
"learning_rate": 5.438339322867411e-05, |
|
"loss": 2.4888, |
|
"theoretical_loss": 3.3595407706953835, |
|
"tokens_seen": 2585395200 |
|
}, |
|
{ |
|
"epoch": 0.92, |
|
"learning_rate": 5.3555891810100246e-05, |
|
"loss": 2.4838, |
|
"theoretical_loss": 3.359203680512131, |
|
"tokens_seen": 2588672000 |
|
}, |
|
{ |
|
"epoch": 0.93, |
|
"learning_rate": 5.2728390391526385e-05, |
|
"loss": 2.4957, |
|
"theoretical_loss": 3.3588671360583247, |
|
"tokens_seen": 2591948800 |
|
}, |
|
{ |
|
"epoch": 0.93, |
|
"learning_rate": 5.190088897295252e-05, |
|
"loss": 2.5058, |
|
"theoretical_loss": 3.3585311357622096, |
|
"tokens_seen": 2595225600 |
|
}, |
|
{ |
|
"epoch": 0.93, |
|
"learning_rate": 5.107338755437867e-05, |
|
"loss": 2.5113, |
|
"theoretical_loss": 3.358195678058536, |
|
"tokens_seen": 2598502400 |
|
}, |
|
{ |
|
"epoch": 0.93, |
|
"learning_rate": 5.024588613580481e-05, |
|
"loss": 2.5538, |
|
"theoretical_loss": 3.3578607613885225, |
|
"tokens_seen": 2601779200 |
|
}, |
|
{ |
|
"epoch": 0.93, |
|
"learning_rate": 4.941838471723094e-05, |
|
"loss": 2.5468, |
|
"theoretical_loss": 3.357526384199822, |
|
"tokens_seen": 2605056000 |
|
}, |
|
{ |
|
"epoch": 0.93, |
|
"learning_rate": 4.859088329865708e-05, |
|
"loss": 2.5422, |
|
"theoretical_loss": 3.3571925449464874, |
|
"tokens_seen": 2608332800 |
|
}, |
|
{ |
|
"epoch": 0.93, |
|
"learning_rate": 4.776338188008322e-05, |
|
"loss": 2.5262, |
|
"theoretical_loss": 3.356859242088935, |
|
"tokens_seen": 2611609600 |
|
}, |
|
{ |
|
"epoch": 0.93, |
|
"learning_rate": 4.693588046150936e-05, |
|
"loss": 2.5143, |
|
"theoretical_loss": 3.356526474093914, |
|
"tokens_seen": 2614886400 |
|
}, |
|
{ |
|
"epoch": 0.94, |
|
"learning_rate": 4.61083790429355e-05, |
|
"loss": 2.5514, |
|
"theoretical_loss": 3.35619423943447, |
|
"tokens_seen": 2618163200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.449855913696806, |
|
"debugging/distinct-1-grams": 0.769778305351557, |
|
"debugging/distinct-2-grams": 0.9467712904639874, |
|
"debugging/entropy-1-grams": 5.998633443012235, |
|
"debugging/entropy-2-grams": 6.951908950814323, |
|
"debugging/length": 469.11764705882354, |
|
"debugging/num_segments": 17, |
|
"epoch": 0.94, |
|
"objective/train/avg_token_score": 0.039095163345336914, |
|
"objective/train/avg_weight": 0.9687336087226868, |
|
"objective/train/docs_used": 1474904, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.573209285736084, |
|
"objective/train/std_weight": 0.08868443965911865, |
|
"objective/train/theoretical_loss": 3.3558625365899117, |
|
"objective/train/tokens_used": 2641900000, |
|
"theoretical_loss": 3.3558625365899117, |
|
"tokens_seen": 2621440000 |
|
}, |
|
{ |
|
"epoch": 0.94, |
|
"learning_rate": 4.528087762436164e-05, |
|
"loss": 2.5746, |
|
"theoretical_loss": 3.3558625365899117, |
|
"tokens_seen": 2621440000 |
|
}, |
|
{ |
|
"epoch": 0.94, |
|
"learning_rate": 4.4453376205787784e-05, |
|
"loss": 2.5703, |
|
"theoretical_loss": 3.3555313640457776, |
|
"tokens_seen": 2624716800 |
|
}, |
|
{ |
|
"epoch": 0.94, |
|
"learning_rate": 4.362587478721392e-05, |
|
"loss": 2.5119, |
|
"theoretical_loss": 3.355200720293803, |
|
"tokens_seen": 2627993600 |
|
}, |
|
{ |
|
"epoch": 0.94, |
|
"learning_rate": 4.279837336864006e-05, |
|
"loss": 2.5345, |
|
"theoretical_loss": 3.3548706038318867, |
|
"tokens_seen": 2631270400 |
|
}, |
|
{ |
|
"epoch": 0.94, |
|
"learning_rate": 4.19708719500662e-05, |
|
"loss": 2.4929, |
|
"theoretical_loss": 3.3545410131640585, |
|
"tokens_seen": 2634547200 |
|
}, |
|
{ |
|
"epoch": 0.94, |
|
"learning_rate": 4.114337053149234e-05, |
|
"loss": 2.5131, |
|
"theoretical_loss": 3.354211946800446, |
|
"tokens_seen": 2637824000 |
|
}, |
|
{ |
|
"epoch": 0.94, |
|
"learning_rate": 4.031586911291848e-05, |
|
"loss": 2.513, |
|
"theoretical_loss": 3.3538834032572424, |
|
"tokens_seen": 2641100800 |
|
}, |
|
{ |
|
"epoch": 0.94, |
|
"learning_rate": 3.948836769434462e-05, |
|
"loss": 2.5339, |
|
"theoretical_loss": 3.3535553810566743, |
|
"tokens_seen": 2644377600 |
|
}, |
|
{ |
|
"epoch": 0.95, |
|
"learning_rate": 3.8660866275770754e-05, |
|
"loss": 2.5302, |
|
"theoretical_loss": 3.3532278787269707, |
|
"tokens_seen": 2647654400 |
|
}, |
|
{ |
|
"epoch": 0.95, |
|
"learning_rate": 3.78333648571969e-05, |
|
"loss": 2.506, |
|
"theoretical_loss": 3.35290089480233, |
|
"tokens_seen": 2650931200 |
|
}, |
|
{ |
|
"epoch": 0.95, |
|
"learning_rate": 3.700586343862304e-05, |
|
"loss": 2.514, |
|
"theoretical_loss": 3.352574427822888, |
|
"tokens_seen": 2654208000 |
|
}, |
|
{ |
|
"epoch": 0.95, |
|
"learning_rate": 3.617836202004918e-05, |
|
"loss": 2.513, |
|
"theoretical_loss": 3.3522484763346885, |
|
"tokens_seen": 2657484800 |
|
}, |
|
{ |
|
"epoch": 0.95, |
|
"learning_rate": 3.5350860601475315e-05, |
|
"loss": 2.506, |
|
"theoretical_loss": 3.3519230388896504, |
|
"tokens_seen": 2660761600 |
|
}, |
|
{ |
|
"epoch": 0.95, |
|
"learning_rate": 3.4523359182901454e-05, |
|
"loss": 2.5168, |
|
"theoretical_loss": 3.3515981140455375, |
|
"tokens_seen": 2664038400 |
|
}, |
|
{ |
|
"epoch": 0.95, |
|
"learning_rate": 3.369585776432759e-05, |
|
"loss": 2.5217, |
|
"theoretical_loss": 3.3512737003659288, |
|
"tokens_seen": 2667315200 |
|
}, |
|
{ |
|
"epoch": 0.95, |
|
"learning_rate": 3.286835634575374e-05, |
|
"loss": 2.522, |
|
"theoretical_loss": 3.350949796420184, |
|
"tokens_seen": 2670592000 |
|
}, |
|
{ |
|
"epoch": 0.95, |
|
"learning_rate": 3.204085492717987e-05, |
|
"loss": 2.5278, |
|
"theoretical_loss": 3.3506264007834186, |
|
"tokens_seen": 2673868800 |
|
}, |
|
{ |
|
"epoch": 0.96, |
|
"learning_rate": 3.1213353508606015e-05, |
|
"loss": 2.5438, |
|
"theoretical_loss": 3.3503035120364704, |
|
"tokens_seen": 2677145600 |
|
}, |
|
{ |
|
"epoch": 0.96, |
|
"learning_rate": 3.0385852090032157e-05, |
|
"loss": 2.5295, |
|
"theoretical_loss": 3.349981128765869, |
|
"tokens_seen": 2680422400 |
|
}, |
|
{ |
|
"epoch": 0.96, |
|
"learning_rate": 2.9558350671458292e-05, |
|
"loss": 2.5271, |
|
"theoretical_loss": 3.3496592495638096, |
|
"tokens_seen": 2683699200 |
|
}, |
|
{ |
|
"epoch": 0.96, |
|
"learning_rate": 2.8730849252884434e-05, |
|
"loss": 2.4913, |
|
"theoretical_loss": 3.349337873028119, |
|
"tokens_seen": 2686976000 |
|
}, |
|
{ |
|
"epoch": 0.96, |
|
"learning_rate": 2.7903347834310573e-05, |
|
"loss": 2.5216, |
|
"theoretical_loss": 3.349016997762229, |
|
"tokens_seen": 2690252800 |
|
}, |
|
{ |
|
"epoch": 0.96, |
|
"learning_rate": 2.7075846415736715e-05, |
|
"loss": 2.4842, |
|
"theoretical_loss": 3.348696622375147, |
|
"tokens_seen": 2693529600 |
|
}, |
|
{ |
|
"epoch": 0.96, |
|
"learning_rate": 2.624834499716285e-05, |
|
"loss": 2.5027, |
|
"theoretical_loss": 3.3483767454814255, |
|
"tokens_seen": 2696806400 |
|
}, |
|
{ |
|
"epoch": 0.96, |
|
"learning_rate": 2.5420843578588992e-05, |
|
"loss": 2.5116, |
|
"theoretical_loss": 3.3480573657011354, |
|
"tokens_seen": 2700083200 |
|
}, |
|
{ |
|
"epoch": 0.97, |
|
"learning_rate": 2.459334216001513e-05, |
|
"loss": 2.4871, |
|
"theoretical_loss": 3.347738481659836, |
|
"tokens_seen": 2703360000 |
|
}, |
|
{ |
|
"epoch": 0.97, |
|
"learning_rate": 2.3765840741441273e-05, |
|
"loss": 2.4743, |
|
"theoretical_loss": 3.347420091988546, |
|
"tokens_seen": 2706636800 |
|
}, |
|
{ |
|
"epoch": 0.97, |
|
"learning_rate": 2.2938339322867408e-05, |
|
"loss": 2.5035, |
|
"theoretical_loss": 3.3471021953237177, |
|
"tokens_seen": 2709913600 |
|
}, |
|
{ |
|
"epoch": 0.97, |
|
"learning_rate": 2.211083790429355e-05, |
|
"loss": 2.5179, |
|
"theoretical_loss": 3.346784790307205, |
|
"tokens_seen": 2713190400 |
|
}, |
|
{ |
|
"epoch": 0.97, |
|
"learning_rate": 2.1283336485719692e-05, |
|
"loss": 2.5056, |
|
"theoretical_loss": 3.346467875586239, |
|
"tokens_seen": 2716467200 |
|
}, |
|
{ |
|
"epoch": 0.97, |
|
"learning_rate": 2.045583506714583e-05, |
|
"loss": 2.5865, |
|
"theoretical_loss": 3.3461514498133997, |
|
"tokens_seen": 2719744000 |
|
}, |
|
{ |
|
"epoch": 0.97, |
|
"learning_rate": 1.962833364857197e-05, |
|
"loss": 2.582, |
|
"theoretical_loss": 3.345835511646587, |
|
"tokens_seen": 2723020800 |
|
}, |
|
{ |
|
"epoch": 0.97, |
|
"learning_rate": 1.8800832229998108e-05, |
|
"loss": 2.5641, |
|
"theoretical_loss": 3.345520059748993, |
|
"tokens_seen": 2726297600 |
|
}, |
|
{ |
|
"epoch": 0.97, |
|
"learning_rate": 1.797333081142425e-05, |
|
"loss": 2.5511, |
|
"theoretical_loss": 3.3452050927890777, |
|
"tokens_seen": 2729574400 |
|
}, |
|
{ |
|
"epoch": 0.98, |
|
"learning_rate": 1.7145829392850388e-05, |
|
"loss": 2.5666, |
|
"theoretical_loss": 3.3448906094405393, |
|
"tokens_seen": 2732851200 |
|
}, |
|
{ |
|
"epoch": 0.98, |
|
"learning_rate": 1.6318327974276527e-05, |
|
"loss": 2.5572, |
|
"theoretical_loss": 3.344576608382288, |
|
"tokens_seen": 2736128000 |
|
}, |
|
{ |
|
"epoch": 0.98, |
|
"learning_rate": 1.5490826555702665e-05, |
|
"loss": 2.5775, |
|
"theoretical_loss": 3.34426308829842, |
|
"tokens_seen": 2739404800 |
|
}, |
|
{ |
|
"epoch": 0.98, |
|
"learning_rate": 1.4663325137128806e-05, |
|
"loss": 2.5761, |
|
"theoretical_loss": 3.343950047878189, |
|
"tokens_seen": 2742681600 |
|
}, |
|
{ |
|
"epoch": 0.98, |
|
"learning_rate": 1.3835823718554946e-05, |
|
"loss": 2.587, |
|
"theoretical_loss": 3.343637485815983, |
|
"tokens_seen": 2745958400 |
|
}, |
|
{ |
|
"epoch": 0.98, |
|
"learning_rate": 1.3008322299981085e-05, |
|
"loss": 2.5711, |
|
"theoretical_loss": 3.3433254008112963, |
|
"tokens_seen": 2749235200 |
|
}, |
|
{ |
|
"epoch": 0.98, |
|
"learning_rate": 1.2180820881407225e-05, |
|
"loss": 2.5751, |
|
"theoretical_loss": 3.343013791568702, |
|
"tokens_seen": 2752512000 |
|
}, |
|
{ |
|
"epoch": 0.98, |
|
"learning_rate": 1.1353319462833365e-05, |
|
"loss": 2.5449, |
|
"theoretical_loss": 3.342702656797829, |
|
"tokens_seen": 2755788800 |
|
}, |
|
{ |
|
"epoch": 0.99, |
|
"learning_rate": 1.0525818044259504e-05, |
|
"loss": 2.524, |
|
"theoretical_loss": 3.342391995213335, |
|
"tokens_seen": 2759065600 |
|
}, |
|
{ |
|
"epoch": 0.99, |
|
"learning_rate": 9.698316625685644e-06, |
|
"loss": 2.5351, |
|
"theoretical_loss": 3.34208180553488, |
|
"tokens_seen": 2762342400 |
|
}, |
|
{ |
|
"epoch": 0.99, |
|
"learning_rate": 8.870815207111783e-06, |
|
"loss": 2.5167, |
|
"theoretical_loss": 3.341772086487102, |
|
"tokens_seen": 2765619200 |
|
}, |
|
{ |
|
"epoch": 0.99, |
|
"learning_rate": 8.043313788537923e-06, |
|
"loss": 2.5747, |
|
"theoretical_loss": 3.341462836799593, |
|
"tokens_seen": 2768896000 |
|
}, |
|
{ |
|
"epoch": 0.99, |
|
"learning_rate": 7.215812369964063e-06, |
|
"loss": 2.5676, |
|
"theoretical_loss": 3.341154055206871, |
|
"tokens_seen": 2772172800 |
|
}, |
|
{ |
|
"epoch": 0.99, |
|
"learning_rate": 6.388310951390203e-06, |
|
"loss": 2.6166, |
|
"theoretical_loss": 3.3408457404483576, |
|
"tokens_seen": 2775449600 |
|
}, |
|
{ |
|
"epoch": 0.99, |
|
"learning_rate": 5.560809532816342e-06, |
|
"loss": 2.5895, |
|
"theoretical_loss": 3.3405378912683514, |
|
"tokens_seen": 2778726400 |
|
}, |
|
{ |
|
"epoch": 0.99, |
|
"learning_rate": 4.7333081142424815e-06, |
|
"loss": 2.611, |
|
"theoretical_loss": 3.340230506416005, |
|
"tokens_seen": 2782003200 |
|
}, |
|
{ |
|
"epoch": 0.99, |
|
"objective/train/avg_token_score": 0.025402138009667397, |
|
"objective/train/avg_weight": 0.9796887040138245, |
|
"objective/train/docs_used": 1566749, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.5940380096435547, |
|
"objective/train/std_weight": 0.09304099529981613, |
|
"objective/train/theoretical_loss": 3.3399235846453004, |
|
"objective/train/tokens_used": 2805740000, |
|
"theoretical_loss": 3.3399235846453004, |
|
"tokens_seen": 2785280000 |
|
}, |
|
{ |
|
"epoch": 0.99, |
|
"learning_rate": 3.922356724040099e-06, |
|
"loss": 2.6241, |
|
"theoretical_loss": 3.3399235846453004, |
|
"tokens_seen": 2785280000 |
|
}, |
|
{ |
|
"epoch": 1.0, |
|
"learning_rate": 3.094855305466238e-06, |
|
"loss": 2.5785, |
|
"theoretical_loss": 3.3396171247150224, |
|
"tokens_seen": 2788556800 |
|
}, |
|
{ |
|
"epoch": 1.0, |
|
"learning_rate": 2.2673538868923775e-06, |
|
"loss": 2.5284, |
|
"theoretical_loss": 3.3393111253887375, |
|
"tokens_seen": 2791833600 |
|
}, |
|
{ |
|
"epoch": 1.0, |
|
"learning_rate": 1.4398524683185171e-06, |
|
"loss": 2.5451, |
|
"theoretical_loss": 3.339005585434769, |
|
"tokens_seen": 2795110400 |
|
}, |
|
{ |
|
"epoch": 1.0, |
|
"learning_rate": 6.123510497446567e-07, |
|
"loss": 2.5105, |
|
"theoretical_loss": 3.338700503626171, |
|
"tokens_seen": 2798387200 |
|
} |
|
], |
|
"max_steps": 42724, |
|
"num_train_epochs": 9223372036854775807, |
|
"total_flos": 1.428920419221504e+18, |
|
"trial_name": null, |
|
"trial_params": null |
|
} |
|
|