| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.5, | |
| "global_step": 21362, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 1.6355140186915887e-06, | |
| "loss": 10.8008, | |
| "theoretical_loss": 20.81281780154715, | |
| "tokens_seen": 65536 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 8.177570093457944e-05, | |
| "loss": 8.7055, | |
| "theoretical_loss": 8.563482664611069, | |
| "tokens_seen": 3276800 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001635514018691589, | |
| "loss": 6.6732, | |
| "theoretical_loss": 7.4777587180480305, | |
| "tokens_seen": 6553600 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0002453271028037383, | |
| "loss": 5.9716, | |
| "theoretical_loss": 6.9337544888949, | |
| "tokens_seen": 9830400 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0003271028037383178, | |
| "loss": 5.5864, | |
| "theoretical_loss": 6.583566228426414, | |
| "tokens_seen": 13107200 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004088785046728972, | |
| "loss": 5.3924, | |
| "theoretical_loss": 6.330713565116083, | |
| "tokens_seen": 16384000 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004906542056074766, | |
| "loss": 5.2081, | |
| "theoretical_loss": 6.135529231940326, | |
| "tokens_seen": 19660800 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0005724299065420561, | |
| "loss": 5.0469, | |
| "theoretical_loss": 5.978101583869607, | |
| "tokens_seen": 22937600 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0006542056074766356, | |
| "loss": 4.9522, | |
| "theoretical_loss": 5.8471173262659235, | |
| "tokens_seen": 26214400 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0006996358993758275, | |
| "loss": 4.7919, | |
| "theoretical_loss": 5.7355768158821245, | |
| "tokens_seen": 29491200 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0006988083979572536, | |
| "loss": 4.7314, | |
| "theoretical_loss": 5.638870144071353, | |
| "tokens_seen": 32768000 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0006979808965386797, | |
| "loss": 4.6075, | |
| "theoretical_loss": 5.553812381844907, | |
| "tokens_seen": 36044800 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000697153395120106, | |
| "loss": 4.5189, | |
| "theoretical_loss": 5.478118080556438, | |
| "tokens_seen": 39321600 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0006963258937015321, | |
| "loss": 4.4329, | |
| "theoretical_loss": 5.410095959579362, | |
| "tokens_seen": 42598400 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0006954983922829582, | |
| "loss": 4.4025, | |
| "theoretical_loss": 5.348462083735834, | |
| "tokens_seen": 45875200 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0006946708908643843, | |
| "loss": 4.25, | |
| "theoretical_loss": 5.292220566937567, | |
| "tokens_seen": 49152000 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0006938433894458105, | |
| "loss": 4.2174, | |
| "theoretical_loss": 5.240584625769978, | |
| "tokens_seen": 52428800 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0006930158880272367, | |
| "loss": 4.1421, | |
| "theoretical_loss": 5.192922724525789, | |
| "tokens_seen": 55705600 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0006921883866086628, | |
| "loss": 4.0643, | |
| "theoretical_loss": 5.1487208633564405, | |
| "tokens_seen": 58982400 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0006913608851900889, | |
| "loss": 3.9375, | |
| "theoretical_loss": 5.107555562405102, | |
| "tokens_seen": 62259200 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.000690533383771515, | |
| "loss": 3.8331, | |
| "theoretical_loss": 5.069074117143246, | |
| "tokens_seen": 65536000 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0006897058823529412, | |
| "loss": 3.7941, | |
| "theoretical_loss": 5.032979909838007, | |
| "tokens_seen": 68812800 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0006888783809343674, | |
| "loss": 3.7297, | |
| "theoretical_loss": 4.999021308224664, | |
| "tokens_seen": 72089600 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0006880508795157935, | |
| "loss": 3.6773, | |
| "theoretical_loss": 4.966983155351962, | |
| "tokens_seen": 75366400 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0006872233780972196, | |
| "loss": 3.5973, | |
| "theoretical_loss": 4.9366801616251355, | |
| "tokens_seen": 78643200 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0006863958766786457, | |
| "loss": 3.5893, | |
| "theoretical_loss": 4.907951713830082, | |
| "tokens_seen": 81920000 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0006855683752600718, | |
| "loss": 3.5262, | |
| "theoretical_loss": 4.880657753812926, | |
| "tokens_seen": 85196800 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.000684740873841498, | |
| "loss": 3.5248, | |
| "theoretical_loss": 4.854675474481779, | |
| "tokens_seen": 88473600 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0006839133724229242, | |
| "loss": 3.4372, | |
| "theoretical_loss": 4.8298966473088125, | |
| "tokens_seen": 91750400 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0006830858710043503, | |
| "loss": 3.4666, | |
| "theoretical_loss": 4.8062254427779205, | |
| "tokens_seen": 95027200 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0006822583695857764, | |
| "loss": 3.4167, | |
| "theoretical_loss": 4.783576639276257, | |
| "tokens_seen": 98304000 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0006814308681672025, | |
| "loss": 3.4719, | |
| "theoretical_loss": 4.761874140772408, | |
| "tokens_seen": 101580800 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0006806033667486286, | |
| "loss": 3.4687, | |
| "theoretical_loss": 4.741049741962473, | |
| "tokens_seen": 104857600 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0006797758653300548, | |
| "loss": 3.4353, | |
| "theoretical_loss": 4.721042093249051, | |
| "tokens_seen": 108134400 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.000678948363911481, | |
| "loss": 3.4114, | |
| "theoretical_loss": 4.701795828231866, | |
| "tokens_seen": 111411200 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0006781208624929071, | |
| "loss": 3.41, | |
| "theoretical_loss": 4.68326082423593, | |
| "tokens_seen": 114688000 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0006772933610743332, | |
| "loss": 3.3813, | |
| "theoretical_loss": 4.665391572426282, | |
| "tokens_seen": 117964800 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0006764658596557593, | |
| "loss": 3.3451, | |
| "theoretical_loss": 4.648146638719739, | |
| "tokens_seen": 121241600 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0006756383582371856, | |
| "loss": 3.3597, | |
| "theoretical_loss": 4.631488200339643, | |
| "tokens_seen": 124518400 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0006748108568186117, | |
| "loss": 3.3776, | |
| "theoretical_loss": 4.615381645715717, | |
| "tokens_seen": 127795200 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0006739833554000378, | |
| "loss": 3.3091, | |
| "theoretical_loss": 4.599795227690505, | |
| "tokens_seen": 131072000 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.000673155853981464, | |
| "loss": 3.3312, | |
| "theoretical_loss": 4.584699761792674, | |
| "tokens_seen": 134348800 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0006723283525628902, | |
| "loss": 3.2886, | |
| "theoretical_loss": 4.570068362778516, | |
| "tokens_seen": 137625600 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0006715008511443163, | |
| "loss": 3.3051, | |
| "theoretical_loss": 4.555876213804037, | |
| "tokens_seen": 140902400 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0006706733497257424, | |
| "loss": 3.2901, | |
| "theoretical_loss": 4.542100363530799, | |
| "tokens_seen": 144179200 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0006698458483071685, | |
| "loss": 3.2702, | |
| "theoretical_loss": 4.528719547234816, | |
| "tokens_seen": 147456000 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0006690183468885946, | |
| "loss": 3.2275, | |
| "theoretical_loss": 4.515714028614996, | |
| "tokens_seen": 150732800 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0006681908454700209, | |
| "loss": 3.2427, | |
| "theoretical_loss": 4.503065459513339, | |
| "tokens_seen": 154009600 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.000667363344051447, | |
| "loss": 3.2513, | |
| "theoretical_loss": 4.4907567551852665, | |
| "tokens_seen": 157286400 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0006665358426328731, | |
| "loss": 3.2329, | |
| "theoretical_loss": 4.478771983111967, | |
| "tokens_seen": 160563200 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "objective/train/avg_token_score": 0.027871694415807724, | |
| "objective/train/avg_weight": 0.977715790271759, | |
| "objective/train/docs_used": 104000, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 3.3409407138824463, | |
| "objective/train/std_weight": 0.06310887634754181, | |
| "objective/train/theoretical_loss": 4.467096263641219, | |
| "objective/train/tokens_used": 184300000, | |
| "theoretical_loss": 4.467096263641219, | |
| "tokens_seen": 163840000 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0006657083412142992, | |
| "loss": 3.2359, | |
| "theoretical_loss": 4.467096263641219, | |
| "tokens_seen": 163840000 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0006648808397957253, | |
| "loss": 3.3137, | |
| "theoretical_loss": 4.455715680989545, | |
| "tokens_seen": 167116800 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0006640533383771514, | |
| "loss": 3.2122, | |
| "theoretical_loss": 4.44461720334543, | |
| "tokens_seen": 170393600 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0006632258369585777, | |
| "loss": 3.1455, | |
| "theoretical_loss": 4.433788610987646, | |
| "tokens_seen": 173670400 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0006623983355400038, | |
| "loss": 3.198, | |
| "theoretical_loss": 4.42321843148016, | |
| "tokens_seen": 176947200 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0006615708341214299, | |
| "loss": 3.1453, | |
| "theoretical_loss": 4.412895881130142, | |
| "tokens_seen": 180224000 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.000660743332702856, | |
| "loss": 3.1834, | |
| "theoretical_loss": 4.4028108120020795, | |
| "tokens_seen": 183500800 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0006599158312842821, | |
| "loss": 3.1772, | |
| "theoretical_loss": 4.392953663871862, | |
| "tokens_seen": 186777600 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0006590883298657083, | |
| "loss": 3.1628, | |
| "theoretical_loss": 4.383315420582533, | |
| "tokens_seen": 190054400 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0006582608284471345, | |
| "loss": 3.1954, | |
| "theoretical_loss": 4.373887570330275, | |
| "tokens_seen": 193331200 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0006574333270285606, | |
| "loss": 3.1504, | |
| "theoretical_loss": 4.364662069466704, | |
| "tokens_seen": 196608000 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0006566058256099867, | |
| "loss": 3.1496, | |
| "theoretical_loss": 4.355631309453283, | |
| "tokens_seen": 199884800 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0006557783241914128, | |
| "loss": 3.1045, | |
| "theoretical_loss": 4.346788086646671, | |
| "tokens_seen": 203161600 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0006549508227728391, | |
| "loss": 3.1352, | |
| "theoretical_loss": 4.33812557463116, | |
| "tokens_seen": 206438400 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0006541233213542652, | |
| "loss": 3.1432, | |
| "theoretical_loss": 4.329637298846812, | |
| "tokens_seen": 209715200 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0006532958199356913, | |
| "loss": 3.106, | |
| "theoretical_loss": 4.321317113290252, | |
| "tokens_seen": 212992000 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0006524683185171174, | |
| "loss": 3.0747, | |
| "theoretical_loss": 4.3131591790897925, | |
| "tokens_seen": 216268800 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0006516408170985437, | |
| "loss": 3.1173, | |
| "theoretical_loss": 4.305157944778228, | |
| "tokens_seen": 219545600 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0006508133156799698, | |
| "loss": 3.1451, | |
| "theoretical_loss": 4.297308128105687, | |
| "tokens_seen": 222822400 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0006499858142613959, | |
| "loss": 3.1329, | |
| "theoretical_loss": 4.2896046992515995, | |
| "tokens_seen": 226099200 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.000649158312842822, | |
| "loss": 3.1677, | |
| "theoretical_loss": 4.282042865309616, | |
| "tokens_seen": 229376000 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0006483308114242481, | |
| "loss": 3.1376, | |
| "theoretical_loss": 4.274618055932298, | |
| "tokens_seen": 232652800 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0006475033100056744, | |
| "loss": 3.1095, | |
| "theoretical_loss": 4.267325910033897, | |
| "tokens_seen": 235929600 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0006466758085871005, | |
| "loss": 3.0555, | |
| "theoretical_loss": 4.260162263459744, | |
| "tokens_seen": 239206400 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0006458483071685266, | |
| "loss": 3.083, | |
| "theoretical_loss": 4.253123137539814, | |
| "tokens_seen": 242483200 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0006450208057499527, | |
| "loss": 3.1657, | |
| "theoretical_loss": 4.246204728452055, | |
| "tokens_seen": 245760000 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0006441933043313788, | |
| "loss": 3.1313, | |
| "theoretical_loss": 4.239403397328261, | |
| "tokens_seen": 249036800 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0006433658029128049, | |
| "loss": 3.0927, | |
| "theoretical_loss": 4.232715661041632, | |
| "tokens_seen": 252313600 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0006425383014942312, | |
| "loss": 3.1078, | |
| "theoretical_loss": 4.226138183620867, | |
| "tokens_seen": 255590400 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0006417108000756573, | |
| "loss": 3.0412, | |
| "theoretical_loss": 4.219667768240775, | |
| "tokens_seen": 258867200 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0006408832986570834, | |
| "loss": 3.045, | |
| "theoretical_loss": 4.213301349743924, | |
| "tokens_seen": 262144000 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0006400557972385095, | |
| "loss": 2.9994, | |
| "theoretical_loss": 4.20703598765197, | |
| "tokens_seen": 265420800 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0006392282958199356, | |
| "loss": 2.9626, | |
| "theoretical_loss": 4.2008688596290025, | |
| "tokens_seen": 268697600 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0006384007944013618, | |
| "loss": 2.9181, | |
| "theoretical_loss": 4.194797255362549, | |
| "tokens_seen": 271974400 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.000637573292982788, | |
| "loss": 3.0047, | |
| "theoretical_loss": 4.188818570830883, | |
| "tokens_seen": 275251200 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0006367457915642141, | |
| "loss": 3.0216, | |
| "theoretical_loss": 4.182930302927963, | |
| "tokens_seen": 278528000 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0006359182901456402, | |
| "loss": 2.9805, | |
| "theoretical_loss": 4.17713004441978, | |
| "tokens_seen": 281804800 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0006350907887270663, | |
| "loss": 3.0196, | |
| "theoretical_loss": 4.1714154792080915, | |
| "tokens_seen": 285081600 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0006342632873084925, | |
| "loss": 3.0324, | |
| "theoretical_loss": 4.165784377879517, | |
| "tokens_seen": 288358400 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0006334357858899187, | |
| "loss": 2.9929, | |
| "theoretical_loss": 4.160234593519768, | |
| "tokens_seen": 291635200 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0006326082844713448, | |
| "loss": 3.0177, | |
| "theoretical_loss": 4.15476405777444, | |
| "tokens_seen": 294912000 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0006317807830527709, | |
| "loss": 2.9921, | |
| "theoretical_loss": 4.149370777139286, | |
| "tokens_seen": 298188800 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0006309532816341972, | |
| "loss": 2.9847, | |
| "theoretical_loss": 4.144052829464249, | |
| "tokens_seen": 301465600 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0006301257802156233, | |
| "loss": 2.9369, | |
| "theoretical_loss": 4.138808360656742, | |
| "tokens_seen": 304742400 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0006292982787970494, | |
| "loss": 2.9483, | |
| "theoretical_loss": 4.133635581570836, | |
| "tokens_seen": 308019200 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0006284707773784755, | |
| "loss": 2.9245, | |
| "theoretical_loss": 4.128532765070004, | |
| "tokens_seen": 311296000 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0006276432759599016, | |
| "loss": 2.9521, | |
| "theoretical_loss": 4.123498243252032, | |
| "tokens_seen": 314572800 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0006268157745413279, | |
| "loss": 2.9375, | |
| "theoretical_loss": 4.118530404825556, | |
| "tokens_seen": 317849600 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.000625988273122754, | |
| "loss": 2.9599, | |
| "theoretical_loss": 4.113627692628464, | |
| "tokens_seen": 321126400 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0006251607717041801, | |
| "loss": 2.9643, | |
| "theoretical_loss": 4.108788601279149, | |
| "tokens_seen": 324403200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.5365128506817183, | |
| "debugging/distinct-1-grams": 0.7612814402327299, | |
| "debugging/distinct-2-grams": 0.9694583753853511, | |
| "debugging/entropy-1-grams": 6.003629944255698, | |
| "debugging/entropy-2-grams": 7.054987089269872, | |
| "debugging/length": 495.25, | |
| "debugging/num_segments": 16, | |
| "epoch": 0.12, | |
| "objective/train/avg_token_score": 0.04385810345411301, | |
| "objective/train/avg_weight": 0.9649326205253601, | |
| "objective/train/docs_used": 197327, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.908684015274048, | |
| "objective/train/std_weight": 0.12546230852603912, | |
| "objective/train/theoretical_loss": 4.10401167495222, | |
| "objective/train/tokens_used": 348140000, | |
| "theoretical_loss": 4.10401167495222, | |
| "tokens_seen": 327680000 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0006243332702856062, | |
| "loss": 2.9932, | |
| "theoretical_loss": 4.10401167495222, | |
| "tokens_seen": 327680000 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0006235057688670323, | |
| "loss": 2.9925, | |
| "theoretical_loss": 4.099295505270921, | |
| "tokens_seen": 330956800 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0006226782674484584, | |
| "loss": 2.9372, | |
| "theoretical_loss": 4.094638729309031, | |
| "tokens_seen": 334233600 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0006218507660298847, | |
| "loss": 2.9488, | |
| "theoretical_loss": 4.090040027695556, | |
| "tokens_seen": 337510400 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0006210232646113108, | |
| "loss": 2.906, | |
| "theoretical_loss": 4.085498122815992, | |
| "tokens_seen": 340787200 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0006201957631927369, | |
| "loss": 2.9313, | |
| "theoretical_loss": 4.081011777104333, | |
| "tokens_seen": 344064000 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.000619368261774163, | |
| "loss": 2.9368, | |
| "theoretical_loss": 4.076579791420469, | |
| "tokens_seen": 347340800 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0006185407603555891, | |
| "loss": 2.9504, | |
| "theoretical_loss": 4.0722010035079155, | |
| "tokens_seen": 350617600 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0006177132589370153, | |
| "loss": 2.9416, | |
| "theoretical_loss": 4.067874286527197, | |
| "tokens_seen": 353894400 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0006168857575184414, | |
| "loss": 2.9402, | |
| "theoretical_loss": 4.063598547660519, | |
| "tokens_seen": 357171200 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0006160582560998676, | |
| "loss": 2.9692, | |
| "theoretical_loss": 4.05937272678363, | |
| "tokens_seen": 360448000 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0006152307546812937, | |
| "loss": 2.957, | |
| "theoretical_loss": 4.055195795201069, | |
| "tokens_seen": 363724800 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0006144032532627198, | |
| "loss": 2.9066, | |
| "theoretical_loss": 4.051066754441235, | |
| "tokens_seen": 367001600 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.000613575751844146, | |
| "loss": 2.9128, | |
| "theoretical_loss": 4.04698463510794, | |
| "tokens_seen": 370278400 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0006127482504255721, | |
| "loss": 2.9181, | |
| "theoretical_loss": 4.042948495785312, | |
| "tokens_seen": 373555200 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0006119207490069983, | |
| "loss": 2.8869, | |
| "theoretical_loss": 4.038957421993153, | |
| "tokens_seen": 376832000 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0006110932475884244, | |
| "loss": 2.9055, | |
| "theoretical_loss": 4.035010525189982, | |
| "tokens_seen": 380108800 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0006102657461698505, | |
| "loss": 2.953, | |
| "theoretical_loss": 4.031106941821218, | |
| "tokens_seen": 383385600 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0006094382447512768, | |
| "loss": 2.9048, | |
| "theoretical_loss": 4.027245832410079, | |
| "tokens_seen": 386662400 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0006086107433327029, | |
| "loss": 2.8699, | |
| "theoretical_loss": 4.023426380688943, | |
| "tokens_seen": 389939200 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.000607783241914129, | |
| "loss": 2.9089, | |
| "theoretical_loss": 4.019647792769048, | |
| "tokens_seen": 393216000 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0006069722905239265, | |
| "loss": 2.9903, | |
| "theoretical_loss": 4.015909296346521, | |
| "tokens_seen": 396492800 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0006061447891053528, | |
| "loss": 2.9199, | |
| "theoretical_loss": 4.012210139942894, | |
| "tokens_seen": 399769600 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0006053172876867789, | |
| "loss": 2.9731, | |
| "theoretical_loss": 4.008549592178291, | |
| "tokens_seen": 403046400 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.000604489786268205, | |
| "loss": 2.9404, | |
| "theoretical_loss": 4.004926941075674, | |
| "tokens_seen": 406323200 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0006036622848496311, | |
| "loss": 2.9768, | |
| "theoretical_loss": 4.001341493394558, | |
| "tokens_seen": 409600000 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0006028347834310573, | |
| "loss": 2.9789, | |
| "theoretical_loss": 3.997792573992726, | |
| "tokens_seen": 412876800 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0006020072820124835, | |
| "loss": 2.9275, | |
| "theoretical_loss": 3.994279525214554, | |
| "tokens_seen": 416153600 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0006011797805939096, | |
| "loss": 2.9751, | |
| "theoretical_loss": 3.990801706304647, | |
| "tokens_seen": 419430400 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0006003522791753358, | |
| "loss": 2.9311, | |
| "theoretical_loss": 3.987358492845532, | |
| "tokens_seen": 422707200 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0005995247777567619, | |
| "loss": 2.9599, | |
| "theoretical_loss": 3.9839492762182647, | |
| "tokens_seen": 425984000 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.000598697276338188, | |
| "loss": 2.9319, | |
| "theoretical_loss": 3.9805734630848306, | |
| "tokens_seen": 429260800 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0005978697749196142, | |
| "loss": 2.9141, | |
| "theoretical_loss": 3.9772304748913054, | |
| "tokens_seen": 432537600 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0005970422735010403, | |
| "loss": 2.9076, | |
| "theoretical_loss": 3.973919747390801, | |
| "tokens_seen": 435814400 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0005962147720824664, | |
| "loss": 2.918, | |
| "theoretical_loss": 3.9706407301852487, | |
| "tokens_seen": 439091200 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0005953872706638926, | |
| "loss": 2.8962, | |
| "theoretical_loss": 3.9673928862851655, | |
| "tokens_seen": 442368000 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0005945597692453187, | |
| "loss": 2.887, | |
| "theoretical_loss": 3.9641756916865463, | |
| "tokens_seen": 445644800 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0005937322678267449, | |
| "loss": 2.8944, | |
| "theoretical_loss": 3.960988634964113, | |
| "tokens_seen": 448921600 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.000592904766408171, | |
| "loss": 2.8689, | |
| "theoretical_loss": 3.9578312168801597, | |
| "tokens_seen": 452198400 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0005920772649895971, | |
| "loss": 2.8671, | |
| "theoretical_loss": 3.954702950008308, | |
| "tokens_seen": 455475200 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0005912497635710232, | |
| "loss": 2.8527, | |
| "theoretical_loss": 3.9516033583714734, | |
| "tokens_seen": 458752000 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0005904222621524494, | |
| "loss": 2.8787, | |
| "theoretical_loss": 3.9485319770934355, | |
| "tokens_seen": 462028800 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0005895947607338756, | |
| "loss": 2.8612, | |
| "theoretical_loss": 3.945488352063391, | |
| "tokens_seen": 465305600 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0005887672593153017, | |
| "loss": 2.8394, | |
| "theoretical_loss": 3.942472039612926, | |
| "tokens_seen": 468582400 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0005879397578967278, | |
| "loss": 2.8409, | |
| "theoretical_loss": 3.939482606204863, | |
| "tokens_seen": 471859200 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0005871122564781539, | |
| "loss": 2.8556, | |
| "theoretical_loss": 3.936519628133466, | |
| "tokens_seen": 475136000 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.00058628475505958, | |
| "loss": 2.8919, | |
| "theoretical_loss": 3.9335826912355114, | |
| "tokens_seen": 478412800 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0005854572536410061, | |
| "loss": 2.9328, | |
| "theoretical_loss": 3.93067139061177, | |
| "tokens_seen": 481689600 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0005846297522224324, | |
| "loss": 2.859, | |
| "theoretical_loss": 3.927785330358441, | |
| "tokens_seen": 484966400 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0005838022508038585, | |
| "loss": 2.8503, | |
| "theoretical_loss": 3.9249241233081333, | |
| "tokens_seen": 488243200 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "objective/train/avg_token_score": 0.004644907079637051, | |
| "objective/train/avg_weight": 0.9962868094444275, | |
| "objective/train/docs_used": 287192, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 3.003862142562866, | |
| "objective/train/std_weight": 0.018475370481610298, | |
| "objective/train/theoretical_loss": 3.92208739077998, | |
| "objective/train/tokens_used": 511980000, | |
| "theoretical_loss": 3.92208739077998, | |
| "tokens_seen": 491520000 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0005829747493852846, | |
| "loss": 2.8251, | |
| "theoretical_loss": 3.92208739077998, | |
| "tokens_seen": 491520000 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0005821472479667107, | |
| "loss": 2.8393, | |
| "theoretical_loss": 3.919274762338519, | |
| "tokens_seen": 494796800 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.000581319746548137, | |
| "loss": 2.8629, | |
| "theoretical_loss": 3.9164858755609613, | |
| "tokens_seen": 498073600 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0005804922451295631, | |
| "loss": 2.8465, | |
| "theoretical_loss": 3.9137203758125176, | |
| "tokens_seen": 501350400 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0005796647437109892, | |
| "loss": 2.8088, | |
| "theoretical_loss": 3.910977916029439, | |
| "tokens_seen": 504627200 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0005788372422924154, | |
| "loss": 2.8193, | |
| "theoretical_loss": 3.908258156509472, | |
| "tokens_seen": 507904000 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0005780097408738415, | |
| "loss": 2.8096, | |
| "theoretical_loss": 3.905560764709417, | |
| "tokens_seen": 511180800 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0005771822394552677, | |
| "loss": 2.7896, | |
| "theoretical_loss": 3.9028854150495143, | |
| "tokens_seen": 514457600 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0005763547380366938, | |
| "loss": 2.8133, | |
| "theoretical_loss": 3.9002317887243834, | |
| "tokens_seen": 517734400 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0005755272366181199, | |
| "loss": 2.8119, | |
| "theoretical_loss": 3.897599573520247, | |
| "tokens_seen": 521011200 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0005746997351995461, | |
| "loss": 2.8456, | |
| "theoretical_loss": 3.8949884636382106, | |
| "tokens_seen": 524288000 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0005738722337809722, | |
| "loss": 2.8278, | |
| "theoretical_loss": 3.892398159523345, | |
| "tokens_seen": 527564800 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0005730447323623984, | |
| "loss": 2.8807, | |
| "theoretical_loss": 3.889828367699349, | |
| "tokens_seen": 530841600 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0005722172309438245, | |
| "loss": 2.8198, | |
| "theoretical_loss": 3.8872788006085894, | |
| "tokens_seen": 534118400 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0005713897295252506, | |
| "loss": 2.8464, | |
| "theoretical_loss": 3.8847491764572926, | |
| "tokens_seen": 537395200 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0005705622281066767, | |
| "loss": 2.8284, | |
| "theoretical_loss": 3.882239219065708, | |
| "tokens_seen": 540672000 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0005697347266881029, | |
| "loss": 2.8548, | |
| "theoretical_loss": 3.879748657723039, | |
| "tokens_seen": 543948800 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0005689072252695291, | |
| "loss": 2.8829, | |
| "theoretical_loss": 3.8772772270469824, | |
| "tokens_seen": 547225600 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0005680797238509552, | |
| "loss": 2.8451, | |
| "theoretical_loss": 3.8748246668476827, | |
| "tokens_seen": 550502400 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0005672522224323813, | |
| "loss": 2.8627, | |
| "theoretical_loss": 3.8723907219959486, | |
| "tokens_seen": 553779200 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0005664247210138074, | |
| "loss": 2.8789, | |
| "theoretical_loss": 3.869975142295573, | |
| "tokens_seen": 557056000 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0005655972195952335, | |
| "loss": 2.9283, | |
| "theoretical_loss": 3.8675776823595998, | |
| "tokens_seen": 560332800 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0005647697181766596, | |
| "loss": 2.8956, | |
| "theoretical_loss": 3.8651981014904027, | |
| "tokens_seen": 563609600 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0005639422167580859, | |
| "loss": 2.8693, | |
| "theoretical_loss": 3.8628361635634265, | |
| "tokens_seen": 566886400 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.000563114715339512, | |
| "loss": 2.8919, | |
| "theoretical_loss": 3.8604916369144666, | |
| "tokens_seen": 570163200 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0005622872139209381, | |
| "loss": 2.8761, | |
| "theoretical_loss": 3.858164294230354, | |
| "tokens_seen": 573440000 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0005614597125023642, | |
| "loss": 2.8493, | |
| "theoretical_loss": 3.85585391244293, | |
| "tokens_seen": 576716800 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0005606322110837905, | |
| "loss": 2.8528, | |
| "theoretical_loss": 3.8535602726261864, | |
| "tokens_seen": 579993600 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0005598047096652166, | |
| "loss": 2.8398, | |
| "theoretical_loss": 3.851283159896468, | |
| "tokens_seen": 583270400 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0005589772082466427, | |
| "loss": 2.8225, | |
| "theoretical_loss": 3.8490223633156173, | |
| "tokens_seen": 586547200 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0005581497068280688, | |
| "loss": 2.8074, | |
| "theoretical_loss": 3.846777675796974, | |
| "tokens_seen": 589824000 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.000557322205409495, | |
| "loss": 2.8092, | |
| "theoretical_loss": 3.844548894014116, | |
| "tokens_seen": 593100800 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0005564947039909212, | |
| "loss": 2.8185, | |
| "theoretical_loss": 3.8423358183122582, | |
| "tokens_seen": 596377600 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0005556672025723473, | |
| "loss": 2.7918, | |
| "theoretical_loss": 3.840138252622208, | |
| "tokens_seen": 599654400 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0005548397011537734, | |
| "loss": 2.8229, | |
| "theoretical_loss": 3.837956004376799, | |
| "tokens_seen": 602931200 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0005540121997351996, | |
| "loss": 2.7877, | |
| "theoretical_loss": 3.8357888844297094, | |
| "tokens_seen": 606208000 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0005531846983166257, | |
| "loss": 2.8112, | |
| "theoretical_loss": 3.8336367069765958, | |
| "tokens_seen": 609484800 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0005523571968980519, | |
| "loss": 2.7626, | |
| "theoretical_loss": 3.8314992894784536, | |
| "tokens_seen": 612761600 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.000551529695479478, | |
| "loss": 2.8083, | |
| "theoretical_loss": 3.829376452587134, | |
| "tokens_seen": 616038400 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0005507021940609041, | |
| "loss": 2.8228, | |
| "theoretical_loss": 3.827268020072948, | |
| "tokens_seen": 619315200 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0005498746926423302, | |
| "loss": 2.8554, | |
| "theoretical_loss": 3.8251738187542843, | |
| "tokens_seen": 622592000 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0005490471912237563, | |
| "loss": 2.8388, | |
| "theoretical_loss": 3.8230936784291787, | |
| "tokens_seen": 625868800 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0005482196898051826, | |
| "loss": 2.8705, | |
| "theoretical_loss": 3.8210274318087656, | |
| "tokens_seen": 629145600 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0005473921883866087, | |
| "loss": 2.8195, | |
| "theoretical_loss": 3.818974914452557, | |
| "tokens_seen": 632422400 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0005465646869680348, | |
| "loss": 2.8152, | |
| "theoretical_loss": 3.8169359647054835, | |
| "tokens_seen": 635699200 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0005457371855494609, | |
| "loss": 2.7996, | |
| "theoretical_loss": 3.8149104236366433, | |
| "tokens_seen": 638976000 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.000544909684130887, | |
| "loss": 2.7787, | |
| "theoretical_loss": 3.8128981349797098, | |
| "tokens_seen": 642252800 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0005440821827123131, | |
| "loss": 2.8041, | |
| "theoretical_loss": 3.8108989450749293, | |
| "tokens_seen": 645529600 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0005432546812937394, | |
| "loss": 2.7924, | |
| "theoretical_loss": 3.8089127028126764, | |
| "tokens_seen": 648806400 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0005424271798751655, | |
| "loss": 2.8324, | |
| "theoretical_loss": 3.8069392595785083, | |
| "tokens_seen": 652083200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.5265375629586004, | |
| "debugging/distinct-1-grams": 0.7435820408094715, | |
| "debugging/distinct-2-grams": 0.9558103821233092, | |
| "debugging/entropy-1-grams": 5.931434510687563, | |
| "debugging/entropy-2-grams": 6.886416755326388, | |
| "debugging/length": 521.9230769230769, | |
| "debugging/num_segments": 13, | |
| "epoch": 0.23, | |
| "objective/train/avg_token_score": 0.022742915898561478, | |
| "objective/train/avg_weight": 0.9818012714385986, | |
| "objective/train/docs_used": 379091, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 3.068922758102417, | |
| "objective/train/std_weight": 0.06274868547916412, | |
| "objective/train/theoretical_loss": 3.804978469199669, | |
| "objective/train/tokens_used": 675820000, | |
| "theoretical_loss": 3.804978469199669, | |
| "tokens_seen": 655360000 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0005415996784565916, | |
| "loss": 2.8223, | |
| "theoretical_loss": 3.804978469199669, | |
| "tokens_seen": 655360000 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0005407721770380177, | |
| "loss": 2.837, | |
| "theoretical_loss": 3.803030187893005, | |
| "tokens_seen": 658636800 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0005399612256478154, | |
| "loss": 2.8345, | |
| "theoretical_loss": 3.8010942742142415, | |
| "tokens_seen": 661913600 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0005391337242292415, | |
| "loss": 2.8643, | |
| "theoretical_loss": 3.799170589008585, | |
| "tokens_seen": 665190400 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0005383062228106677, | |
| "loss": 2.8481, | |
| "theoretical_loss": 3.7972589953626006, | |
| "tokens_seen": 668467200 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0005374787213920938, | |
| "loss": 2.8602, | |
| "theoretical_loss": 3.795359358557337, | |
| "tokens_seen": 671744000 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0005366512199735199, | |
| "loss": 2.8011, | |
| "theoretical_loss": 3.79347154602265, | |
| "tokens_seen": 675020800 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0005358237185549461, | |
| "loss": 2.8665, | |
| "theoretical_loss": 3.7915954272926955, | |
| "tokens_seen": 678297600 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0005350127671647437, | |
| "loss": 2.8685, | |
| "theoretical_loss": 3.789730873962557, | |
| "tokens_seen": 681574400 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0005341852657461698, | |
| "loss": 2.7516, | |
| "theoretical_loss": 3.787877759645963, | |
| "tokens_seen": 684851200 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.000533357764327596, | |
| "loss": 2.7768, | |
| "theoretical_loss": 3.7860359599340776, | |
| "tokens_seen": 688128000 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0005325302629090222, | |
| "loss": 2.8109, | |
| "theoretical_loss": 3.784205352355321, | |
| "tokens_seen": 691404800 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0005317027614904483, | |
| "loss": 2.8397, | |
| "theoretical_loss": 3.782385816336189, | |
| "tokens_seen": 694681600 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0005308752600718744, | |
| "loss": 2.7955, | |
| "theoretical_loss": 3.7805772331630516, | |
| "tokens_seen": 697958400 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0005300477586533005, | |
| "loss": 2.7725, | |
| "theoretical_loss": 3.7787794859448898, | |
| "tokens_seen": 701235200 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0005292202572347266, | |
| "loss": 2.754, | |
| "theoretical_loss": 3.7769924595769546, | |
| "tokens_seen": 704512000 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0005283927558161528, | |
| "loss": 2.7445, | |
| "theoretical_loss": 3.7752160407053115, | |
| "tokens_seen": 707788800 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.000527565254397579, | |
| "loss": 2.7262, | |
| "theoretical_loss": 3.7734501176922493, | |
| "tokens_seen": 711065600 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0005267377529790051, | |
| "loss": 2.7704, | |
| "theoretical_loss": 3.7716945805825337, | |
| "tokens_seen": 714342400 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0005259102515604312, | |
| "loss": 2.8113, | |
| "theoretical_loss": 3.7699493210704667, | |
| "tokens_seen": 717619200 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0005250827501418573, | |
| "loss": 2.7852, | |
| "theoretical_loss": 3.7682142324677455, | |
| "tokens_seen": 720896000 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0005242552487232835, | |
| "loss": 2.8514, | |
| "theoretical_loss": 3.7664892096720886, | |
| "tokens_seen": 724172800 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0005234277473047097, | |
| "loss": 2.7829, | |
| "theoretical_loss": 3.7647741491366067, | |
| "tokens_seen": 727449600 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0005226002458861358, | |
| "loss": 2.7945, | |
| "theoretical_loss": 3.7630689488399027, | |
| "tokens_seen": 730726400 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0005217727444675619, | |
| "loss": 2.7599, | |
| "theoretical_loss": 3.7613735082568764, | |
| "tokens_seen": 734003200 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0005209452430489881, | |
| "loss": 2.7937, | |
| "theoretical_loss": 3.759687728330217, | |
| "tokens_seen": 737280000 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0005201177416304143, | |
| "loss": 2.7877, | |
| "theoretical_loss": 3.75801151144256, | |
| "tokens_seen": 740556800 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0005192902402118404, | |
| "loss": 2.7713, | |
| "theoretical_loss": 3.756344761389295, | |
| "tokens_seen": 743833600 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0005184627387932665, | |
| "loss": 2.7429, | |
| "theoretical_loss": 3.754687383352003, | |
| "tokens_seen": 747110400 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0005176352373746927, | |
| "loss": 2.747, | |
| "theoretical_loss": 3.7530392838725097, | |
| "tokens_seen": 750387200 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0005168077359561188, | |
| "loss": 2.7654, | |
| "theoretical_loss": 3.751400370827529, | |
| "tokens_seen": 753664000 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.000515980234537545, | |
| "loss": 2.7878, | |
| "theoretical_loss": 3.749770553403895, | |
| "tokens_seen": 756940800 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0005151527331189711, | |
| "loss": 2.7522, | |
| "theoretical_loss": 3.748149742074355, | |
| "tokens_seen": 760217600 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0005143252317003972, | |
| "loss": 2.782, | |
| "theoretical_loss": 3.746537848573908, | |
| "tokens_seen": 763494400 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0005134977302818233, | |
| "loss": 2.7967, | |
| "theoretical_loss": 3.744934785876686, | |
| "tokens_seen": 766771200 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0005126702288632494, | |
| "loss": 2.802, | |
| "theoretical_loss": 3.7433404681733475, | |
| "tokens_seen": 770048000 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0005118427274446757, | |
| "loss": 2.8081, | |
| "theoretical_loss": 3.7417548108489846, | |
| "tokens_seen": 773324800 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0005110152260261018, | |
| "loss": 2.781, | |
| "theoretical_loss": 3.740177730461517, | |
| "tokens_seen": 776601600 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0005101877246075279, | |
| "loss": 2.8304, | |
| "theoretical_loss": 3.73860914472057, | |
| "tokens_seen": 779878400 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.000509360223188954, | |
| "loss": 2.7936, | |
| "theoretical_loss": 3.7370489724668197, | |
| "tokens_seen": 783155200 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0005085327217703801, | |
| "loss": 2.8074, | |
| "theoretical_loss": 3.735497133651788, | |
| "tokens_seen": 786432000 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0005077052203518063, | |
| "loss": 2.7948, | |
| "theoretical_loss": 3.733953549318091, | |
| "tokens_seen": 789708800 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0005068777189332325, | |
| "loss": 2.8081, | |
| "theoretical_loss": 3.7324181415801094, | |
| "tokens_seen": 792985600 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0005060502175146586, | |
| "loss": 2.8048, | |
| "theoretical_loss": 3.7308908336050814, | |
| "tokens_seen": 796262400 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0005052227160960847, | |
| "loss": 2.7886, | |
| "theoretical_loss": 3.729371549594614, | |
| "tokens_seen": 799539200 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0005043952146775108, | |
| "loss": 2.8075, | |
| "theoretical_loss": 3.7278602147665776, | |
| "tokens_seen": 802816000 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.000503567713258937, | |
| "loss": 2.7927, | |
| "theoretical_loss": 3.726356755337407, | |
| "tokens_seen": 806092800 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0005027402118403631, | |
| "loss": 2.7687, | |
| "theoretical_loss": 3.724861098504767, | |
| "tokens_seen": 809369600 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0005019127104217893, | |
| "loss": 2.748, | |
| "theoretical_loss": 3.7233731724305974, | |
| "tokens_seen": 812646400 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0005010852090032154, | |
| "loss": 2.7755, | |
| "theoretical_loss": 3.7218929062245105, | |
| "tokens_seen": 815923200 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "objective/train/avg_token_score": 0.009068925864994526, | |
| "objective/train/avg_weight": 0.9927405714988708, | |
| "objective/train/docs_used": 471128, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.671570301055908, | |
| "objective/train/std_weight": 0.04655551165342331, | |
| "objective/train/theoretical_loss": 3.7204202299275475, | |
| "objective/train/tokens_used": 839660000, | |
| "theoretical_loss": 3.7204202299275475, | |
| "tokens_seen": 819200000 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0005002577075846415, | |
| "loss": 2.7207, | |
| "theoretical_loss": 3.7204202299275475, | |
| "tokens_seen": 819200000 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0004994302061660678, | |
| "loss": 2.744, | |
| "theoretical_loss": 3.7189550744962707, | |
| "tokens_seen": 822476800 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0004986027047474939, | |
| "loss": 2.7245, | |
| "theoretical_loss": 3.717497371787192, | |
| "tokens_seen": 825753600 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.00049777520332892, | |
| "loss": 2.7221, | |
| "theoretical_loss": 3.7160470545415274, | |
| "tokens_seen": 829030400 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0004969477019103461, | |
| "loss": 2.721, | |
| "theoretical_loss": 3.714604056370267, | |
| "tokens_seen": 832307200 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0004961202004917723, | |
| "loss": 2.7335, | |
| "theoretical_loss": 3.713168311739558, | |
| "tokens_seen": 835584000 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0004952926990731985, | |
| "loss": 2.7169, | |
| "theoretical_loss": 3.7117397559563843, | |
| "tokens_seen": 838860800 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0004944651976546246, | |
| "loss": 2.7024, | |
| "theoretical_loss": 3.710318325154545, | |
| "tokens_seen": 842137600 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0004936376962360507, | |
| "loss": 2.7488, | |
| "theoretical_loss": 3.7089039562809223, | |
| "tokens_seen": 845414400 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0004928101948174768, | |
| "loss": 2.7216, | |
| "theoretical_loss": 3.7074965870820193, | |
| "tokens_seen": 848691200 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0004919826933989029, | |
| "loss": 2.7227, | |
| "theoretical_loss": 3.7060961560907857, | |
| "tokens_seen": 851968000 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.0004911551919803292, | |
| "loss": 2.78, | |
| "theoretical_loss": 3.7047026026137, | |
| "tokens_seen": 855244800 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.0004903276905617553, | |
| "loss": 2.7405, | |
| "theoretical_loss": 3.7033158667181154, | |
| "tokens_seen": 858521600 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.0004895001891431814, | |
| "loss": 2.7717, | |
| "theoretical_loss": 3.701935889219863, | |
| "tokens_seen": 861798400 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.0004886726877246075, | |
| "loss": 2.7393, | |
| "theoretical_loss": 3.7005626116710966, | |
| "tokens_seen": 865075200 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00048784518630603363, | |
| "loss": 2.7463, | |
| "theoretical_loss": 3.69919597634839, | |
| "tokens_seen": 868352000 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00048701768488745975, | |
| "loss": 2.7256, | |
| "theoretical_loss": 3.6978359262410603, | |
| "tokens_seen": 871628800 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.000486190183468886, | |
| "loss": 2.7512, | |
| "theoretical_loss": 3.6964824050397276, | |
| "tokens_seen": 874905600 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.0004853626820503121, | |
| "loss": 2.6954, | |
| "theoretical_loss": 3.6951353571251015, | |
| "tokens_seen": 878182400 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.0004845351806317382, | |
| "loss": 2.695, | |
| "theoretical_loss": 3.693794727556988, | |
| "tokens_seen": 881459200 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0004837076792131644, | |
| "loss": 2.699, | |
| "theoretical_loss": 3.692460462063506, | |
| "tokens_seen": 884736000 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0004828801777945905, | |
| "loss": 2.7523, | |
| "theoretical_loss": 3.691132507030521, | |
| "tokens_seen": 888012800 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0004820526763760166, | |
| "loss": 2.7663, | |
| "theoretical_loss": 3.6898108094912816, | |
| "tokens_seen": 891289600 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.00048122517495744274, | |
| "loss": 2.7245, | |
| "theoretical_loss": 3.6884953171162556, | |
| "tokens_seen": 894566400 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.00048039767353886897, | |
| "loss": 2.7305, | |
| "theoretical_loss": 3.6871859782031624, | |
| "tokens_seen": 897843200 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0004795701721202951, | |
| "loss": 2.7417, | |
| "theoretical_loss": 3.685882741667202, | |
| "tokens_seen": 901120000 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0004787426707017212, | |
| "loss": 2.744, | |
| "theoretical_loss": 3.684585557031461, | |
| "tokens_seen": 904396800 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0004779151692831473, | |
| "loss": 2.7572, | |
| "theoretical_loss": 3.6832943744175126, | |
| "tokens_seen": 907673600 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00047708766786457344, | |
| "loss": 2.7303, | |
| "theoretical_loss": 3.682009144536188, | |
| "tokens_seen": 910950400 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00047626016644599956, | |
| "loss": 2.7572, | |
| "theoretical_loss": 3.680729818678526, | |
| "tokens_seen": 914227200 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0004754326650274258, | |
| "loss": 2.7362, | |
| "theoretical_loss": 3.6794563487068936, | |
| "tokens_seen": 917504000 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0004746051636088519, | |
| "loss": 2.7454, | |
| "theoretical_loss": 3.6781886870462692, | |
| "tokens_seen": 920780800 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.000473777662190278, | |
| "loss": 2.7626, | |
| "theoretical_loss": 3.676926786675698, | |
| "tokens_seen": 924057600 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0004729501607717042, | |
| "loss": 2.7011, | |
| "theoretical_loss": 3.6756706011198963, | |
| "tokens_seen": 927334400 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0004721226593531303, | |
| "loss": 2.6682, | |
| "theoretical_loss": 3.6744200844410217, | |
| "tokens_seen": 930611200 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00047129515793455643, | |
| "loss": 2.6695, | |
| "theoretical_loss": 3.6731751912305914, | |
| "tokens_seen": 933888000 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00047046765651598266, | |
| "loss": 2.7057, | |
| "theoretical_loss": 3.671935876601547, | |
| "tokens_seen": 937164800 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0004696401550974088, | |
| "loss": 2.6439, | |
| "theoretical_loss": 3.6707020961804715, | |
| "tokens_seen": 940441600 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0004688126536788349, | |
| "loss": 2.6978, | |
| "theoretical_loss": 3.6694738060999468, | |
| "tokens_seen": 943718400 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.000467985152260261, | |
| "loss": 2.7145, | |
| "theoretical_loss": 3.668250962991049, | |
| "tokens_seen": 946995200 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00046715765084168713, | |
| "loss": 2.7161, | |
| "theoretical_loss": 3.667033523975983, | |
| "tokens_seen": 950272000 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00046633014942311325, | |
| "loss": 2.6986, | |
| "theoretical_loss": 3.66582144666085, | |
| "tokens_seen": 953548800 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0004655026480045394, | |
| "loss": 2.7021, | |
| "theoretical_loss": 3.664614689128546, | |
| "tokens_seen": 956825600 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0004646751465859656, | |
| "loss": 2.6985, | |
| "theoretical_loss": 3.6634132099317886, | |
| "tokens_seen": 960102400 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0004638476451673917, | |
| "loss": 2.7076, | |
| "theoretical_loss": 3.662216968086267, | |
| "tokens_seen": 963379200 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00046302014374881783, | |
| "loss": 2.7369, | |
| "theoretical_loss": 3.6610259230639217, | |
| "tokens_seen": 966656000 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.000462192642330244, | |
| "loss": 2.7711, | |
| "theoretical_loss": 3.659840034786333, | |
| "tokens_seen": 969932800 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0004613651409116701, | |
| "loss": 2.8314, | |
| "theoretical_loss": 3.6586592636182376, | |
| "tokens_seen": 973209600 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0004605541895214677, | |
| "loss": 2.8239, | |
| "theoretical_loss": 3.6574835703611566, | |
| "tokens_seen": 976486400 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00045972668810289393, | |
| "loss": 2.8097, | |
| "theoretical_loss": 3.6563129162471313, | |
| "tokens_seen": 979763200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.4286046663919377, | |
| "debugging/distinct-1-grams": 0.8147567798871364, | |
| "debugging/distinct-2-grams": 0.9823269374342457, | |
| "debugging/entropy-1-grams": 6.1671920556004824, | |
| "debugging/entropy-2-grams": 6.947028138756313, | |
| "debugging/length": 477.53333333333336, | |
| "debugging/num_segments": 15, | |
| "epoch": 0.35, | |
| "objective/train/avg_token_score": 0.020611366257071495, | |
| "objective/train/avg_weight": 0.9834998250007629, | |
| "objective/train/docs_used": 560408, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.9286718368530273, | |
| "objective/train/std_weight": 0.0680035874247551, | |
| "objective/train/theoretical_loss": 3.6551472629325787, | |
| "objective/train/tokens_used": 1003500000, | |
| "theoretical_loss": 3.6551472629325787, | |
| "tokens_seen": 983040000 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00045889918668432005, | |
| "loss": 2.8094, | |
| "theoretical_loss": 3.6551472629325787, | |
| "tokens_seen": 983040000 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00045807168526574616, | |
| "loss": 2.8104, | |
| "theoretical_loss": 3.653986572492247, | |
| "tokens_seen": 986316800 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0004572441838471723, | |
| "loss": 2.7787, | |
| "theoretical_loss": 3.65283080741328, | |
| "tokens_seen": 989593600 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0004564166824285984, | |
| "loss": 2.7906, | |
| "theoretical_loss": 3.6516799305893866, | |
| "tokens_seen": 992870400 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0004555891810100246, | |
| "loss": 2.7836, | |
| "theoretical_loss": 3.6505339053151076, | |
| "tokens_seen": 996147200 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.00045476167959145075, | |
| "loss": 2.7921, | |
| "theoretical_loss": 3.649392695280186, | |
| "tokens_seen": 999424000 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.00045393417817287686, | |
| "loss": 2.7558, | |
| "theoretical_loss": 3.6482562645640337, | |
| "tokens_seen": 1002700800 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.00045310667675430304, | |
| "loss": 2.761, | |
| "theoretical_loss": 3.6471245776302883, | |
| "tokens_seen": 1005977600 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.00045227917533572916, | |
| "loss": 2.7962, | |
| "theoretical_loss": 3.6459975993214724, | |
| "tokens_seen": 1009254400 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0004514516739171553, | |
| "loss": 2.7592, | |
| "theoretical_loss": 3.6448752948537377, | |
| "tokens_seen": 1012531200 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0004506241724985814, | |
| "loss": 2.7952, | |
| "theoretical_loss": 3.6437576298116996, | |
| "tokens_seen": 1015808000 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0004497966710800076, | |
| "loss": 2.7877, | |
| "theoretical_loss": 3.6426445701433607, | |
| "tokens_seen": 1019084800 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00044896916966143374, | |
| "loss": 2.8108, | |
| "theoretical_loss": 3.6415360821551226, | |
| "tokens_seen": 1022361600 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00044814166824285986, | |
| "loss": 2.7629, | |
| "theoretical_loss": 3.6404321325068754, | |
| "tokens_seen": 1025638400 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.000447314166824286, | |
| "loss": 2.7842, | |
| "theoretical_loss": 3.639332688207178, | |
| "tokens_seen": 1028915200 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0004464866654057121, | |
| "loss": 2.7649, | |
| "theoretical_loss": 3.6382377166085096, | |
| "tokens_seen": 1032192000 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0004456591639871382, | |
| "loss": 2.7244, | |
| "theoretical_loss": 3.6371471854026147, | |
| "tokens_seen": 1035468800 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0004448316625685644, | |
| "loss": 2.7313, | |
| "theoretical_loss": 3.6360610626159087, | |
| "tokens_seen": 1038745600 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00044400416114999055, | |
| "loss": 2.6993, | |
| "theoretical_loss": 3.634979316604973, | |
| "tokens_seen": 1042022400 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0004431766597314167, | |
| "loss": 2.714, | |
| "theoretical_loss": 3.6339019160521198, | |
| "tokens_seen": 1045299200 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00044234915831284285, | |
| "loss": 2.7069, | |
| "theoretical_loss": 3.632828829961029, | |
| "tokens_seen": 1048576000 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.00044152165689426896, | |
| "loss": 2.7232, | |
| "theoretical_loss": 3.631760027652461, | |
| "tokens_seen": 1051852800 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0004406941554756951, | |
| "loss": 2.7506, | |
| "theoretical_loss": 3.630695478760034, | |
| "tokens_seen": 1055129600 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0004398666540571212, | |
| "loss": 2.7249, | |
| "theoretical_loss": 3.6296351532260767, | |
| "tokens_seen": 1058406400 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0004390391526385474, | |
| "loss": 2.7194, | |
| "theoretical_loss": 3.6285790212975435, | |
| "tokens_seen": 1061683200 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.00043821165121997355, | |
| "loss": 2.6962, | |
| "theoretical_loss": 3.6275270535220008, | |
| "tokens_seen": 1064960000 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.00043738414980139966, | |
| "loss": 2.7203, | |
| "theoretical_loss": 3.626479220743673, | |
| "tokens_seen": 1068236800 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0004365566483828258, | |
| "loss": 2.7303, | |
| "theoretical_loss": 3.6254354940995586, | |
| "tokens_seen": 1071513600 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0004357291469642519, | |
| "loss": 2.7215, | |
| "theoretical_loss": 3.624395845015602, | |
| "tokens_seen": 1074790400 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00043490164554567807, | |
| "loss": 2.6763, | |
| "theoretical_loss": 3.6233602452029348, | |
| "tokens_seen": 1078067200 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00043407414412710424, | |
| "loss": 2.6939, | |
| "theoretical_loss": 3.6223286666541683, | |
| "tokens_seen": 1081344000 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00043324664270853036, | |
| "loss": 2.7216, | |
| "theoretical_loss": 3.621301081639753, | |
| "tokens_seen": 1084620800 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0004324191412899565, | |
| "loss": 2.7593, | |
| "theoretical_loss": 3.6202774627043923, | |
| "tokens_seen": 1087897600 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00043159163987138265, | |
| "loss": 2.7155, | |
| "theoretical_loss": 3.619257782663513, | |
| "tokens_seen": 1091174400 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00043078068848118023, | |
| "loss": 2.7244, | |
| "theoretical_loss": 3.618242014599793, | |
| "tokens_seen": 1094451200 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00042995318706260635, | |
| "loss": 2.7124, | |
| "theoretical_loss": 3.617230131859743, | |
| "tokens_seen": 1097728000 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00042912568564403247, | |
| "loss": 2.6959, | |
| "theoretical_loss": 3.6162221080503416, | |
| "tokens_seen": 1101004800 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0004282981842254587, | |
| "loss": 2.6816, | |
| "theoretical_loss": 3.615217917035726, | |
| "tokens_seen": 1104281600 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0004274706828068848, | |
| "loss": 2.7135, | |
| "theoretical_loss": 3.614217532933929, | |
| "tokens_seen": 1107558400 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00042664318138831093, | |
| "loss": 2.7048, | |
| "theoretical_loss": 3.6132209301136715, | |
| "tokens_seen": 1110835200 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00042581567996973705, | |
| "loss": 2.6877, | |
| "theoretical_loss": 3.612228083191205, | |
| "tokens_seen": 1114112000 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0004249881785511632, | |
| "loss": 2.7443, | |
| "theoretical_loss": 3.611238967027199, | |
| "tokens_seen": 1117388800 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00042416067713258934, | |
| "loss": 2.7844, | |
| "theoretical_loss": 3.610253556723679, | |
| "tokens_seen": 1120665600 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0004233331757140155, | |
| "loss": 2.7489, | |
| "theoretical_loss": 3.609271827621014, | |
| "tokens_seen": 1123942400 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0004225056742954417, | |
| "loss": 2.7523, | |
| "theoretical_loss": 3.6082937552949463, | |
| "tokens_seen": 1127219200 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0004216781728768678, | |
| "loss": 2.7191, | |
| "theoretical_loss": 3.607319315553669, | |
| "tokens_seen": 1130496000 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0004208506714582939, | |
| "loss": 2.6832, | |
| "theoretical_loss": 3.6063484844349456, | |
| "tokens_seen": 1133772800 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00042002317003972004, | |
| "loss": 2.7085, | |
| "theoretical_loss": 3.605381238203279, | |
| "tokens_seen": 1137049600 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00041919566862114616, | |
| "loss": 2.7165, | |
| "theoretical_loss": 3.604417553347117, | |
| "tokens_seen": 1140326400 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0004183681672025724, | |
| "loss": 2.7073, | |
| "theoretical_loss": 3.603457406576106, | |
| "tokens_seen": 1143603200 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "objective/train/avg_token_score": 0.022877871990203857, | |
| "objective/train/avg_weight": 0.9816967248916626, | |
| "objective/train/docs_used": 649861, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.648519992828369, | |
| "objective/train/std_weight": 0.07637551426887512, | |
| "objective/train/theoretical_loss": 3.602500774818379, | |
| "objective/train/tokens_used": 1167340000, | |
| "theoretical_loss": 3.602500774818379, | |
| "tokens_seen": 1146880000 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0004175406657839985, | |
| "loss": 2.6952, | |
| "theoretical_loss": 3.602500774818379, | |
| "tokens_seen": 1146880000 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0004167131643654246, | |
| "loss": 2.7354, | |
| "theoretical_loss": 3.601547635217892, | |
| "tokens_seen": 1150156800 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00041588566294685074, | |
| "loss": 2.7348, | |
| "theoretical_loss": 3.6005979651317976, | |
| "tokens_seen": 1153433600 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00041505816152827686, | |
| "loss": 2.7431, | |
| "theoretical_loss": 3.599651742127855, | |
| "tokens_seen": 1156710400 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00041423066010970303, | |
| "loss": 2.6952, | |
| "theoretical_loss": 3.5987089439818805, | |
| "tokens_seen": 1159987200 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00041340315869112915, | |
| "loss": 2.6743, | |
| "theoretical_loss": 3.5977695486752426, | |
| "tokens_seen": 1163264000 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0004125756572725553, | |
| "loss": 2.709, | |
| "theoretical_loss": 3.596833534392379, | |
| "tokens_seen": 1166540800 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0004117481558539815, | |
| "loss": 2.7274, | |
| "theoretical_loss": 3.595900879518368, | |
| "tokens_seen": 1169817600 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0004109206544354076, | |
| "loss": 2.7434, | |
| "theoretical_loss": 3.594971562636521, | |
| "tokens_seen": 1173094400 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00041009315301683373, | |
| "loss": 2.6976, | |
| "theoretical_loss": 3.5940455625260226, | |
| "tokens_seen": 1176371200 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00040926565159825985, | |
| "loss": 2.7078, | |
| "theoretical_loss": 3.5931228581595938, | |
| "tokens_seen": 1179648000 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00040843815017968597, | |
| "loss": 2.7067, | |
| "theoretical_loss": 3.5922034287011995, | |
| "tokens_seen": 1182924800 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0004076106487611122, | |
| "loss": 2.6717, | |
| "theoretical_loss": 3.5912872535037828, | |
| "tokens_seen": 1186201600 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0004067831473425383, | |
| "loss": 2.7545, | |
| "theoretical_loss": 3.590374312107035, | |
| "tokens_seen": 1189478400 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.00040595564592396443, | |
| "loss": 2.7398, | |
| "theoretical_loss": 3.5894645842351993, | |
| "tokens_seen": 1192755200 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.00040512814450539055, | |
| "loss": 2.7241, | |
| "theoretical_loss": 3.588558049794902, | |
| "tokens_seen": 1196032000 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0004043006430868167, | |
| "loss": 2.7036, | |
| "theoretical_loss": 3.5876546888730187, | |
| "tokens_seen": 1199308800 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.00040347314166824284, | |
| "loss": 2.7239, | |
| "theoretical_loss": 3.5867544817345713, | |
| "tokens_seen": 1202585600 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.000402645640249669, | |
| "loss": 2.7503, | |
| "theoretical_loss": 3.585857408820652, | |
| "tokens_seen": 1205862400 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.00040181813883109513, | |
| "loss": 2.7573, | |
| "theoretical_loss": 3.58496345074638, | |
| "tokens_seen": 1209139200 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0004009906374125213, | |
| "loss": 2.7716, | |
| "theoretical_loss": 3.5840725882988873, | |
| "tokens_seen": 1212416000 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0004001631359939474, | |
| "loss": 2.7395, | |
| "theoretical_loss": 3.5831848024353317, | |
| "tokens_seen": 1215692800 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00039933563457537354, | |
| "loss": 2.7634, | |
| "theoretical_loss": 3.5823000742809374, | |
| "tokens_seen": 1218969600 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00039850813315679966, | |
| "loss": 2.7348, | |
| "theoretical_loss": 3.5814183851270673, | |
| "tokens_seen": 1222246400 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0003976806317382258, | |
| "loss": 2.7239, | |
| "theoretical_loss": 3.5805397164293167, | |
| "tokens_seen": 1225523200 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.000396853130319652, | |
| "loss": 2.7059, | |
| "theoretical_loss": 3.5796640498056407, | |
| "tokens_seen": 1228800000 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0003960256289010781, | |
| "loss": 2.7119, | |
| "theoretical_loss": 3.5787913670345013, | |
| "tokens_seen": 1232076800 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00039519812748250424, | |
| "loss": 2.6905, | |
| "theoretical_loss": 3.577921650053045, | |
| "tokens_seen": 1235353600 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00039437062606393036, | |
| "loss": 2.687, | |
| "theoretical_loss": 3.577054880955303, | |
| "tokens_seen": 1238630400 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00039354312464535653, | |
| "loss": 2.6705, | |
| "theoretical_loss": 3.5761910419904193, | |
| "tokens_seen": 1241907200 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00039271562322678265, | |
| "loss": 2.7329, | |
| "theoretical_loss": 3.5753301155609014, | |
| "tokens_seen": 1245184000 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.0003918881218082088, | |
| "loss": 2.6739, | |
| "theoretical_loss": 3.574472084220896, | |
| "tokens_seen": 1248460800 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00039106062038963494, | |
| "loss": 2.6639, | |
| "theoretical_loss": 3.5736169306744885, | |
| "tokens_seen": 1251737600 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.0003902331189710611, | |
| "loss": 2.6337, | |
| "theoretical_loss": 3.572764637774024, | |
| "tokens_seen": 1255014400 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00038940561755248723, | |
| "loss": 2.6435, | |
| "theoretical_loss": 3.571915188518457, | |
| "tokens_seen": 1258291200 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00038857811613391335, | |
| "loss": 2.662, | |
| "theoretical_loss": 3.571068566051716, | |
| "tokens_seen": 1261568000 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00038775061471533947, | |
| "loss": 2.704, | |
| "theoretical_loss": 3.5702247536610976, | |
| "tokens_seen": 1264844800 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.0003869231132967657, | |
| "loss": 2.6831, | |
| "theoretical_loss": 3.5693837347756783, | |
| "tokens_seen": 1268121600 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.0003860956118781918, | |
| "loss": 2.6552, | |
| "theoretical_loss": 3.5685454929647475, | |
| "tokens_seen": 1271398400 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00038526811045961793, | |
| "loss": 2.6671, | |
| "theoretical_loss": 3.5677100119362675, | |
| "tokens_seen": 1274675200 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00038444060904104405, | |
| "loss": 2.6461, | |
| "theoretical_loss": 3.566877275535345, | |
| "tokens_seen": 1277952000 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00038361310762247017, | |
| "loss": 2.6451, | |
| "theoretical_loss": 3.566047267742733, | |
| "tokens_seen": 1281228800 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00038278560620389634, | |
| "loss": 2.6221, | |
| "theoretical_loss": 3.5652199726733453, | |
| "tokens_seen": 1284505600 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0003819581047853225, | |
| "loss": 2.6172, | |
| "theoretical_loss": 3.564395374574796, | |
| "tokens_seen": 1287782400 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00038113060336674863, | |
| "loss": 2.6767, | |
| "theoretical_loss": 3.5635734578259557, | |
| "tokens_seen": 1291059200 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0003803031019481748, | |
| "loss": 2.6908, | |
| "theoretical_loss": 3.5627542069355282, | |
| "tokens_seen": 1294336000 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0003794756005296009, | |
| "loss": 2.6578, | |
| "theoretical_loss": 3.5619376065406474, | |
| "tokens_seen": 1297612800 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00037864809911102704, | |
| "loss": 2.6722, | |
| "theoretical_loss": 3.5611236414054868, | |
| "tokens_seen": 1300889600 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00037782059769245316, | |
| "loss": 2.6776, | |
| "theoretical_loss": 3.560312296419899, | |
| "tokens_seen": 1304166400 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0003769930962738793, | |
| "loss": 2.6728, | |
| "theoretical_loss": 3.55950355659806, | |
| "tokens_seen": 1307443200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.49020908264157476, | |
| "debugging/distinct-1-grams": 0.768901113497886, | |
| "debugging/distinct-2-grams": 0.9428782333551957, | |
| "debugging/entropy-1-grams": 6.085999550681761, | |
| "debugging/entropy-2-grams": 7.0033060167714964, | |
| "debugging/length": 490.2352941176471, | |
| "debugging/num_segments": 17, | |
| "epoch": 0.47, | |
| "objective/train/avg_token_score": 0.02056093141436577, | |
| "objective/train/avg_weight": 0.983538031578064, | |
| "objective/train/docs_used": 741674, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.6528468132019043, | |
| "objective/train/std_weight": 0.0878894254565239, | |
| "objective/train/theoretical_loss": 3.558697407077142, | |
| "objective/train/tokens_used": 1331180000, | |
| "theoretical_loss": 3.558697407077142, | |
| "tokens_seen": 1310720000 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0003761655948553055, | |
| "loss": 2.6652, | |
| "theoretical_loss": 3.558697407077142, | |
| "tokens_seen": 1310720000 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0003753380934367316, | |
| "loss": 2.6785, | |
| "theoretical_loss": 3.5578938331159975, | |
| "tokens_seen": 1313996800 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00037451059201815774, | |
| "loss": 2.6523, | |
| "theoretical_loss": 3.557092820093863, | |
| "tokens_seen": 1317273600 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00037368309059958386, | |
| "loss": 2.6731, | |
| "theoretical_loss": 3.556294353509079, | |
| "tokens_seen": 1320550400 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00037285558918101, | |
| "loss": 2.6639, | |
| "theoretical_loss": 3.555498418977828, | |
| "tokens_seen": 1323827200 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00037202808776243615, | |
| "loss": 2.6579, | |
| "theoretical_loss": 3.5547050022328874, | |
| "tokens_seen": 1327104000 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.0003712005863438623, | |
| "loss": 2.6554, | |
| "theoretical_loss": 3.553914089122399, | |
| "tokens_seen": 1330380800 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00037037308492528844, | |
| "loss": 2.7062, | |
| "theoretical_loss": 3.553125665608655, | |
| "tokens_seen": 1333657600 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.0003695455835067146, | |
| "loss": 2.6814, | |
| "theoretical_loss": 3.5523397177669005, | |
| "tokens_seen": 1336934400 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00036871808208814073, | |
| "loss": 2.691, | |
| "theoretical_loss": 3.551556231784149, | |
| "tokens_seen": 1340211200 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00036789058066956685, | |
| "loss": 2.6452, | |
| "theoretical_loss": 3.5507751939580148, | |
| "tokens_seen": 1343488000 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00036706307925099297, | |
| "loss": 2.646, | |
| "theoretical_loss": 3.5499965906955606, | |
| "tokens_seen": 1346764800 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.0003662355778324192, | |
| "loss": 2.6952, | |
| "theoretical_loss": 3.549220408512161, | |
| "tokens_seen": 1350041600 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.0003654080764138453, | |
| "loss": 2.6716, | |
| "theoretical_loss": 3.5484466340303755, | |
| "tokens_seen": 1353318400 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00036458057499527143, | |
| "loss": 2.6566, | |
| "theoretical_loss": 3.547675253978843, | |
| "tokens_seen": 1356595200 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00036375307357669755, | |
| "loss": 2.6947, | |
| "theoretical_loss": 3.5469062551911854, | |
| "tokens_seen": 1359872000 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00036292557215812367, | |
| "loss": 2.7012, | |
| "theoretical_loss": 3.5461396246049244, | |
| "tokens_seen": 1363148800 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.0003620980707395498, | |
| "loss": 2.6675, | |
| "theoretical_loss": 3.545375349260419, | |
| "tokens_seen": 1366425600 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00036127056932097596, | |
| "loss": 2.7088, | |
| "theoretical_loss": 3.544613416299808, | |
| "tokens_seen": 1369702400 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00036044306790240213, | |
| "loss": 2.6847, | |
| "theoretical_loss": 3.5438538129659687, | |
| "tokens_seen": 1372979200 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00035961556648382825, | |
| "loss": 2.7015, | |
| "theoretical_loss": 3.5430965266014933, | |
| "tokens_seen": 1376256000 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.0003587880650652544, | |
| "loss": 2.7051, | |
| "theoretical_loss": 3.5423415446476705, | |
| "tokens_seen": 1379532800 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00035796056364668054, | |
| "loss": 2.6407, | |
| "theoretical_loss": 3.541588854643487, | |
| "tokens_seen": 1382809600 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.00035713306222810666, | |
| "loss": 2.6672, | |
| "theoretical_loss": 3.5408384442246343, | |
| "tokens_seen": 1386086400 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.0003563055608095328, | |
| "loss": 2.623, | |
| "theoretical_loss": 3.540090301122535, | |
| "tokens_seen": 1389363200 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.000355478059390959, | |
| "loss": 2.6465, | |
| "theoretical_loss": 3.5393444131633762, | |
| "tokens_seen": 1392640000 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.0003546505579723851, | |
| "loss": 2.696, | |
| "theoretical_loss": 3.5386007682671576, | |
| "tokens_seen": 1395916800 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.00035382305655381124, | |
| "loss": 2.6751, | |
| "theoretical_loss": 3.5378593544467494, | |
| "tokens_seen": 1399193600 | |
| } | |
| ], | |
| "max_steps": 42724, | |
| "num_train_epochs": 9223372036854775807, | |
| "total_flos": 7.14460209610752e+17, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |