| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.0, | |
| "global_step": 42724, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 1.6355140186915887e-06, | |
| "loss": 10.7824, | |
| "theoretical_loss": 20.81281780154715, | |
| "tokens_seen": 65536 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 8.177570093457944e-05, | |
| "loss": 8.6889, | |
| "theoretical_loss": 8.563482664611069, | |
| "tokens_seen": 3276800 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0001635514018691589, | |
| "loss": 6.6636, | |
| "theoretical_loss": 7.4777587180480305, | |
| "tokens_seen": 6553600 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0002453271028037383, | |
| "loss": 5.9745, | |
| "theoretical_loss": 6.9337544888949, | |
| "tokens_seen": 9830400 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0003271028037383178, | |
| "loss": 5.5783, | |
| "theoretical_loss": 6.583566228426414, | |
| "tokens_seen": 13107200 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004088785046728972, | |
| "loss": 5.3806, | |
| "theoretical_loss": 6.330713565116083, | |
| "tokens_seen": 16384000 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004906542056074766, | |
| "loss": 5.2075, | |
| "theoretical_loss": 6.135529231940326, | |
| "tokens_seen": 19660800 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0005724299065420561, | |
| "loss": 5.0385, | |
| "theoretical_loss": 5.978101583869607, | |
| "tokens_seen": 22937600 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0006542056074766356, | |
| "loss": 4.946, | |
| "theoretical_loss": 5.8471173262659235, | |
| "tokens_seen": 26214400 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0006996358993758275, | |
| "loss": 4.7943, | |
| "theoretical_loss": 5.7355768158821245, | |
| "tokens_seen": 29491200 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0006988083979572536, | |
| "loss": 4.7314, | |
| "theoretical_loss": 5.638870144071353, | |
| "tokens_seen": 32768000 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0006979808965386797, | |
| "loss": 4.6052, | |
| "theoretical_loss": 5.553812381844907, | |
| "tokens_seen": 36044800 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000697153395120106, | |
| "loss": 4.5231, | |
| "theoretical_loss": 5.478118080556438, | |
| "tokens_seen": 39321600 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0006963258937015321, | |
| "loss": 4.4354, | |
| "theoretical_loss": 5.410095959579362, | |
| "tokens_seen": 42598400 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0006954983922829582, | |
| "loss": 4.4109, | |
| "theoretical_loss": 5.348462083735834, | |
| "tokens_seen": 45875200 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0006946708908643843, | |
| "loss": 4.258, | |
| "theoretical_loss": 5.292220566937567, | |
| "tokens_seen": 49152000 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0006938433894458105, | |
| "loss": 4.2162, | |
| "theoretical_loss": 5.240584625769978, | |
| "tokens_seen": 52428800 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0006930158880272367, | |
| "loss": 4.1529, | |
| "theoretical_loss": 5.192922724525789, | |
| "tokens_seen": 55705600 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0006921883866086628, | |
| "loss": 4.0864, | |
| "theoretical_loss": 5.1487208633564405, | |
| "tokens_seen": 58982400 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0006913608851900889, | |
| "loss": 3.9637, | |
| "theoretical_loss": 5.107555562405102, | |
| "tokens_seen": 62259200 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.000690533383771515, | |
| "loss": 3.8525, | |
| "theoretical_loss": 5.069074117143246, | |
| "tokens_seen": 65536000 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0006897058823529412, | |
| "loss": 3.8185, | |
| "theoretical_loss": 5.032979909838007, | |
| "tokens_seen": 68812800 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0006888783809343674, | |
| "loss": 3.759, | |
| "theoretical_loss": 4.999021308224664, | |
| "tokens_seen": 72089600 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0006880508795157935, | |
| "loss": 3.703, | |
| "theoretical_loss": 4.966983155351962, | |
| "tokens_seen": 75366400 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0006872233780972196, | |
| "loss": 3.6079, | |
| "theoretical_loss": 4.9366801616251355, | |
| "tokens_seen": 78643200 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0006863958766786457, | |
| "loss": 3.5951, | |
| "theoretical_loss": 4.907951713830082, | |
| "tokens_seen": 81920000 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0006855683752600718, | |
| "loss": 3.533, | |
| "theoretical_loss": 4.880657753812926, | |
| "tokens_seen": 85196800 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.000684740873841498, | |
| "loss": 3.5388, | |
| "theoretical_loss": 4.854675474481779, | |
| "tokens_seen": 88473600 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0006839133724229242, | |
| "loss": 3.4465, | |
| "theoretical_loss": 4.8298966473088125, | |
| "tokens_seen": 91750400 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0006830858710043503, | |
| "loss": 3.4702, | |
| "theoretical_loss": 4.8062254427779205, | |
| "tokens_seen": 95027200 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0006822583695857764, | |
| "loss": 3.4191, | |
| "theoretical_loss": 4.783576639276257, | |
| "tokens_seen": 98304000 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0006814308681672025, | |
| "loss": 3.4562, | |
| "theoretical_loss": 4.761874140772408, | |
| "tokens_seen": 101580800 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0006806033667486286, | |
| "loss": 3.4558, | |
| "theoretical_loss": 4.741049741962473, | |
| "tokens_seen": 104857600 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0006797758653300548, | |
| "loss": 3.4434, | |
| "theoretical_loss": 4.721042093249051, | |
| "tokens_seen": 108134400 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.000678948363911481, | |
| "loss": 3.4108, | |
| "theoretical_loss": 4.701795828231866, | |
| "tokens_seen": 111411200 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0006781208624929071, | |
| "loss": 3.4049, | |
| "theoretical_loss": 4.68326082423593, | |
| "tokens_seen": 114688000 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0006772933610743332, | |
| "loss": 3.3773, | |
| "theoretical_loss": 4.665391572426282, | |
| "tokens_seen": 117964800 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0006764658596557593, | |
| "loss": 3.3413, | |
| "theoretical_loss": 4.648146638719739, | |
| "tokens_seen": 121241600 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0006756383582371856, | |
| "loss": 3.356, | |
| "theoretical_loss": 4.631488200339643, | |
| "tokens_seen": 124518400 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0006748108568186117, | |
| "loss": 3.3753, | |
| "theoretical_loss": 4.615381645715717, | |
| "tokens_seen": 127795200 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0006739833554000378, | |
| "loss": 3.3066, | |
| "theoretical_loss": 4.599795227690505, | |
| "tokens_seen": 131072000 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.000673155853981464, | |
| "loss": 3.3297, | |
| "theoretical_loss": 4.584699761792674, | |
| "tokens_seen": 134348800 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0006723283525628902, | |
| "loss": 3.284, | |
| "theoretical_loss": 4.570068362778516, | |
| "tokens_seen": 137625600 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0006715008511443163, | |
| "loss": 3.3003, | |
| "theoretical_loss": 4.555876213804037, | |
| "tokens_seen": 140902400 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0006706733497257424, | |
| "loss": 3.2875, | |
| "theoretical_loss": 4.542100363530799, | |
| "tokens_seen": 144179200 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0006698458483071685, | |
| "loss": 3.2669, | |
| "theoretical_loss": 4.528719547234816, | |
| "tokens_seen": 147456000 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0006690183468885946, | |
| "loss": 3.2242, | |
| "theoretical_loss": 4.515714028614996, | |
| "tokens_seen": 150732800 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0006681908454700209, | |
| "loss": 3.2343, | |
| "theoretical_loss": 4.503065459513339, | |
| "tokens_seen": 154009600 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.000667363344051447, | |
| "loss": 3.2442, | |
| "theoretical_loss": 4.4907567551852665, | |
| "tokens_seen": 157286400 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0006665358426328731, | |
| "loss": 3.2286, | |
| "theoretical_loss": 4.478771983111967, | |
| "tokens_seen": 160563200 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "objective/train/avg_token_score": 0.027871694415807724, | |
| "objective/train/avg_weight": 0.9746770858764648, | |
| "objective/train/docs_used": 104000, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 3.3261609077453613, | |
| "objective/train/std_weight": 0.07171463221311569, | |
| "objective/train/theoretical_loss": 4.467096263641219, | |
| "objective/train/tokens_used": 184300000, | |
| "theoretical_loss": 4.467096263641219, | |
| "tokens_seen": 163840000 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0006657083412142992, | |
| "loss": 3.2241, | |
| "theoretical_loss": 4.467096263641219, | |
| "tokens_seen": 163840000 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0006648808397957253, | |
| "loss": 3.2466, | |
| "theoretical_loss": 4.455715680989545, | |
| "tokens_seen": 167116800 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0006640533383771514, | |
| "loss": 3.1999, | |
| "theoretical_loss": 4.44461720334543, | |
| "tokens_seen": 170393600 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0006632258369585777, | |
| "loss": 3.1426, | |
| "theoretical_loss": 4.433788610987646, | |
| "tokens_seen": 173670400 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0006623983355400038, | |
| "loss": 3.1918, | |
| "theoretical_loss": 4.42321843148016, | |
| "tokens_seen": 176947200 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0006615708341214299, | |
| "loss": 3.1379, | |
| "theoretical_loss": 4.412895881130142, | |
| "tokens_seen": 180224000 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.000660743332702856, | |
| "loss": 3.1736, | |
| "theoretical_loss": 4.4028108120020795, | |
| "tokens_seen": 183500800 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0006599158312842821, | |
| "loss": 3.1695, | |
| "theoretical_loss": 4.392953663871862, | |
| "tokens_seen": 186777600 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0006590883298657083, | |
| "loss": 3.1562, | |
| "theoretical_loss": 4.383315420582533, | |
| "tokens_seen": 190054400 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0006582608284471345, | |
| "loss": 3.1895, | |
| "theoretical_loss": 4.373887570330275, | |
| "tokens_seen": 193331200 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0006574333270285606, | |
| "loss": 3.1465, | |
| "theoretical_loss": 4.364662069466704, | |
| "tokens_seen": 196608000 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0006566058256099867, | |
| "loss": 3.146, | |
| "theoretical_loss": 4.355631309453283, | |
| "tokens_seen": 199884800 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0006557783241914128, | |
| "loss": 3.0991, | |
| "theoretical_loss": 4.346788086646671, | |
| "tokens_seen": 203161600 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0006549508227728391, | |
| "loss": 3.1286, | |
| "theoretical_loss": 4.33812557463116, | |
| "tokens_seen": 206438400 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0006541233213542652, | |
| "loss": 3.1357, | |
| "theoretical_loss": 4.329637298846812, | |
| "tokens_seen": 209715200 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0006532958199356913, | |
| "loss": 3.0929, | |
| "theoretical_loss": 4.321317113290252, | |
| "tokens_seen": 212992000 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0006524683185171174, | |
| "loss": 3.0686, | |
| "theoretical_loss": 4.3131591790897925, | |
| "tokens_seen": 216268800 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0006516408170985437, | |
| "loss": 3.1167, | |
| "theoretical_loss": 4.305157944778228, | |
| "tokens_seen": 219545600 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0006508133156799698, | |
| "loss": 3.1412, | |
| "theoretical_loss": 4.297308128105687, | |
| "tokens_seen": 222822400 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0006499858142613959, | |
| "loss": 3.1237, | |
| "theoretical_loss": 4.2896046992515995, | |
| "tokens_seen": 226099200 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.000649158312842822, | |
| "loss": 3.1605, | |
| "theoretical_loss": 4.282042865309616, | |
| "tokens_seen": 229376000 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0006483308114242481, | |
| "loss": 3.1312, | |
| "theoretical_loss": 4.274618055932298, | |
| "tokens_seen": 232652800 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0006475033100056744, | |
| "loss": 3.0994, | |
| "theoretical_loss": 4.267325910033897, | |
| "tokens_seen": 235929600 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0006466758085871005, | |
| "loss": 3.049, | |
| "theoretical_loss": 4.260162263459744, | |
| "tokens_seen": 239206400 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0006458483071685266, | |
| "loss": 3.0873, | |
| "theoretical_loss": 4.253123137539814, | |
| "tokens_seen": 242483200 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0006450208057499527, | |
| "loss": 3.1018, | |
| "theoretical_loss": 4.246204728452055, | |
| "tokens_seen": 245760000 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0006441933043313788, | |
| "loss": 3.116, | |
| "theoretical_loss": 4.239403397328261, | |
| "tokens_seen": 249036800 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0006433658029128049, | |
| "loss": 3.0758, | |
| "theoretical_loss": 4.232715661041632, | |
| "tokens_seen": 252313600 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0006425383014942312, | |
| "loss": 3.0471, | |
| "theoretical_loss": 4.226138183620867, | |
| "tokens_seen": 255590400 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0006417108000756573, | |
| "loss": 3.0065, | |
| "theoretical_loss": 4.219667768240775, | |
| "tokens_seen": 258867200 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0006408832986570834, | |
| "loss": 3.0265, | |
| "theoretical_loss": 4.213301349743924, | |
| "tokens_seen": 262144000 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0006400557972385095, | |
| "loss": 2.978, | |
| "theoretical_loss": 4.20703598765197, | |
| "tokens_seen": 265420800 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0006392282958199356, | |
| "loss": 2.9473, | |
| "theoretical_loss": 4.2008688596290025, | |
| "tokens_seen": 268697600 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0006384007944013618, | |
| "loss": 2.9058, | |
| "theoretical_loss": 4.194797255362549, | |
| "tokens_seen": 271974400 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.000637573292982788, | |
| "loss": 2.9845, | |
| "theoretical_loss": 4.188818570830883, | |
| "tokens_seen": 275251200 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0006367457915642141, | |
| "loss": 3.0175, | |
| "theoretical_loss": 4.182930302927963, | |
| "tokens_seen": 278528000 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0006359182901456402, | |
| "loss": 2.9712, | |
| "theoretical_loss": 4.17713004441978, | |
| "tokens_seen": 281804800 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0006350907887270663, | |
| "loss": 3.0091, | |
| "theoretical_loss": 4.1714154792080915, | |
| "tokens_seen": 285081600 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0006342632873084925, | |
| "loss": 3.0192, | |
| "theoretical_loss": 4.165784377879517, | |
| "tokens_seen": 288358400 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0006334357858899187, | |
| "loss": 2.9853, | |
| "theoretical_loss": 4.160234593519768, | |
| "tokens_seen": 291635200 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0006326082844713448, | |
| "loss": 3.0073, | |
| "theoretical_loss": 4.15476405777444, | |
| "tokens_seen": 294912000 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0006317807830527709, | |
| "loss": 2.9819, | |
| "theoretical_loss": 4.149370777139286, | |
| "tokens_seen": 298188800 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0006309532816341972, | |
| "loss": 2.9752, | |
| "theoretical_loss": 4.144052829464249, | |
| "tokens_seen": 301465600 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0006301257802156233, | |
| "loss": 2.9458, | |
| "theoretical_loss": 4.138808360656742, | |
| "tokens_seen": 304742400 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0006292982787970494, | |
| "loss": 2.945, | |
| "theoretical_loss": 4.133635581570836, | |
| "tokens_seen": 308019200 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0006284707773784755, | |
| "loss": 2.9216, | |
| "theoretical_loss": 4.128532765070004, | |
| "tokens_seen": 311296000 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0006276432759599016, | |
| "loss": 2.9457, | |
| "theoretical_loss": 4.123498243252032, | |
| "tokens_seen": 314572800 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0006268157745413279, | |
| "loss": 2.9304, | |
| "theoretical_loss": 4.118530404825556, | |
| "tokens_seen": 317849600 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.000625988273122754, | |
| "loss": 2.9513, | |
| "theoretical_loss": 4.113627692628464, | |
| "tokens_seen": 321126400 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0006251607717041801, | |
| "loss": 2.9564, | |
| "theoretical_loss": 4.108788601279149, | |
| "tokens_seen": 324403200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.5365128506817183, | |
| "debugging/distinct-1-grams": 0.7612814402327299, | |
| "debugging/distinct-2-grams": 0.9694583753853511, | |
| "debugging/entropy-1-grams": 6.003629944255698, | |
| "debugging/entropy-2-grams": 7.054987089269872, | |
| "debugging/length": 495.25, | |
| "debugging/num_segments": 16, | |
| "epoch": 0.12, | |
| "objective/train/avg_token_score": 0.04385810345411301, | |
| "objective/train/avg_weight": 0.9601507186889648, | |
| "objective/train/docs_used": 197327, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.9074478149414062, | |
| "objective/train/std_weight": 0.14257079362869263, | |
| "objective/train/theoretical_loss": 4.10401167495222, | |
| "objective/train/tokens_used": 348140000, | |
| "theoretical_loss": 4.10401167495222, | |
| "tokens_seen": 327680000 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0006243332702856062, | |
| "loss": 2.986, | |
| "theoretical_loss": 4.10401167495222, | |
| "tokens_seen": 327680000 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0006235057688670323, | |
| "loss": 2.9852, | |
| "theoretical_loss": 4.099295505270921, | |
| "tokens_seen": 330956800 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0006226782674484584, | |
| "loss": 2.9316, | |
| "theoretical_loss": 4.094638729309031, | |
| "tokens_seen": 334233600 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0006218507660298847, | |
| "loss": 2.9442, | |
| "theoretical_loss": 4.090040027695556, | |
| "tokens_seen": 337510400 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0006210232646113108, | |
| "loss": 2.9001, | |
| "theoretical_loss": 4.085498122815992, | |
| "tokens_seen": 340787200 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0006201957631927369, | |
| "loss": 2.9256, | |
| "theoretical_loss": 4.081011777104333, | |
| "tokens_seen": 344064000 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.000619368261774163, | |
| "loss": 2.9297, | |
| "theoretical_loss": 4.076579791420469, | |
| "tokens_seen": 347340800 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0006185407603555891, | |
| "loss": 2.9439, | |
| "theoretical_loss": 4.0722010035079155, | |
| "tokens_seen": 350617600 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0006177132589370153, | |
| "loss": 2.9351, | |
| "theoretical_loss": 4.067874286527197, | |
| "tokens_seen": 353894400 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0006168857575184414, | |
| "loss": 2.9326, | |
| "theoretical_loss": 4.063598547660519, | |
| "tokens_seen": 357171200 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0006160582560998676, | |
| "loss": 2.9614, | |
| "theoretical_loss": 4.05937272678363, | |
| "tokens_seen": 360448000 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0006152307546812937, | |
| "loss": 2.9476, | |
| "theoretical_loss": 4.055195795201069, | |
| "tokens_seen": 363724800 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0006144032532627198, | |
| "loss": 2.9058, | |
| "theoretical_loss": 4.051066754441235, | |
| "tokens_seen": 367001600 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.000613575751844146, | |
| "loss": 2.9081, | |
| "theoretical_loss": 4.04698463510794, | |
| "tokens_seen": 370278400 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0006127482504255721, | |
| "loss": 2.9117, | |
| "theoretical_loss": 4.042948495785312, | |
| "tokens_seen": 373555200 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0006119207490069983, | |
| "loss": 2.8717, | |
| "theoretical_loss": 4.038957421993153, | |
| "tokens_seen": 376832000 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0006110932475884244, | |
| "loss": 2.881, | |
| "theoretical_loss": 4.035010525189982, | |
| "tokens_seen": 380108800 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0006102657461698505, | |
| "loss": 2.9145, | |
| "theoretical_loss": 4.031106941821218, | |
| "tokens_seen": 383385600 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0006094382447512768, | |
| "loss": 2.877, | |
| "theoretical_loss": 4.027245832410079, | |
| "tokens_seen": 386662400 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0006086107433327029, | |
| "loss": 2.8523, | |
| "theoretical_loss": 4.023426380688943, | |
| "tokens_seen": 389939200 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.000607783241914129, | |
| "loss": 2.894, | |
| "theoretical_loss": 4.019647792769048, | |
| "tokens_seen": 393216000 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0006069557404955551, | |
| "loss": 2.9554, | |
| "theoretical_loss": 4.015909296346521, | |
| "tokens_seen": 396492800 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0006061282390769814, | |
| "loss": 2.8873, | |
| "theoretical_loss": 4.012210139942894, | |
| "tokens_seen": 399769600 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0006053007376584075, | |
| "loss": 2.9151, | |
| "theoretical_loss": 4.008549592178291, | |
| "tokens_seen": 403046400 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0006044732362398336, | |
| "loss": 2.8973, | |
| "theoretical_loss": 4.004926941075674, | |
| "tokens_seen": 406323200 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0006036457348212597, | |
| "loss": 2.9519, | |
| "theoretical_loss": 4.001341493394558, | |
| "tokens_seen": 409600000 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0006028182334026858, | |
| "loss": 2.9595, | |
| "theoretical_loss": 3.997792573992726, | |
| "tokens_seen": 412876800 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0006019907319841119, | |
| "loss": 2.9127, | |
| "theoretical_loss": 3.994279525214554, | |
| "tokens_seen": 416153600 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0006011632305655381, | |
| "loss": 2.9624, | |
| "theoretical_loss": 3.990801706304647, | |
| "tokens_seen": 419430400 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0006003357291469643, | |
| "loss": 2.9186, | |
| "theoretical_loss": 3.987358492845532, | |
| "tokens_seen": 422707200 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0005995082277283904, | |
| "loss": 2.9497, | |
| "theoretical_loss": 3.9839492762182647, | |
| "tokens_seen": 425984000 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0005986807263098165, | |
| "loss": 2.9201, | |
| "theoretical_loss": 3.9805734630848306, | |
| "tokens_seen": 429260800 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0005978532248912426, | |
| "loss": 2.9038, | |
| "theoretical_loss": 3.9772304748913054, | |
| "tokens_seen": 432537600 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0005970257234726688, | |
| "loss": 2.8974, | |
| "theoretical_loss": 3.973919747390801, | |
| "tokens_seen": 435814400 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0005961982220540949, | |
| "loss": 2.9079, | |
| "theoretical_loss": 3.9706407301852487, | |
| "tokens_seen": 439091200 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0005953707206355211, | |
| "loss": 2.8871, | |
| "theoretical_loss": 3.9673928862851655, | |
| "tokens_seen": 442368000 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0005945432192169472, | |
| "loss": 2.8781, | |
| "theoretical_loss": 3.9641756916865463, | |
| "tokens_seen": 445644800 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0005937157177983733, | |
| "loss": 2.8865, | |
| "theoretical_loss": 3.960988634964113, | |
| "tokens_seen": 448921600 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0005928882163797995, | |
| "loss": 2.8596, | |
| "theoretical_loss": 3.9578312168801597, | |
| "tokens_seen": 452198400 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0005920607149612256, | |
| "loss": 2.8591, | |
| "theoretical_loss": 3.954702950008308, | |
| "tokens_seen": 455475200 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0005912332135426518, | |
| "loss": 2.8435, | |
| "theoretical_loss": 3.9516033583714734, | |
| "tokens_seen": 458752000 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0005904057121240779, | |
| "loss": 2.8688, | |
| "theoretical_loss": 3.9485319770934355, | |
| "tokens_seen": 462028800 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.000589578210705504, | |
| "loss": 2.8546, | |
| "theoretical_loss": 3.945488352063391, | |
| "tokens_seen": 465305600 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0005887507092869302, | |
| "loss": 2.8741, | |
| "theoretical_loss": 3.942472039612926, | |
| "tokens_seen": 468582400 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0005879232078683564, | |
| "loss": 2.8452, | |
| "theoretical_loss": 3.939482606204863, | |
| "tokens_seen": 471859200 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0005870957064497825, | |
| "loss": 2.8528, | |
| "theoretical_loss": 3.936519628133466, | |
| "tokens_seen": 475136000 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0005862682050312086, | |
| "loss": 2.8866, | |
| "theoretical_loss": 3.9335826912355114, | |
| "tokens_seen": 478412800 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0005854407036126347, | |
| "loss": 2.9264, | |
| "theoretical_loss": 3.93067139061177, | |
| "tokens_seen": 481689600 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.000584613202194061, | |
| "loss": 2.8535, | |
| "theoretical_loss": 3.927785330358441, | |
| "tokens_seen": 484966400 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0005837857007754871, | |
| "loss": 2.8422, | |
| "theoretical_loss": 3.9249241233081333, | |
| "tokens_seen": 488243200 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "objective/train/avg_token_score": 0.004644907079637051, | |
| "objective/train/avg_weight": 0.9957804679870605, | |
| "objective/train/docs_used": 287192, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 3.0019359588623047, | |
| "objective/train/std_weight": 0.02099473774433136, | |
| "objective/train/theoretical_loss": 3.92208739077998, | |
| "objective/train/tokens_used": 511980000, | |
| "theoretical_loss": 3.92208739077998, | |
| "tokens_seen": 491520000 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0005829581993569132, | |
| "loss": 2.8177, | |
| "theoretical_loss": 3.92208739077998, | |
| "tokens_seen": 491520000 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0005821306979383393, | |
| "loss": 2.829, | |
| "theoretical_loss": 3.919274762338519, | |
| "tokens_seen": 494796800 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0005813031965197654, | |
| "loss": 2.8542, | |
| "theoretical_loss": 3.9164858755609613, | |
| "tokens_seen": 498073600 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0005804756951011916, | |
| "loss": 2.8375, | |
| "theoretical_loss": 3.9137203758125176, | |
| "tokens_seen": 501350400 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0005796481936826178, | |
| "loss": 2.7995, | |
| "theoretical_loss": 3.910977916029439, | |
| "tokens_seen": 504627200 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0005788206922640439, | |
| "loss": 2.8108, | |
| "theoretical_loss": 3.908258156509472, | |
| "tokens_seen": 507904000 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.00057799319084547, | |
| "loss": 2.8003, | |
| "theoretical_loss": 3.905560764709417, | |
| "tokens_seen": 511180800 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0005771656894268961, | |
| "loss": 2.7796, | |
| "theoretical_loss": 3.9028854150495143, | |
| "tokens_seen": 514457600 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0005763381880083223, | |
| "loss": 2.8048, | |
| "theoretical_loss": 3.9002317887243834, | |
| "tokens_seen": 517734400 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0005755106865897484, | |
| "loss": 2.803, | |
| "theoretical_loss": 3.897599573520247, | |
| "tokens_seen": 521011200 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0005746831851711746, | |
| "loss": 2.834, | |
| "theoretical_loss": 3.8949884636382106, | |
| "tokens_seen": 524288000 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0005738556837526007, | |
| "loss": 2.8202, | |
| "theoretical_loss": 3.892398159523345, | |
| "tokens_seen": 527564800 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0005730281823340268, | |
| "loss": 2.8716, | |
| "theoretical_loss": 3.889828367699349, | |
| "tokens_seen": 530841600 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.000572200680915453, | |
| "loss": 2.8106, | |
| "theoretical_loss": 3.8872788006085894, | |
| "tokens_seen": 534118400 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0005713731794968791, | |
| "loss": 2.8356, | |
| "theoretical_loss": 3.8847491764572926, | |
| "tokens_seen": 537395200 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0005705456780783052, | |
| "loss": 2.8189, | |
| "theoretical_loss": 3.882239219065708, | |
| "tokens_seen": 540672000 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0005697181766597314, | |
| "loss": 2.8447, | |
| "theoretical_loss": 3.879748657723039, | |
| "tokens_seen": 543948800 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0005688906752411575, | |
| "loss": 2.8729, | |
| "theoretical_loss": 3.8772772270469824, | |
| "tokens_seen": 547225600 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0005680631738225837, | |
| "loss": 2.8352, | |
| "theoretical_loss": 3.8748246668476827, | |
| "tokens_seen": 550502400 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0005672356724040099, | |
| "loss": 2.8532, | |
| "theoretical_loss": 3.8723907219959486, | |
| "tokens_seen": 553779200 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.000566408170985436, | |
| "loss": 2.8698, | |
| "theoretical_loss": 3.869975142295573, | |
| "tokens_seen": 557056000 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0005655806695668621, | |
| "loss": 2.9208, | |
| "theoretical_loss": 3.8675776823595998, | |
| "tokens_seen": 560332800 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0005647531681482882, | |
| "loss": 2.889, | |
| "theoretical_loss": 3.8651981014904027, | |
| "tokens_seen": 563609600 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0005639422167580859, | |
| "loss": 2.8663, | |
| "theoretical_loss": 3.8628361635634265, | |
| "tokens_seen": 566886400 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.000563114715339512, | |
| "loss": 2.9028, | |
| "theoretical_loss": 3.8604916369144666, | |
| "tokens_seen": 570163200 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0005622872139209381, | |
| "loss": 2.8744, | |
| "theoretical_loss": 3.858164294230354, | |
| "tokens_seen": 573440000 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0005614597125023642, | |
| "loss": 2.8465, | |
| "theoretical_loss": 3.85585391244293, | |
| "tokens_seen": 576716800 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0005606322110837905, | |
| "loss": 2.8498, | |
| "theoretical_loss": 3.8535602726261864, | |
| "tokens_seen": 579993600 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0005598047096652166, | |
| "loss": 2.8343, | |
| "theoretical_loss": 3.851283159896468, | |
| "tokens_seen": 583270400 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0005589772082466427, | |
| "loss": 2.8178, | |
| "theoretical_loss": 3.8490223633156173, | |
| "tokens_seen": 586547200 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0005581497068280688, | |
| "loss": 2.8002, | |
| "theoretical_loss": 3.846777675796974, | |
| "tokens_seen": 589824000 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.000557322205409495, | |
| "loss": 2.801, | |
| "theoretical_loss": 3.844548894014116, | |
| "tokens_seen": 593100800 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0005564947039909212, | |
| "loss": 2.8109, | |
| "theoretical_loss": 3.8423358183122582, | |
| "tokens_seen": 596377600 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0005556672025723473, | |
| "loss": 2.7858, | |
| "theoretical_loss": 3.840138252622208, | |
| "tokens_seen": 599654400 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0005548397011537734, | |
| "loss": 2.8172, | |
| "theoretical_loss": 3.837956004376799, | |
| "tokens_seen": 602931200 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0005540121997351996, | |
| "loss": 2.7818, | |
| "theoretical_loss": 3.8357888844297094, | |
| "tokens_seen": 606208000 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0005531846983166257, | |
| "loss": 2.8023, | |
| "theoretical_loss": 3.8336367069765958, | |
| "tokens_seen": 609484800 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0005523571968980519, | |
| "loss": 2.755, | |
| "theoretical_loss": 3.8314992894784536, | |
| "tokens_seen": 612761600 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.000551529695479478, | |
| "loss": 2.7988, | |
| "theoretical_loss": 3.829376452587134, | |
| "tokens_seen": 616038400 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0005507021940609041, | |
| "loss": 2.8135, | |
| "theoretical_loss": 3.827268020072948, | |
| "tokens_seen": 619315200 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0005498746926423302, | |
| "loss": 2.8462, | |
| "theoretical_loss": 3.8251738187542843, | |
| "tokens_seen": 622592000 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0005490471912237563, | |
| "loss": 2.8284, | |
| "theoretical_loss": 3.8230936784291787, | |
| "tokens_seen": 625868800 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0005482196898051826, | |
| "loss": 2.8606, | |
| "theoretical_loss": 3.8210274318087656, | |
| "tokens_seen": 629145600 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0005473921883866087, | |
| "loss": 2.8068, | |
| "theoretical_loss": 3.818974914452557, | |
| "tokens_seen": 632422400 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0005465646869680348, | |
| "loss": 2.8042, | |
| "theoretical_loss": 3.8169359647054835, | |
| "tokens_seen": 635699200 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0005457371855494609, | |
| "loss": 2.7892, | |
| "theoretical_loss": 3.8149104236366433, | |
| "tokens_seen": 638976000 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.000544909684130887, | |
| "loss": 2.7712, | |
| "theoretical_loss": 3.8128981349797098, | |
| "tokens_seen": 642252800 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0005440821827123131, | |
| "loss": 2.7953, | |
| "theoretical_loss": 3.8108989450749293, | |
| "tokens_seen": 645529600 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0005432546812937394, | |
| "loss": 2.7863, | |
| "theoretical_loss": 3.8089127028126764, | |
| "tokens_seen": 648806400 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0005424271798751655, | |
| "loss": 2.8319, | |
| "theoretical_loss": 3.8069392595785083, | |
| "tokens_seen": 652083200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.5265375629586004, | |
| "debugging/distinct-1-grams": 0.7435820408094715, | |
| "debugging/distinct-2-grams": 0.9558103821233092, | |
| "debugging/entropy-1-grams": 5.931434510687563, | |
| "debugging/entropy-2-grams": 6.886416755326388, | |
| "debugging/length": 521.9230769230769, | |
| "debugging/num_segments": 13, | |
| "epoch": 0.23, | |
| "objective/train/avg_token_score": 0.022742915898561478, | |
| "objective/train/avg_weight": 0.9793195724487305, | |
| "objective/train/docs_used": 379091, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 3.077260732650757, | |
| "objective/train/std_weight": 0.07130531966686249, | |
| "objective/train/theoretical_loss": 3.804978469199669, | |
| "objective/train/tokens_used": 675820000, | |
| "theoretical_loss": 3.804978469199669, | |
| "tokens_seen": 655360000 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0005416162284849631, | |
| "loss": 2.8518, | |
| "theoretical_loss": 3.804978469199669, | |
| "tokens_seen": 655360000 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0005407887270663892, | |
| "loss": 2.8406, | |
| "theoretical_loss": 3.803030187893005, | |
| "tokens_seen": 658636800 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0005399612256478154, | |
| "loss": 2.8343, | |
| "theoretical_loss": 3.8010942742142415, | |
| "tokens_seen": 661913600 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0005391337242292415, | |
| "loss": 2.8597, | |
| "theoretical_loss": 3.799170589008585, | |
| "tokens_seen": 665190400 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0005383062228106677, | |
| "loss": 2.8447, | |
| "theoretical_loss": 3.7972589953626006, | |
| "tokens_seen": 668467200 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0005374787213920938, | |
| "loss": 2.8528, | |
| "theoretical_loss": 3.795359358557337, | |
| "tokens_seen": 671744000 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0005366512199735199, | |
| "loss": 2.7855, | |
| "theoretical_loss": 3.79347154602265, | |
| "tokens_seen": 675020800 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0005358237185549461, | |
| "loss": 2.8356, | |
| "theoretical_loss": 3.7915954272926955, | |
| "tokens_seen": 678297600 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0005349962171363722, | |
| "loss": 2.7915, | |
| "theoretical_loss": 3.789730873962557, | |
| "tokens_seen": 681574400 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0005341687157177983, | |
| "loss": 2.7291, | |
| "theoretical_loss": 3.787877759645963, | |
| "tokens_seen": 684851200 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0005333412142992244, | |
| "loss": 2.7625, | |
| "theoretical_loss": 3.7860359599340776, | |
| "tokens_seen": 688128000 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0005325137128806506, | |
| "loss": 2.8011, | |
| "theoretical_loss": 3.784205352355321, | |
| "tokens_seen": 691404800 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0005316862114620768, | |
| "loss": 2.83, | |
| "theoretical_loss": 3.782385816336189, | |
| "tokens_seen": 694681600 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0005308587100435029, | |
| "loss": 2.7875, | |
| "theoretical_loss": 3.7805772331630516, | |
| "tokens_seen": 697958400 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0005300312086249291, | |
| "loss": 2.7623, | |
| "theoretical_loss": 3.7787794859448898, | |
| "tokens_seen": 701235200 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0005292037072063552, | |
| "loss": 2.7454, | |
| "theoretical_loss": 3.7769924595769546, | |
| "tokens_seen": 704512000 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0005283762057877813, | |
| "loss": 2.7341, | |
| "theoretical_loss": 3.7752160407053115, | |
| "tokens_seen": 707788800 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0005275487043692075, | |
| "loss": 2.7184, | |
| "theoretical_loss": 3.7734501176922493, | |
| "tokens_seen": 711065600 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0005267212029506337, | |
| "loss": 2.7617, | |
| "theoretical_loss": 3.7716945805825337, | |
| "tokens_seen": 714342400 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0005258937015320598, | |
| "loss": 2.799, | |
| "theoretical_loss": 3.7699493210704667, | |
| "tokens_seen": 717619200 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0005250662001134859, | |
| "loss": 2.775, | |
| "theoretical_loss": 3.7682142324677455, | |
| "tokens_seen": 720896000 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.000524238698694912, | |
| "loss": 2.8333, | |
| "theoretical_loss": 3.7664892096720886, | |
| "tokens_seen": 724172800 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0005234111972763382, | |
| "loss": 2.7697, | |
| "theoretical_loss": 3.7647741491366067, | |
| "tokens_seen": 727449600 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0005225836958577644, | |
| "loss": 2.7844, | |
| "theoretical_loss": 3.7630689488399027, | |
| "tokens_seen": 730726400 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0005217561944391905, | |
| "loss": 2.7504, | |
| "theoretical_loss": 3.7613735082568764, | |
| "tokens_seen": 734003200 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0005209286930206166, | |
| "loss": 2.7869, | |
| "theoretical_loss": 3.759687728330217, | |
| "tokens_seen": 737280000 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0005201011916020427, | |
| "loss": 2.7799, | |
| "theoretical_loss": 3.75801151144256, | |
| "tokens_seen": 740556800 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0005192736901834689, | |
| "loss": 2.7636, | |
| "theoretical_loss": 3.756344761389295, | |
| "tokens_seen": 743833600 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.000518446188764895, | |
| "loss": 2.7358, | |
| "theoretical_loss": 3.754687383352003, | |
| "tokens_seen": 747110400 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0005176186873463211, | |
| "loss": 2.7389, | |
| "theoretical_loss": 3.7530392838725097, | |
| "tokens_seen": 750387200 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0005167911859277473, | |
| "loss": 2.7568, | |
| "theoretical_loss": 3.751400370827529, | |
| "tokens_seen": 753664000 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0005159636845091734, | |
| "loss": 2.7797, | |
| "theoretical_loss": 3.749770553403895, | |
| "tokens_seen": 756940800 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0005151361830905996, | |
| "loss": 2.7431, | |
| "theoretical_loss": 3.748149742074355, | |
| "tokens_seen": 760217600 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0005143086816720257, | |
| "loss": 2.7718, | |
| "theoretical_loss": 3.746537848573908, | |
| "tokens_seen": 763494400 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0005134811802534518, | |
| "loss": 2.7869, | |
| "theoretical_loss": 3.744934785876686, | |
| "tokens_seen": 766771200 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0005126536788348779, | |
| "loss": 2.7932, | |
| "theoretical_loss": 3.7433404681733475, | |
| "tokens_seen": 770048000 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0005118261774163041, | |
| "loss": 2.7999, | |
| "theoretical_loss": 3.7417548108489846, | |
| "tokens_seen": 773324800 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0005109986759977303, | |
| "loss": 2.7733, | |
| "theoretical_loss": 3.740177730461517, | |
| "tokens_seen": 776601600 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0005101711745791564, | |
| "loss": 2.822, | |
| "theoretical_loss": 3.73860914472057, | |
| "tokens_seen": 779878400 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0005093436731605825, | |
| "loss": 2.7869, | |
| "theoretical_loss": 3.7370489724668197, | |
| "tokens_seen": 783155200 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0005085161717420087, | |
| "loss": 2.7992, | |
| "theoretical_loss": 3.735497133651788, | |
| "tokens_seen": 786432000 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0005076886703234348, | |
| "loss": 2.7876, | |
| "theoretical_loss": 3.733953549318091, | |
| "tokens_seen": 789708800 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.000506861168904861, | |
| "loss": 2.8014, | |
| "theoretical_loss": 3.7324181415801094, | |
| "tokens_seen": 792985600 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0005060336674862872, | |
| "loss": 2.7963, | |
| "theoretical_loss": 3.7308908336050814, | |
| "tokens_seen": 796262400 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0005052061660677133, | |
| "loss": 2.7868, | |
| "theoretical_loss": 3.729371549594614, | |
| "tokens_seen": 799539200 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0005043786646491394, | |
| "loss": 2.8003, | |
| "theoretical_loss": 3.7278602147665776, | |
| "tokens_seen": 802816000 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0005035511632305655, | |
| "loss": 2.7947, | |
| "theoretical_loss": 3.726356755337407, | |
| "tokens_seen": 806092800 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0005027236618119917, | |
| "loss": 2.7845, | |
| "theoretical_loss": 3.724861098504767, | |
| "tokens_seen": 809369600 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0005018961603934179, | |
| "loss": 2.7496, | |
| "theoretical_loss": 3.7233731724305974, | |
| "tokens_seen": 812646400 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.000501068658974844, | |
| "loss": 2.7717, | |
| "theoretical_loss": 3.7218929062245105, | |
| "tokens_seen": 815923200 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "objective/train/avg_token_score": 0.009068925864994526, | |
| "objective/train/avg_weight": 0.9917506575584412, | |
| "objective/train/docs_used": 471128, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.670485496520996, | |
| "objective/train/std_weight": 0.05290398746728897, | |
| "objective/train/theoretical_loss": 3.7204202299275475, | |
| "objective/train/tokens_used": 839660000, | |
| "theoretical_loss": 3.7204202299275475, | |
| "tokens_seen": 819200000 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0005002411575562701, | |
| "loss": 2.7158, | |
| "theoretical_loss": 3.7204202299275475, | |
| "tokens_seen": 819200000 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0004994136561376962, | |
| "loss": 2.7397, | |
| "theoretical_loss": 3.7189550744962707, | |
| "tokens_seen": 822476800 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0004985861547191224, | |
| "loss": 2.7188, | |
| "theoretical_loss": 3.717497371787192, | |
| "tokens_seen": 825753600 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0004977586533005485, | |
| "loss": 2.7159, | |
| "theoretical_loss": 3.7160470545415274, | |
| "tokens_seen": 829030400 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0004969311518819746, | |
| "loss": 2.7166, | |
| "theoretical_loss": 3.714604056370267, | |
| "tokens_seen": 832307200 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0004961036504634008, | |
| "loss": 2.726, | |
| "theoretical_loss": 3.713168311739558, | |
| "tokens_seen": 835584000 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0004952761490448269, | |
| "loss": 2.7102, | |
| "theoretical_loss": 3.7117397559563843, | |
| "tokens_seen": 838860800 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0004944486476262531, | |
| "loss": 2.6939, | |
| "theoretical_loss": 3.710318325154545, | |
| "tokens_seen": 842137600 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0004936211462076792, | |
| "loss": 2.7416, | |
| "theoretical_loss": 3.7089039562809223, | |
| "tokens_seen": 845414400 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0004927936447891053, | |
| "loss": 2.7152, | |
| "theoretical_loss": 3.7074965870820193, | |
| "tokens_seen": 848691200 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0004919661433705314, | |
| "loss": 2.7146, | |
| "theoretical_loss": 3.7060961560907857, | |
| "tokens_seen": 851968000 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.0004911386419519576, | |
| "loss": 2.7725, | |
| "theoretical_loss": 3.7047026026137, | |
| "tokens_seen": 855244800 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.0004903111405333838, | |
| "loss": 2.7333, | |
| "theoretical_loss": 3.7033158667181154, | |
| "tokens_seen": 858521600 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.0004894836391148099, | |
| "loss": 2.7624, | |
| "theoretical_loss": 3.701935889219863, | |
| "tokens_seen": 861798400 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.000488656137696236, | |
| "loss": 2.7316, | |
| "theoretical_loss": 3.7005626116710966, | |
| "tokens_seen": 865075200 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00048782863627766217, | |
| "loss": 2.7403, | |
| "theoretical_loss": 3.69919597634839, | |
| "tokens_seen": 868352000 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.0004870011348590883, | |
| "loss": 2.719, | |
| "theoretical_loss": 3.6978359262410603, | |
| "tokens_seen": 871628800 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.0004861736334405145, | |
| "loss": 2.7432, | |
| "theoretical_loss": 3.6964824050397276, | |
| "tokens_seen": 874905600 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00048534613202194063, | |
| "loss": 2.6861, | |
| "theoretical_loss": 3.6951353571251015, | |
| "tokens_seen": 878182400 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00048451863060336675, | |
| "loss": 2.6866, | |
| "theoretical_loss": 3.693794727556988, | |
| "tokens_seen": 881459200 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.00048369112918479287, | |
| "loss": 2.6974, | |
| "theoretical_loss": 3.692460462063506, | |
| "tokens_seen": 884736000 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.000482863627766219, | |
| "loss": 2.7507, | |
| "theoretical_loss": 3.691132507030521, | |
| "tokens_seen": 888012800 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.00048203612634764516, | |
| "loss": 2.7619, | |
| "theoretical_loss": 3.6898108094912816, | |
| "tokens_seen": 891289600 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0004812086249290713, | |
| "loss": 2.719, | |
| "theoretical_loss": 3.6884953171162556, | |
| "tokens_seen": 894566400 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.00048038112351049745, | |
| "loss": 2.7252, | |
| "theoretical_loss": 3.6871859782031624, | |
| "tokens_seen": 897843200 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0004795536220919236, | |
| "loss": 2.7355, | |
| "theoretical_loss": 3.685882741667202, | |
| "tokens_seen": 901120000 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.00047872612067334974, | |
| "loss": 2.7378, | |
| "theoretical_loss": 3.684585557031461, | |
| "tokens_seen": 904396800 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.00047789861925477586, | |
| "loss": 2.7513, | |
| "theoretical_loss": 3.6832943744175126, | |
| "tokens_seen": 907673600 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.000477071117836202, | |
| "loss": 2.7248, | |
| "theoretical_loss": 3.682009144536188, | |
| "tokens_seen": 910950400 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0004762436164176281, | |
| "loss": 2.7513, | |
| "theoretical_loss": 3.680729818678526, | |
| "tokens_seen": 914227200 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0004754161149990543, | |
| "loss": 2.7298, | |
| "theoretical_loss": 3.6794563487068936, | |
| "tokens_seen": 917504000 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00047458861358048044, | |
| "loss": 2.7376, | |
| "theoretical_loss": 3.6781886870462692, | |
| "tokens_seen": 920780800 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00047376111216190656, | |
| "loss": 2.7544, | |
| "theoretical_loss": 3.676926786675698, | |
| "tokens_seen": 924057600 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0004729336107433327, | |
| "loss": 2.6955, | |
| "theoretical_loss": 3.6756706011198963, | |
| "tokens_seen": 927334400 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0004721061093247588, | |
| "loss": 2.6633, | |
| "theoretical_loss": 3.6744200844410217, | |
| "tokens_seen": 930611200 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00047127860790618497, | |
| "loss": 2.6639, | |
| "theoretical_loss": 3.6731751912305914, | |
| "tokens_seen": 933888000 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00047045110648761114, | |
| "loss": 2.6981, | |
| "theoretical_loss": 3.671935876601547, | |
| "tokens_seen": 937164800 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00046962360506903726, | |
| "loss": 2.6376, | |
| "theoretical_loss": 3.6707020961804715, | |
| "tokens_seen": 940441600 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00046879610365046343, | |
| "loss": 2.6905, | |
| "theoretical_loss": 3.6694738060999468, | |
| "tokens_seen": 943718400 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00046796860223188955, | |
| "loss": 2.7087, | |
| "theoretical_loss": 3.668250962991049, | |
| "tokens_seen": 946995200 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00046714110081331567, | |
| "loss": 2.7094, | |
| "theoretical_loss": 3.667033523975983, | |
| "tokens_seen": 950272000 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0004663135993947418, | |
| "loss": 2.6905, | |
| "theoretical_loss": 3.66582144666085, | |
| "tokens_seen": 953548800 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0004654860979761679, | |
| "loss": 2.6958, | |
| "theoretical_loss": 3.664614689128546, | |
| "tokens_seen": 956825600 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00046465859655759413, | |
| "loss": 2.6934, | |
| "theoretical_loss": 3.6634132099317886, | |
| "tokens_seen": 960102400 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00046383109513902025, | |
| "loss": 2.7015, | |
| "theoretical_loss": 3.662216968086267, | |
| "tokens_seen": 963379200 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00046300359372044637, | |
| "loss": 2.731, | |
| "theoretical_loss": 3.6610259230639217, | |
| "tokens_seen": 966656000 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0004621760923018725, | |
| "loss": 2.7641, | |
| "theoretical_loss": 3.659840034786333, | |
| "tokens_seen": 969932800 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00046134859088329866, | |
| "loss": 2.8244, | |
| "theoretical_loss": 3.6586592636182376, | |
| "tokens_seen": 973209600 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0004605210894647248, | |
| "loss": 2.8167, | |
| "theoretical_loss": 3.6574835703611566, | |
| "tokens_seen": 976486400 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00045969358804615095, | |
| "loss": 2.8036, | |
| "theoretical_loss": 3.6563129162471313, | |
| "tokens_seen": 979763200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.4286046663919377, | |
| "debugging/distinct-1-grams": 0.8147567798871364, | |
| "debugging/distinct-2-grams": 0.9823269374342457, | |
| "debugging/entropy-1-grams": 6.1671920556004824, | |
| "debugging/entropy-2-grams": 6.947028138756313, | |
| "debugging/length": 477.53333333333336, | |
| "debugging/num_segments": 15, | |
| "epoch": 0.35, | |
| "objective/train/avg_token_score": 0.020611366257071495, | |
| "objective/train/avg_weight": 0.9812498092651367, | |
| "objective/train/docs_used": 560408, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.92838716506958, | |
| "objective/train/std_weight": 0.0772768035531044, | |
| "objective/train/theoretical_loss": 3.6551472629325787, | |
| "objective/train/tokens_used": 1003500000, | |
| "theoretical_loss": 3.6551472629325787, | |
| "tokens_seen": 983040000 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00045886608662757707, | |
| "loss": 2.8015, | |
| "theoretical_loss": 3.6551472629325787, | |
| "tokens_seen": 983040000 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00045803858520900324, | |
| "loss": 2.8028, | |
| "theoretical_loss": 3.653986572492247, | |
| "tokens_seen": 986316800 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00045721108379042936, | |
| "loss": 2.771, | |
| "theoretical_loss": 3.65283080741328, | |
| "tokens_seen": 989593600 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0004563835823718555, | |
| "loss": 2.7847, | |
| "theoretical_loss": 3.6516799305893866, | |
| "tokens_seen": 992870400 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0004555560809532816, | |
| "loss": 2.7772, | |
| "theoretical_loss": 3.6505339053151076, | |
| "tokens_seen": 996147200 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0004547285795347078, | |
| "loss": 2.7861, | |
| "theoretical_loss": 3.649392695280186, | |
| "tokens_seen": 999424000 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.00045390107811613394, | |
| "loss": 2.7509, | |
| "theoretical_loss": 3.6482562645640337, | |
| "tokens_seen": 1002700800 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.00045307357669756006, | |
| "loss": 2.7548, | |
| "theoretical_loss": 3.6471245776302883, | |
| "tokens_seen": 1005977600 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0004522460752789862, | |
| "loss": 2.7905, | |
| "theoretical_loss": 3.6459975993214724, | |
| "tokens_seen": 1009254400 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0004514185738604123, | |
| "loss": 2.7515, | |
| "theoretical_loss": 3.6448752948537377, | |
| "tokens_seen": 1012531200 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.00045059107244183847, | |
| "loss": 2.7866, | |
| "theoretical_loss": 3.6437576298116996, | |
| "tokens_seen": 1015808000 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0004497635710232646, | |
| "loss": 2.7797, | |
| "theoretical_loss": 3.6426445701433607, | |
| "tokens_seen": 1019084800 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00044893606960469076, | |
| "loss": 2.8016, | |
| "theoretical_loss": 3.6415360821551226, | |
| "tokens_seen": 1022361600 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0004481085681861169, | |
| "loss": 2.7548, | |
| "theoretical_loss": 3.6404321325068754, | |
| "tokens_seen": 1025638400 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00044728106676754305, | |
| "loss": 2.7788, | |
| "theoretical_loss": 3.639332688207178, | |
| "tokens_seen": 1028915200 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00044645356534896917, | |
| "loss": 2.7593, | |
| "theoretical_loss": 3.6382377166085096, | |
| "tokens_seen": 1032192000 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0004456260639303953, | |
| "loss": 2.7166, | |
| "theoretical_loss": 3.6371471854026147, | |
| "tokens_seen": 1035468800 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0004447985625118214, | |
| "loss": 2.7249, | |
| "theoretical_loss": 3.6360610626159087, | |
| "tokens_seen": 1038745600 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00044397106109324763, | |
| "loss": 2.6919, | |
| "theoretical_loss": 3.634979316604973, | |
| "tokens_seen": 1042022400 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00044314355967467375, | |
| "loss": 2.7069, | |
| "theoretical_loss": 3.6339019160521198, | |
| "tokens_seen": 1045299200 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00044231605825609987, | |
| "loss": 2.6991, | |
| "theoretical_loss": 3.632828829961029, | |
| "tokens_seen": 1048576000 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.000441488556837526, | |
| "loss": 2.7159, | |
| "theoretical_loss": 3.631760027652461, | |
| "tokens_seen": 1051852800 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0004406610554189521, | |
| "loss": 2.7444, | |
| "theoretical_loss": 3.630695478760034, | |
| "tokens_seen": 1055129600 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0004398335540003783, | |
| "loss": 2.718, | |
| "theoretical_loss": 3.6296351532260767, | |
| "tokens_seen": 1058406400 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.00043900605258180445, | |
| "loss": 2.7124, | |
| "theoretical_loss": 3.6285790212975435, | |
| "tokens_seen": 1061683200 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.00043817855116323057, | |
| "loss": 2.6883, | |
| "theoretical_loss": 3.6275270535220008, | |
| "tokens_seen": 1064960000 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.00043735104974465674, | |
| "loss": 2.7127, | |
| "theoretical_loss": 3.626479220743673, | |
| "tokens_seen": 1068236800 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.00043652354832608286, | |
| "loss": 2.7173, | |
| "theoretical_loss": 3.6254354940995586, | |
| "tokens_seen": 1071513600 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.000435696046907509, | |
| "loss": 2.7093, | |
| "theoretical_loss": 3.624395845015602, | |
| "tokens_seen": 1074790400 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0004348685454889351, | |
| "loss": 2.6681, | |
| "theoretical_loss": 3.6233602452029348, | |
| "tokens_seen": 1078067200 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0004340410440703612, | |
| "loss": 2.6861, | |
| "theoretical_loss": 3.6223286666541683, | |
| "tokens_seen": 1081344000 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00043321354265178744, | |
| "loss": 2.7143, | |
| "theoretical_loss": 3.621301081639753, | |
| "tokens_seen": 1084620800 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00043238604123321356, | |
| "loss": 2.7513, | |
| "theoretical_loss": 3.6202774627043923, | |
| "tokens_seen": 1087897600 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0004315585398146397, | |
| "loss": 2.7077, | |
| "theoretical_loss": 3.619257782663513, | |
| "tokens_seen": 1091174400 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0004307310383960658, | |
| "loss": 2.7183, | |
| "theoretical_loss": 3.618242014599793, | |
| "tokens_seen": 1094451200 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0004299035369774919, | |
| "loss": 2.7047, | |
| "theoretical_loss": 3.617230131859743, | |
| "tokens_seen": 1097728000 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0004290760355589181, | |
| "loss": 2.6888, | |
| "theoretical_loss": 3.6162221080503416, | |
| "tokens_seen": 1101004800 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00042824853414034426, | |
| "loss": 2.6753, | |
| "theoretical_loss": 3.615217917035726, | |
| "tokens_seen": 1104281600 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0004274210327217704, | |
| "loss": 2.7061, | |
| "theoretical_loss": 3.614217532933929, | |
| "tokens_seen": 1107558400 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00042659353130319655, | |
| "loss": 2.694, | |
| "theoretical_loss": 3.6132209301136715, | |
| "tokens_seen": 1110835200 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00042576602988462267, | |
| "loss": 2.6781, | |
| "theoretical_loss": 3.612228083191205, | |
| "tokens_seen": 1114112000 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0004249385284660488, | |
| "loss": 2.737, | |
| "theoretical_loss": 3.611238967027199, | |
| "tokens_seen": 1117388800 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0004241110270474749, | |
| "loss": 2.7779, | |
| "theoretical_loss": 3.610253556723679, | |
| "tokens_seen": 1120665600 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00042328352562890113, | |
| "loss": 2.7414, | |
| "theoretical_loss": 3.609271827621014, | |
| "tokens_seen": 1123942400 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00042245602421032725, | |
| "loss": 2.7457, | |
| "theoretical_loss": 3.6082937552949463, | |
| "tokens_seen": 1127219200 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00042162852279175337, | |
| "loss": 2.7128, | |
| "theoretical_loss": 3.607319315553669, | |
| "tokens_seen": 1130496000 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0004208010213731795, | |
| "loss": 2.6753, | |
| "theoretical_loss": 3.6063484844349456, | |
| "tokens_seen": 1133772800 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0004199735199546056, | |
| "loss": 2.7007, | |
| "theoretical_loss": 3.605381238203279, | |
| "tokens_seen": 1137049600 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0004191460185360317, | |
| "loss": 2.7095, | |
| "theoretical_loss": 3.604417553347117, | |
| "tokens_seen": 1140326400 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0004183185171174579, | |
| "loss": 2.7005, | |
| "theoretical_loss": 3.603457406576106, | |
| "tokens_seen": 1143603200 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "objective/train/avg_token_score": 0.022877871990203857, | |
| "objective/train/avg_weight": 0.9792008399963379, | |
| "objective/train/docs_used": 649861, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.647454261779785, | |
| "objective/train/std_weight": 0.08679035305976868, | |
| "objective/train/theoretical_loss": 3.602500774818379, | |
| "objective/train/tokens_used": 1167340000, | |
| "theoretical_loss": 3.602500774818379, | |
| "tokens_seen": 1146880000 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00041750756572725553, | |
| "loss": 2.6892, | |
| "theoretical_loss": 3.602500774818379, | |
| "tokens_seen": 1146880000 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0004166800643086817, | |
| "loss": 2.7282, | |
| "theoretical_loss": 3.601547635217892, | |
| "tokens_seen": 1150156800 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0004158525628901078, | |
| "loss": 2.729, | |
| "theoretical_loss": 3.6005979651317976, | |
| "tokens_seen": 1153433600 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00041502506147153394, | |
| "loss": 2.7375, | |
| "theoretical_loss": 3.599651742127855, | |
| "tokens_seen": 1156710400 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00041419756005296005, | |
| "loss": 2.6894, | |
| "theoretical_loss": 3.5987089439818805, | |
| "tokens_seen": 1159987200 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00041337005863438617, | |
| "loss": 2.6691, | |
| "theoretical_loss": 3.5977695486752426, | |
| "tokens_seen": 1163264000 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0004125425572158124, | |
| "loss": 2.7023, | |
| "theoretical_loss": 3.596833534392379, | |
| "tokens_seen": 1166540800 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0004117150557972385, | |
| "loss": 2.72, | |
| "theoretical_loss": 3.595900879518368, | |
| "tokens_seen": 1169817600 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00041088755437866464, | |
| "loss": 2.7353, | |
| "theoretical_loss": 3.594971562636521, | |
| "tokens_seen": 1173094400 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00041006005296009075, | |
| "loss": 2.6902, | |
| "theoretical_loss": 3.5940455625260226, | |
| "tokens_seen": 1176371200 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0004092325515415169, | |
| "loss": 2.7018, | |
| "theoretical_loss": 3.5931228581595938, | |
| "tokens_seen": 1179648000 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00040840505012294305, | |
| "loss": 2.7002, | |
| "theoretical_loss": 3.5922034287011995, | |
| "tokens_seen": 1182924800 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0004075775487043692, | |
| "loss": 2.6656, | |
| "theoretical_loss": 3.5912872535037828, | |
| "tokens_seen": 1186201600 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0004067500472857954, | |
| "loss": 2.7475, | |
| "theoretical_loss": 3.590374312107035, | |
| "tokens_seen": 1189478400 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0004059225458672215, | |
| "loss": 2.733, | |
| "theoretical_loss": 3.5894645842351993, | |
| "tokens_seen": 1192755200 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0004050950444486476, | |
| "loss": 2.7186, | |
| "theoretical_loss": 3.588558049794902, | |
| "tokens_seen": 1196032000 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.00040426754303007374, | |
| "loss": 2.698, | |
| "theoretical_loss": 3.5876546888730187, | |
| "tokens_seen": 1199308800 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.00040344004161149986, | |
| "loss": 2.7175, | |
| "theoretical_loss": 3.5867544817345713, | |
| "tokens_seen": 1202585600 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0004026125401929261, | |
| "loss": 2.7441, | |
| "theoretical_loss": 3.585857408820652, | |
| "tokens_seen": 1205862400 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0004017850387743522, | |
| "loss": 2.751, | |
| "theoretical_loss": 3.58496345074638, | |
| "tokens_seen": 1209139200 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0004009575373557783, | |
| "loss": 2.7643, | |
| "theoretical_loss": 3.5840725882988873, | |
| "tokens_seen": 1212416000 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.00040013003593720444, | |
| "loss": 2.7308, | |
| "theoretical_loss": 3.5831848024353317, | |
| "tokens_seen": 1215692800 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00039930253451863056, | |
| "loss": 2.756, | |
| "theoretical_loss": 3.5823000742809374, | |
| "tokens_seen": 1218969600 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00039847503310005674, | |
| "loss": 2.7264, | |
| "theoretical_loss": 3.5814183851270673, | |
| "tokens_seen": 1222246400 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00039764753168148285, | |
| "loss": 2.7167, | |
| "theoretical_loss": 3.5805397164293167, | |
| "tokens_seen": 1225523200 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.000396820030262909, | |
| "loss": 2.6978, | |
| "theoretical_loss": 3.5796640498056407, | |
| "tokens_seen": 1228800000 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0003959925288443352, | |
| "loss": 2.7057, | |
| "theoretical_loss": 3.5787913670345013, | |
| "tokens_seen": 1232076800 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0003951650274257613, | |
| "loss": 2.6838, | |
| "theoretical_loss": 3.577921650053045, | |
| "tokens_seen": 1235353600 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00039433752600718743, | |
| "loss": 2.6807, | |
| "theoretical_loss": 3.577054880955303, | |
| "tokens_seen": 1238630400 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00039351002458861355, | |
| "loss": 2.6645, | |
| "theoretical_loss": 3.5761910419904193, | |
| "tokens_seen": 1241907200 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00039268252317003967, | |
| "loss": 2.7265, | |
| "theoretical_loss": 3.5753301155609014, | |
| "tokens_seen": 1245184000 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.0003918550217514659, | |
| "loss": 2.6666, | |
| "theoretical_loss": 3.574472084220896, | |
| "tokens_seen": 1248460800 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.000391027520332892, | |
| "loss": 2.656, | |
| "theoretical_loss": 3.5736169306744885, | |
| "tokens_seen": 1251737600 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00039020001891431813, | |
| "loss": 2.6265, | |
| "theoretical_loss": 3.572764637774024, | |
| "tokens_seen": 1255014400 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00038937251749574425, | |
| "loss": 2.6361, | |
| "theoretical_loss": 3.571915188518457, | |
| "tokens_seen": 1258291200 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00038854501607717037, | |
| "loss": 2.6557, | |
| "theoretical_loss": 3.571068566051716, | |
| "tokens_seen": 1261568000 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00038771751465859654, | |
| "loss": 2.6973, | |
| "theoretical_loss": 3.5702247536610976, | |
| "tokens_seen": 1264844800 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.0003868900132400227, | |
| "loss": 2.6764, | |
| "theoretical_loss": 3.5693837347756783, | |
| "tokens_seen": 1268121600 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00038606251182144883, | |
| "loss": 2.6491, | |
| "theoretical_loss": 3.5685454929647475, | |
| "tokens_seen": 1271398400 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.000385235010402875, | |
| "loss": 2.6612, | |
| "theoretical_loss": 3.5677100119362675, | |
| "tokens_seen": 1274675200 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0003844075089843011, | |
| "loss": 2.6389, | |
| "theoretical_loss": 3.566877275535345, | |
| "tokens_seen": 1277952000 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00038358000756572724, | |
| "loss": 2.6385, | |
| "theoretical_loss": 3.566047267742733, | |
| "tokens_seen": 1281228800 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00038275250614715336, | |
| "loss": 2.6157, | |
| "theoretical_loss": 3.5652199726733453, | |
| "tokens_seen": 1284505600 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0003819250047285795, | |
| "loss": 2.6116, | |
| "theoretical_loss": 3.564395374574796, | |
| "tokens_seen": 1287782400 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0003810975033100057, | |
| "loss": 2.6711, | |
| "theoretical_loss": 3.5635734578259557, | |
| "tokens_seen": 1291059200 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0003802700018914318, | |
| "loss": 2.6847, | |
| "theoretical_loss": 3.5627542069355282, | |
| "tokens_seen": 1294336000 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00037944250047285794, | |
| "loss": 2.6503, | |
| "theoretical_loss": 3.5619376065406474, | |
| "tokens_seen": 1297612800 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00037861499905428406, | |
| "loss": 2.665, | |
| "theoretical_loss": 3.5611236414054868, | |
| "tokens_seen": 1300889600 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00037778749763571023, | |
| "loss": 2.67, | |
| "theoretical_loss": 3.560312296419899, | |
| "tokens_seen": 1304166400 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00037695999621713635, | |
| "loss": 2.6669, | |
| "theoretical_loss": 3.55950355659806, | |
| "tokens_seen": 1307443200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.49020908264157476, | |
| "debugging/distinct-1-grams": 0.768901113497886, | |
| "debugging/distinct-2-grams": 0.9428782333551957, | |
| "debugging/entropy-1-grams": 6.085999550681761, | |
| "debugging/entropy-2-grams": 7.0033060167714964, | |
| "debugging/length": 490.2352941176471, | |
| "debugging/num_segments": 17, | |
| "epoch": 0.47, | |
| "objective/train/avg_token_score": 0.02056093141436577, | |
| "objective/train/avg_weight": 0.9812931418418884, | |
| "objective/train/docs_used": 741674, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.6537561416625977, | |
| "objective/train/std_weight": 0.0998743399977684, | |
| "objective/train/theoretical_loss": 3.558697407077142, | |
| "objective/train/tokens_used": 1331180000, | |
| "theoretical_loss": 3.558697407077142, | |
| "tokens_seen": 1310720000 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0003761324947985625, | |
| "loss": 2.6587, | |
| "theoretical_loss": 3.558697407077142, | |
| "tokens_seen": 1310720000 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00037530499337998864, | |
| "loss": 2.6727, | |
| "theoretical_loss": 3.5578938331159975, | |
| "tokens_seen": 1313996800 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0003744774919614148, | |
| "loss": 2.6464, | |
| "theoretical_loss": 3.557092820093863, | |
| "tokens_seen": 1317273600 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00037364999054284093, | |
| "loss": 2.6667, | |
| "theoretical_loss": 3.556294353509079, | |
| "tokens_seen": 1320550400 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00037282248912426705, | |
| "loss": 2.6582, | |
| "theoretical_loss": 3.555498418977828, | |
| "tokens_seen": 1323827200 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00037199498770569317, | |
| "loss": 2.6524, | |
| "theoretical_loss": 3.5547050022328874, | |
| "tokens_seen": 1327104000 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.0003711674862871194, | |
| "loss": 2.6501, | |
| "theoretical_loss": 3.553914089122399, | |
| "tokens_seen": 1330380800 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.0003703399848685455, | |
| "loss": 2.7001, | |
| "theoretical_loss": 3.553125665608655, | |
| "tokens_seen": 1333657600 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00036951248344997163, | |
| "loss": 2.6745, | |
| "theoretical_loss": 3.5523397177669005, | |
| "tokens_seen": 1336934400 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00036868498203139775, | |
| "loss": 2.6856, | |
| "theoretical_loss": 3.551556231784149, | |
| "tokens_seen": 1340211200 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00036785748061282387, | |
| "loss": 2.6404, | |
| "theoretical_loss": 3.5507751939580148, | |
| "tokens_seen": 1343488000 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00036702997919425004, | |
| "loss": 2.6402, | |
| "theoretical_loss": 3.5499965906955606, | |
| "tokens_seen": 1346764800 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00036620247777567616, | |
| "loss": 2.6887, | |
| "theoretical_loss": 3.549220408512161, | |
| "tokens_seen": 1350041600 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00036537497635710233, | |
| "loss": 2.665, | |
| "theoretical_loss": 3.5484466340303755, | |
| "tokens_seen": 1353318400 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00036454747493852845, | |
| "loss": 2.65, | |
| "theoretical_loss": 3.547675253978843, | |
| "tokens_seen": 1356595200 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.0003637199735199546, | |
| "loss": 2.6883, | |
| "theoretical_loss": 3.5469062551911854, | |
| "tokens_seen": 1359872000 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00036289247210138074, | |
| "loss": 2.6945, | |
| "theoretical_loss": 3.5461396246049244, | |
| "tokens_seen": 1363148800 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.0003620815207111783, | |
| "loss": 2.6619, | |
| "theoretical_loss": 3.545375349260419, | |
| "tokens_seen": 1366425600 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00036127056932097596, | |
| "loss": 2.6992, | |
| "theoretical_loss": 3.544613416299808, | |
| "tokens_seen": 1369702400 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00036044306790240213, | |
| "loss": 2.6756, | |
| "theoretical_loss": 3.5438538129659687, | |
| "tokens_seen": 1372979200 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00035961556648382825, | |
| "loss": 2.6955, | |
| "theoretical_loss": 3.5430965266014933, | |
| "tokens_seen": 1376256000 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.0003587880650652544, | |
| "loss": 2.6993, | |
| "theoretical_loss": 3.5423415446476705, | |
| "tokens_seen": 1379532800 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00035796056364668054, | |
| "loss": 2.6348, | |
| "theoretical_loss": 3.541588854643487, | |
| "tokens_seen": 1382809600 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.00035713306222810666, | |
| "loss": 2.6618, | |
| "theoretical_loss": 3.5408384442246343, | |
| "tokens_seen": 1386086400 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.0003563055608095328, | |
| "loss": 2.6165, | |
| "theoretical_loss": 3.540090301122535, | |
| "tokens_seen": 1389363200 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.000355478059390959, | |
| "loss": 2.6408, | |
| "theoretical_loss": 3.5393444131633762, | |
| "tokens_seen": 1392640000 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.0003546505579723851, | |
| "loss": 2.6885, | |
| "theoretical_loss": 3.5386007682671576, | |
| "tokens_seen": 1395916800 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.00035382305655381124, | |
| "loss": 2.6683, | |
| "theoretical_loss": 3.5378593544467494, | |
| "tokens_seen": 1399193600 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.00035299555513523736, | |
| "loss": 2.6501, | |
| "theoretical_loss": 3.537120159806962, | |
| "tokens_seen": 1402470400 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.0003521680537166635, | |
| "loss": 2.6392, | |
| "theoretical_loss": 3.5363831725436317, | |
| "tokens_seen": 1405747200 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.00035134055229808965, | |
| "loss": 2.6113, | |
| "theoretical_loss": 3.5356483809427104, | |
| "tokens_seen": 1409024000 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.0003505130508795158, | |
| "loss": 2.61, | |
| "theoretical_loss": 3.53491577337937, | |
| "tokens_seen": 1412300800 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.00034968554946094194, | |
| "loss": 2.6208, | |
| "theoretical_loss": 3.5341853383171205, | |
| "tokens_seen": 1415577600 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.00034885804804236806, | |
| "loss": 2.653, | |
| "theoretical_loss": 3.5334570643069325, | |
| "tokens_seen": 1418854400 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.00034803054662379423, | |
| "loss": 2.6326, | |
| "theoretical_loss": 3.5327309399863767, | |
| "tokens_seen": 1422131200 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.00034720304520522035, | |
| "loss": 2.6544, | |
| "theoretical_loss": 3.5320069540787706, | |
| "tokens_seen": 1425408000 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.0003463755437866465, | |
| "loss": 2.6169, | |
| "theoretical_loss": 3.531285095392335, | |
| "tokens_seen": 1428684800 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.00034554804236807264, | |
| "loss": 2.6349, | |
| "theoretical_loss": 3.5305653528193623, | |
| "tokens_seen": 1431961600 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.00034472054094949876, | |
| "loss": 2.6083, | |
| "theoretical_loss": 3.5298477153353955, | |
| "tokens_seen": 1435238400 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.00034389303953092493, | |
| "loss": 2.5753, | |
| "theoretical_loss": 3.5291321719984134, | |
| "tokens_seen": 1438515200 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "learning_rate": 0.00034306553811235105, | |
| "loss": 2.593, | |
| "theoretical_loss": 3.5284187119480315, | |
| "tokens_seen": 1441792000 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 0.00034223803669377717, | |
| "loss": 2.5751, | |
| "theoretical_loss": 3.527707324404704, | |
| "tokens_seen": 1445068800 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 0.00034141053527520334, | |
| "loss": 2.6007, | |
| "theoretical_loss": 3.5269979986689464, | |
| "tokens_seen": 1448345600 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 0.00034058303385662946, | |
| "loss": 2.6103, | |
| "theoretical_loss": 3.5262907241205568, | |
| "tokens_seen": 1451622400 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 0.0003397555324380556, | |
| "loss": 2.5882, | |
| "theoretical_loss": 3.5255854902178534, | |
| "tokens_seen": 1454899200 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 0.00033892803101948175, | |
| "loss": 2.5822, | |
| "theoretical_loss": 3.524882286496917, | |
| "tokens_seen": 1458176000 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 0.00033810052960090787, | |
| "loss": 2.6077, | |
| "theoretical_loss": 3.524181102570846, | |
| "tokens_seen": 1461452800 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 0.00033727302818233404, | |
| "loss": 2.6048, | |
| "theoretical_loss": 3.5234819281290166, | |
| "tokens_seen": 1464729600 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 0.00033644552676376016, | |
| "loss": 2.6283, | |
| "theoretical_loss": 3.5227847529363547, | |
| "tokens_seen": 1468006400 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.00033561802534518633, | |
| "loss": 2.6237, | |
| "theoretical_loss": 3.522089566832615, | |
| "tokens_seen": 1471283200 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "objective/train/avg_token_score": 0.03047107346355915, | |
| "objective/train/avg_weight": 0.9722864627838135, | |
| "objective/train/docs_used": 835915, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.72432804107666, | |
| "objective/train/std_weight": 0.1033627986907959, | |
| "objective/train/theoretical_loss": 3.521396359731668, | |
| "objective/train/tokens_used": 1495020000, | |
| "theoretical_loss": 3.521396359731668, | |
| "tokens_seen": 1474560000 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.00033479052392661245, | |
| "loss": 2.618, | |
| "theoretical_loss": 3.521396359731668, | |
| "tokens_seen": 1474560000 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.00033396302250803857, | |
| "loss": 2.6484, | |
| "theoretical_loss": 3.520705121620797, | |
| "tokens_seen": 1477836800 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.00033313552108946474, | |
| "loss": 2.6207, | |
| "theoretical_loss": 3.520015842560001, | |
| "tokens_seen": 1481113600 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.00033230801967089086, | |
| "loss": 2.6435, | |
| "theoretical_loss": 3.5193285126813105, | |
| "tokens_seen": 1484390400 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.000331480518252317, | |
| "loss": 2.6667, | |
| "theoretical_loss": 3.5186431221881023, | |
| "tokens_seen": 1487667200 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.00033065301683374315, | |
| "loss": 2.6736, | |
| "theoretical_loss": 3.517959661354433, | |
| "tokens_seen": 1490944000 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.00032982551541516926, | |
| "loss": 2.6664, | |
| "theoretical_loss": 3.517278120524372, | |
| "tokens_seen": 1494220800 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "learning_rate": 0.0003289980139965954, | |
| "loss": 2.6753, | |
| "theoretical_loss": 3.516598490111348, | |
| "tokens_seen": 1497497600 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 0.00032817051257802156, | |
| "loss": 2.6891, | |
| "theoretical_loss": 3.515920760597498, | |
| "tokens_seen": 1500774400 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 0.00032734301115944773, | |
| "loss": 2.6438, | |
| "theoretical_loss": 3.515244922533025, | |
| "tokens_seen": 1504051200 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 0.00032651550974087385, | |
| "loss": 2.6331, | |
| "theoretical_loss": 3.514570966535569, | |
| "tokens_seen": 1507328000 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 0.0003256880083223, | |
| "loss": 2.6759, | |
| "theoretical_loss": 3.513898883289575, | |
| "tokens_seen": 1510604800 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 0.00032486050690372614, | |
| "loss": 2.6527, | |
| "theoretical_loss": 3.513228663545678, | |
| "tokens_seen": 1513881600 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 0.00032403300548515226, | |
| "loss": 2.6711, | |
| "theoretical_loss": 3.5125602981200874, | |
| "tokens_seen": 1517158400 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 0.0003232055040665784, | |
| "loss": 2.6301, | |
| "theoretical_loss": 3.5118937778939836, | |
| "tokens_seen": 1520435200 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 0.00032237800264800455, | |
| "loss": 2.6331, | |
| "theoretical_loss": 3.5112290938129176, | |
| "tokens_seen": 1523712000 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.00032155050122943066, | |
| "loss": 2.6571, | |
| "theoretical_loss": 3.5105662368862234, | |
| "tokens_seen": 1526988800 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.0003207229998108568, | |
| "loss": 2.6291, | |
| "theoretical_loss": 3.5099051981864275, | |
| "tokens_seen": 1530265600 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.00031989549839228296, | |
| "loss": 2.6096, | |
| "theoretical_loss": 3.509245968848674, | |
| "tokens_seen": 1533542400 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.0003190679969737091, | |
| "loss": 2.6087, | |
| "theoretical_loss": 3.5085885400701535, | |
| "tokens_seen": 1536819200 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.00031824049555513525, | |
| "loss": 2.5919, | |
| "theoretical_loss": 3.507932903109534, | |
| "tokens_seen": 1540096000 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.00031741299413656136, | |
| "loss": 2.6015, | |
| "theoretical_loss": 3.5072790492864057, | |
| "tokens_seen": 1543372800 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.00031658549271798754, | |
| "loss": 2.6226, | |
| "theoretical_loss": 3.506626969980724, | |
| "tokens_seen": 1546649600 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.00031575799129941365, | |
| "loss": 2.6531, | |
| "theoretical_loss": 3.505976656632269, | |
| "tokens_seen": 1549926400 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "learning_rate": 0.0003149470399092113, | |
| "loss": 2.6297, | |
| "theoretical_loss": 3.505328100740097, | |
| "tokens_seen": 1553203200 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 0.0003141195384906374, | |
| "loss": 2.6249, | |
| "theoretical_loss": 3.5046812938620135, | |
| "tokens_seen": 1556480000 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 0.0003132920370720635, | |
| "loss": 2.6219, | |
| "theoretical_loss": 3.5040362276140393, | |
| "tokens_seen": 1559756800 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 0.0003124645356534897, | |
| "loss": 2.6021, | |
| "theoretical_loss": 3.5033928936698904, | |
| "tokens_seen": 1563033600 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 0.0003116370342349158, | |
| "loss": 2.6312, | |
| "theoretical_loss": 3.50275128376046, | |
| "tokens_seen": 1566310400 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 0.00031080953281634194, | |
| "loss": 2.5846, | |
| "theoretical_loss": 3.5021113896733076, | |
| "tokens_seen": 1569587200 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 0.0003099820313977681, | |
| "loss": 2.5956, | |
| "theoretical_loss": 3.501473203252153, | |
| "tokens_seen": 1572864000 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 0.0003091545299791942, | |
| "loss": 2.6509, | |
| "theoretical_loss": 3.500836716396377, | |
| "tokens_seen": 1576140800 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 0.0003083270285606204, | |
| "loss": 2.6404, | |
| "theoretical_loss": 3.5002019210605253, | |
| "tokens_seen": 1579417600 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.0003074995271420465, | |
| "loss": 2.6457, | |
| "theoretical_loss": 3.4995688092538213, | |
| "tokens_seen": 1582694400 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.0003066720257234727, | |
| "loss": 2.5931, | |
| "theoretical_loss": 3.49893737303968, | |
| "tokens_seen": 1585971200 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.0003058445243048988, | |
| "loss": 2.576, | |
| "theoretical_loss": 3.4983076045352317, | |
| "tokens_seen": 1589248000 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.0003050170228863249, | |
| "loss": 2.563, | |
| "theoretical_loss": 3.4976794959108464, | |
| "tokens_seen": 1592524800 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.0003041895214677511, | |
| "loss": 2.624, | |
| "theoretical_loss": 3.4970530393896673, | |
| "tokens_seen": 1595801600 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.0003033620200491772, | |
| "loss": 2.5936, | |
| "theoretical_loss": 3.4964282272471467, | |
| "tokens_seen": 1599078400 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.00030253451863060333, | |
| "loss": 2.5789, | |
| "theoretical_loss": 3.495805051810588, | |
| "tokens_seen": 1602355200 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.0003017070172120295, | |
| "loss": 2.6065, | |
| "theoretical_loss": 3.4951835054586926, | |
| "tokens_seen": 1605632000 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "learning_rate": 0.0003008795157934556, | |
| "loss": 2.5827, | |
| "theoretical_loss": 3.4945635806211115, | |
| "tokens_seen": 1608908800 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 0.00030005201437488174, | |
| "loss": 2.6266, | |
| "theoretical_loss": 3.493945269778002, | |
| "tokens_seen": 1612185600 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 0.0002992245129563079, | |
| "loss": 2.6316, | |
| "theoretical_loss": 3.4933285654595867, | |
| "tokens_seen": 1615462400 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 0.00029839701153773403, | |
| "loss": 2.621, | |
| "theoretical_loss": 3.4927134602457244, | |
| "tokens_seen": 1618739200 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 0.0002975695101191602, | |
| "loss": 2.6249, | |
| "theoretical_loss": 3.4920999467654754, | |
| "tokens_seen": 1622016000 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 0.0002967420087005864, | |
| "loss": 2.6071, | |
| "theoretical_loss": 3.49148801769668, | |
| "tokens_seen": 1625292800 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 0.00029593105731038396, | |
| "loss": 2.6416, | |
| "theoretical_loss": 3.4908776657655354, | |
| "tokens_seen": 1628569600 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 0.0002951035558918101, | |
| "loss": 2.5477, | |
| "theoretical_loss": 3.4902688837461824, | |
| "tokens_seen": 1631846400 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "learning_rate": 0.00029427605447323625, | |
| "loss": 2.5673, | |
| "theoretical_loss": 3.489661664460292, | |
| "tokens_seen": 1635123200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.4862537472725578, | |
| "debugging/distinct-1-grams": 0.7771784690107355, | |
| "debugging/distinct-2-grams": 0.9668669456652452, | |
| "debugging/entropy-1-grams": 6.202714981817353, | |
| "debugging/entropy-2-grams": 7.211883548596637, | |
| "debugging/length": 471.0, | |
| "debugging/num_segments": 21, | |
| "epoch": 0.59, | |
| "objective/train/avg_token_score": 0.013972360640764236, | |
| "objective/train/avg_weight": 0.9872912168502808, | |
| "objective/train/docs_used": 927402, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.8986141681671143, | |
| "objective/train/std_weight": 0.056864529848098755, | |
| "objective/train/theoretical_loss": 3.4890560007766607, | |
| "objective/train/tokens_used": 1658860000, | |
| "theoretical_loss": 3.4890560007766607, | |
| "tokens_seen": 1638400000 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.00029344855305466237, | |
| "loss": 2.5643, | |
| "theoretical_loss": 3.4890560007766607, | |
| "tokens_seen": 1638400000 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.0002926210516360885, | |
| "loss": 2.5784, | |
| "theoretical_loss": 3.4884518856108038, | |
| "tokens_seen": 1641676800 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.00029179355021751466, | |
| "loss": 2.586, | |
| "theoretical_loss": 3.4878493119245606, | |
| "tokens_seen": 1644953600 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.0002909660487989408, | |
| "loss": 2.5828, | |
| "theoretical_loss": 3.4872482727256964, | |
| "tokens_seen": 1648230400 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.00029013854738036695, | |
| "loss": 2.6001, | |
| "theoretical_loss": 3.4866487610675163, | |
| "tokens_seen": 1651507200 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.00028931104596179307, | |
| "loss": 2.5817, | |
| "theoretical_loss": 3.4860507700484757, | |
| "tokens_seen": 1654784000 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.00028848354454321924, | |
| "loss": 2.5392, | |
| "theoretical_loss": 3.4854542928117986, | |
| "tokens_seen": 1658060800 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.00028765604312464536, | |
| "loss": 2.5514, | |
| "theoretical_loss": 3.484859322545101, | |
| "tokens_seen": 1661337600 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 0.0002868285417060715, | |
| "loss": 2.5721, | |
| "theoretical_loss": 3.484265852480015, | |
| "tokens_seen": 1664614400 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "learning_rate": 0.00028600104028749765, | |
| "loss": 2.5913, | |
| "theoretical_loss": 3.48367387589182, | |
| "tokens_seen": 1667891200 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "learning_rate": 0.00028517353886892377, | |
| "loss": 2.6041, | |
| "theoretical_loss": 3.4830833860990733, | |
| "tokens_seen": 1671168000 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "learning_rate": 0.0002843460374503499, | |
| "loss": 2.6071, | |
| "theoretical_loss": 3.4824943764632503, | |
| "tokens_seen": 1674444800 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "learning_rate": 0.00028351853603177606, | |
| "loss": 2.6159, | |
| "theoretical_loss": 3.4819068403883855, | |
| "tokens_seen": 1677721600 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "learning_rate": 0.0002826910346132022, | |
| "loss": 2.6002, | |
| "theoretical_loss": 3.481320771320714, | |
| "tokens_seen": 1680998400 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "learning_rate": 0.0002818635331946283, | |
| "loss": 2.5879, | |
| "theoretical_loss": 3.4807361627483235, | |
| "tokens_seen": 1684275200 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "learning_rate": 0.00028103603177605447, | |
| "loss": 2.6451, | |
| "theoretical_loss": 3.480153008200804, | |
| "tokens_seen": 1687552000 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "learning_rate": 0.0002802085303574806, | |
| "loss": 2.6002, | |
| "theoretical_loss": 3.4795713012489045, | |
| "tokens_seen": 1690828800 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.00027938102893890676, | |
| "loss": 2.5693, | |
| "theoretical_loss": 3.478991035504191, | |
| "tokens_seen": 1694105600 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.0002785535275203329, | |
| "loss": 2.6427, | |
| "theoretical_loss": 3.478412204618709, | |
| "tokens_seen": 1697382400 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.00027772602610175905, | |
| "loss": 2.6074, | |
| "theoretical_loss": 3.4778348022846517, | |
| "tokens_seen": 1700659200 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.00027689852468318517, | |
| "loss": 2.639, | |
| "theoretical_loss": 3.477258822234026, | |
| "tokens_seen": 1703936000 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.00027607102326461134, | |
| "loss": 2.5851, | |
| "theoretical_loss": 3.4766842582383277, | |
| "tokens_seen": 1707212800 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.00027524352184603746, | |
| "loss": 2.5803, | |
| "theoretical_loss": 3.476111104108216, | |
| "tokens_seen": 1710489600 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.0002744160204274636, | |
| "loss": 2.5923, | |
| "theoretical_loss": 3.4755393536931942, | |
| "tokens_seen": 1713766400 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.00027358851900888975, | |
| "loss": 2.6014, | |
| "theoretical_loss": 3.4749690008812903, | |
| "tokens_seen": 1717043200 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 0.00027276101759031587, | |
| "loss": 2.5995, | |
| "theoretical_loss": 3.474400039598745, | |
| "tokens_seen": 1720320000 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.000271933516171742, | |
| "loss": 2.6277, | |
| "theoretical_loss": 3.473832463809698, | |
| "tokens_seen": 1723596800 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.00027110601475316816, | |
| "loss": 2.6256, | |
| "theoretical_loss": 3.473266267515883, | |
| "tokens_seen": 1726873600 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.0002702785133345943, | |
| "loss": 2.649, | |
| "theoretical_loss": 3.4727014447563196, | |
| "tokens_seen": 1730150400 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.0002694510119160204, | |
| "loss": 2.639, | |
| "theoretical_loss": 3.4721379896070133, | |
| "tokens_seen": 1733427200 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.00026862351049744657, | |
| "loss": 2.6488, | |
| "theoretical_loss": 3.4715758961806573, | |
| "tokens_seen": 1736704000 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.0002677960090788727, | |
| "loss": 2.6296, | |
| "theoretical_loss": 3.471015158626335, | |
| "tokens_seen": 1739980800 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.00026696850766029886, | |
| "loss": 2.6315, | |
| "theoretical_loss": 3.4704557711292265, | |
| "tokens_seen": 1743257600 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.000266141006241725, | |
| "loss": 2.6421, | |
| "theoretical_loss": 3.469897727910321, | |
| "tokens_seen": 1746534400 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "learning_rate": 0.00026531350482315115, | |
| "loss": 2.6608, | |
| "theoretical_loss": 3.4693410232261277, | |
| "tokens_seen": 1749811200 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 0.00026448600340457727, | |
| "loss": 2.6775, | |
| "theoretical_loss": 3.4687856513683917, | |
| "tokens_seen": 1753088000 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 0.0002636585019860034, | |
| "loss": 2.6537, | |
| "theoretical_loss": 3.468231606663813, | |
| "tokens_seen": 1756364800 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 0.00026283100056742956, | |
| "loss": 2.6439, | |
| "theoretical_loss": 3.4676788834737677, | |
| "tokens_seen": 1759641600 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 0.0002620034991488557, | |
| "loss": 2.6221, | |
| "theoretical_loss": 3.4671274761940314, | |
| "tokens_seen": 1762918400 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 0.0002611759977302818, | |
| "loss": 2.6041, | |
| "theoretical_loss": 3.466577379254506, | |
| "tokens_seen": 1766195200 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 0.00026034849631170797, | |
| "loss": 2.6109, | |
| "theoretical_loss": 3.4660285871189496, | |
| "tokens_seen": 1769472000 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 0.0002595209948931341, | |
| "loss": 2.6307, | |
| "theoretical_loss": 3.465481094284707, | |
| "tokens_seen": 1772748800 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 0.0002586934934745602, | |
| "loss": 2.6173, | |
| "theoretical_loss": 3.4649348952824477, | |
| "tokens_seen": 1776025600 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.0002578659920559864, | |
| "loss": 2.6012, | |
| "theoretical_loss": 3.4643899846758983, | |
| "tokens_seen": 1779302400 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.00025703849063741255, | |
| "loss": 2.5491, | |
| "theoretical_loss": 3.4638463570615854, | |
| "tokens_seen": 1782579200 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.00025621098921883867, | |
| "loss": 2.6049, | |
| "theoretical_loss": 3.4633040070685777, | |
| "tokens_seen": 1785856000 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.00025538348780026484, | |
| "loss": 2.6025, | |
| "theoretical_loss": 3.46276292935823, | |
| "tokens_seen": 1789132800 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.00025455598638169096, | |
| "loss": 2.5827, | |
| "theoretical_loss": 3.4622231186239296, | |
| "tokens_seen": 1792409600 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.0002537284849631171, | |
| "loss": 2.5837, | |
| "theoretical_loss": 3.461684569590848, | |
| "tokens_seen": 1795686400 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.0002529009835445432, | |
| "loss": 2.5667, | |
| "theoretical_loss": 3.4611472770156917, | |
| "tokens_seen": 1798963200 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "objective/train/avg_token_score": 0.013695425353944302, | |
| "objective/train/avg_weight": 0.9875440001487732, | |
| "objective/train/docs_used": 1018281, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.689345598220825, | |
| "objective/train/std_weight": 0.058852631598711014, | |
| "objective/train/theoretical_loss": 3.460611235686456, | |
| "objective/train/tokens_used": 1822700000, | |
| "theoretical_loss": 3.460611235686456, | |
| "tokens_seen": 1802240000 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.00025207348212596937, | |
| "loss": 2.5688, | |
| "theoretical_loss": 3.460611235686456, | |
| "tokens_seen": 1802240000 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "learning_rate": 0.0002512459807073955, | |
| "loss": 2.552, | |
| "theoretical_loss": 3.4600764404221835, | |
| "tokens_seen": 1805516800 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 0.0002504184792888216, | |
| "loss": 2.546, | |
| "theoretical_loss": 3.4595428860727218, | |
| "tokens_seen": 1808793600 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 0.0002495909778702478, | |
| "loss": 2.5316, | |
| "theoretical_loss": 3.4590105675184866, | |
| "tokens_seen": 1812070400 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 0.0002487634764516739, | |
| "loss": 2.5719, | |
| "theoretical_loss": 3.4584794796702223, | |
| "tokens_seen": 1815347200 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 0.00024793597503310007, | |
| "loss": 2.5774, | |
| "theoretical_loss": 3.457949617468771, | |
| "tokens_seen": 1818624000 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 0.0002471084736145262, | |
| "loss": 2.6226, | |
| "theoretical_loss": 3.4574209758848395, | |
| "tokens_seen": 1821900800 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 0.00024628097219595236, | |
| "loss": 2.5626, | |
| "theoretical_loss": 3.4568935499187696, | |
| "tokens_seen": 1825177600 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 0.0002454534707773785, | |
| "loss": 2.5795, | |
| "theoretical_loss": 3.456367334600311, | |
| "tokens_seen": 1828454400 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 0.00024462596935880465, | |
| "loss": 2.5753, | |
| "theoretical_loss": 3.455842324988396, | |
| "tokens_seen": 1831731200 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.00024379846794023077, | |
| "loss": 2.5741, | |
| "theoretical_loss": 3.455318516170914, | |
| "tokens_seen": 1835008000 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.00024297096652165688, | |
| "loss": 2.5662, | |
| "theoretical_loss": 3.454795903264496, | |
| "tokens_seen": 1838284800 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.00024214346510308306, | |
| "loss": 2.581, | |
| "theoretical_loss": 3.4542744814142896, | |
| "tokens_seen": 1841561600 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.00024131596368450918, | |
| "loss": 2.6053, | |
| "theoretical_loss": 3.4537542457937453, | |
| "tokens_seen": 1844838400 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.0002404884622659353, | |
| "loss": 2.6309, | |
| "theoretical_loss": 3.4532351916044, | |
| "tokens_seen": 1848115200 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.00023966096084736147, | |
| "loss": 2.6354, | |
| "theoretical_loss": 3.4527173140756666, | |
| "tokens_seen": 1851392000 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.00023883345942878758, | |
| "loss": 2.6457, | |
| "theoretical_loss": 3.4522006084646204, | |
| "tokens_seen": 1854668800 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.00023800595801021373, | |
| "loss": 2.6497, | |
| "theoretical_loss": 3.4516850700557917, | |
| "tokens_seen": 1857945600 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "learning_rate": 0.00023717845659163985, | |
| "loss": 2.6881, | |
| "theoretical_loss": 3.4511706941609566, | |
| "tokens_seen": 1861222400 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 0.00023635095517306602, | |
| "loss": 2.6235, | |
| "theoretical_loss": 3.450657476118936, | |
| "tokens_seen": 1864499200 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 0.00023552345375449214, | |
| "loss": 2.5967, | |
| "theoretical_loss": 3.450145411295389, | |
| "tokens_seen": 1867776000 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 0.00023469595233591828, | |
| "loss": 2.5875, | |
| "theoretical_loss": 3.4496344950826114, | |
| "tokens_seen": 1871052800 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 0.00023386845091734443, | |
| "loss": 2.5806, | |
| "theoretical_loss": 3.449124722899339, | |
| "tokens_seen": 1874329600 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 0.00023304094949877057, | |
| "loss": 2.563, | |
| "theoretical_loss": 3.4486160901905487, | |
| "tokens_seen": 1877606400 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 0.0002322134480801967, | |
| "loss": 2.5564, | |
| "theoretical_loss": 3.448108592427261, | |
| "tokens_seen": 1880883200 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 0.00023138594666162287, | |
| "loss": 2.5393, | |
| "theoretical_loss": 3.4476022251063485, | |
| "tokens_seen": 1884160000 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 0.00023055844524304898, | |
| "loss": 2.5329, | |
| "theoretical_loss": 3.447096983750342, | |
| "tokens_seen": 1887436800 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.0002297309438244751, | |
| "loss": 2.5374, | |
| "theoretical_loss": 3.4465928639072416, | |
| "tokens_seen": 1890713600 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.00022890344240590127, | |
| "loss": 2.5575, | |
| "theoretical_loss": 3.446089861150326, | |
| "tokens_seen": 1893990400 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.0002280759409873274, | |
| "loss": 2.59, | |
| "theoretical_loss": 3.445587971077968, | |
| "tokens_seen": 1897267200 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.00022724843956875354, | |
| "loss": 2.5497, | |
| "theoretical_loss": 3.4450871893134467, | |
| "tokens_seen": 1900544000 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.0002264209381501797, | |
| "loss": 2.5495, | |
| "theoretical_loss": 3.4445875115047655, | |
| "tokens_seen": 1903820800 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.00022559343673160583, | |
| "loss": 2.5219, | |
| "theoretical_loss": 3.4440889333244695, | |
| "tokens_seen": 1907097600 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.00022476593531303195, | |
| "loss": 2.4975, | |
| "theoretical_loss": 3.443591450469466, | |
| "tokens_seen": 1910374400 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.00022393843389445812, | |
| "loss": 2.543, | |
| "theoretical_loss": 3.4430950586608438, | |
| "tokens_seen": 1913651200 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "learning_rate": 0.00022311093247588424, | |
| "loss": 2.5449, | |
| "theoretical_loss": 3.4425997536437, | |
| "tokens_seen": 1916928000 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 0.00022228343105731038, | |
| "loss": 2.5648, | |
| "theoretical_loss": 3.4421055311869604, | |
| "tokens_seen": 1920204800 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 0.00022145592963873653, | |
| "loss": 2.5935, | |
| "theoretical_loss": 3.441612387083208, | |
| "tokens_seen": 1923481600 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 0.00022062842822016267, | |
| "loss": 2.62, | |
| "theoretical_loss": 3.44112031714851, | |
| "tokens_seen": 1926758400 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 0.0002198009268015888, | |
| "loss": 2.6544, | |
| "theoretical_loss": 3.4406293172222466, | |
| "tokens_seen": 1930035200 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 0.0002189734253830149, | |
| "loss": 2.6152, | |
| "theoretical_loss": 3.4401393831669447, | |
| "tokens_seen": 1933312000 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 0.00021814592396444108, | |
| "loss": 2.5888, | |
| "theoretical_loss": 3.4396505108681046, | |
| "tokens_seen": 1936588800 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 0.00021731842254586723, | |
| "loss": 2.5924, | |
| "theoretical_loss": 3.4391626962340394, | |
| "tokens_seen": 1939865600 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 0.00021649092112729335, | |
| "loss": 2.6304, | |
| "theoretical_loss": 3.4386759351957066, | |
| "tokens_seen": 1943142400 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.00021566341970871952, | |
| "loss": 2.5666, | |
| "theoretical_loss": 3.438190223706548, | |
| "tokens_seen": 1946419200 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.00021483591829014564, | |
| "loss": 2.5957, | |
| "theoretical_loss": 3.437705557742323, | |
| "tokens_seen": 1949696000 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.00021402496689994325, | |
| "loss": 2.5613, | |
| "theoretical_loss": 3.4372219333009535, | |
| "tokens_seen": 1952972800 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.00021319746548136942, | |
| "loss": 2.6084, | |
| "theoretical_loss": 3.436739346402362, | |
| "tokens_seen": 1956249600 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.00021236996406279554, | |
| "loss": 2.6285, | |
| "theoretical_loss": 3.4362577930883154, | |
| "tokens_seen": 1959526400 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.00021154246264422165, | |
| "loss": 2.6072, | |
| "theoretical_loss": 3.435777269422266, | |
| "tokens_seen": 1962803200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.4796244782337672, | |
| "debugging/distinct-1-grams": 0.7874466763773558, | |
| "debugging/distinct-2-grams": 0.9537592626603371, | |
| "debugging/entropy-1-grams": 6.225892551400493, | |
| "debugging/entropy-2-grams": 7.143796200176747, | |
| "debugging/length": 450.05, | |
| "debugging/num_segments": 20, | |
| "epoch": 0.7, | |
| "objective/train/avg_token_score": 0.028914527967572212, | |
| "objective/train/avg_weight": 0.9737117886543274, | |
| "objective/train/docs_used": 1112341, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.6371700763702393, | |
| "objective/train/std_weight": 0.08570484071969986, | |
| "objective/train/theoretical_loss": 3.4352977714892017, | |
| "objective/train/tokens_used": 1986540000, | |
| "theoretical_loss": 3.4352977714892017, | |
| "tokens_seen": 1966080000 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.00021071496122564783, | |
| "loss": 2.6052, | |
| "theoretical_loss": 3.4352977714892017, | |
| "tokens_seen": 1966080000 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.00020988745980707394, | |
| "loss": 2.5535, | |
| "theoretical_loss": 3.434819295395487, | |
| "tokens_seen": 1969356800 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "learning_rate": 0.0002090599583885001, | |
| "loss": 2.5805, | |
| "theoretical_loss": 3.4343418372687156, | |
| "tokens_seen": 1972633600 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "learning_rate": 0.00020823245696992624, | |
| "loss": 2.5809, | |
| "theoretical_loss": 3.433865393257557, | |
| "tokens_seen": 1975910400 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "learning_rate": 0.00020740495555135238, | |
| "loss": 2.5361, | |
| "theoretical_loss": 3.433389959531607, | |
| "tokens_seen": 1979187200 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "learning_rate": 0.0002065774541327785, | |
| "loss": 2.5526, | |
| "theoretical_loss": 3.4329155322812417, | |
| "tokens_seen": 1982464000 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "learning_rate": 0.00020574995271420467, | |
| "loss": 2.5466, | |
| "theoretical_loss": 3.432442107717467, | |
| "tokens_seen": 1985740800 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "learning_rate": 0.0002049224512956308, | |
| "loss": 2.5827, | |
| "theoretical_loss": 3.4319696820717778, | |
| "tokens_seen": 1989017600 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "learning_rate": 0.00020409494987705694, | |
| "loss": 2.617, | |
| "theoretical_loss": 3.4314982515960093, | |
| "tokens_seen": 1992294400 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "learning_rate": 0.00020326744845848308, | |
| "loss": 2.6301, | |
| "theoretical_loss": 3.4310278125621965, | |
| "tokens_seen": 1995571200 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "learning_rate": 0.00020243994703990923, | |
| "loss": 2.629, | |
| "theoretical_loss": 3.430558361262433, | |
| "tokens_seen": 1998848000 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.00020161244562133534, | |
| "loss": 2.5992, | |
| "theoretical_loss": 3.4300898940087254, | |
| "tokens_seen": 2002124800 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.00020078494420276146, | |
| "loss": 2.6197, | |
| "theoretical_loss": 3.4296224071328623, | |
| "tokens_seen": 2005401600 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.00019995744278418763, | |
| "loss": 2.5997, | |
| "theoretical_loss": 3.429155896986268, | |
| "tokens_seen": 2008678400 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.00019912994136561375, | |
| "loss": 2.57, | |
| "theoretical_loss": 3.4286903599398704, | |
| "tokens_seen": 2011955200 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.0001983024399470399, | |
| "loss": 2.6082, | |
| "theoretical_loss": 3.428225792383964, | |
| "tokens_seen": 2015232000 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.00019747493852846607, | |
| "loss": 2.6016, | |
| "theoretical_loss": 3.4277621907280746, | |
| "tokens_seen": 2018508800 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.0001966474371098922, | |
| "loss": 2.5576, | |
| "theoretical_loss": 3.427299551400826, | |
| "tokens_seen": 2021785600 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.0001958199356913183, | |
| "loss": 2.5728, | |
| "theoretical_loss": 3.4268378708498086, | |
| "tokens_seen": 2025062400 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "learning_rate": 0.00019499243427274448, | |
| "loss": 2.5568, | |
| "theoretical_loss": 3.426377145541448, | |
| "tokens_seen": 2028339200 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.0001941649328541706, | |
| "loss": 2.5375, | |
| "theoretical_loss": 3.425917371960872, | |
| "tokens_seen": 2031616000 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.00019333743143559674, | |
| "loss": 2.5052, | |
| "theoretical_loss": 3.425458546611787, | |
| "tokens_seen": 2034892800 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.0001925099300170229, | |
| "loss": 2.5051, | |
| "theoretical_loss": 3.425000666016346, | |
| "tokens_seen": 2038169600 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.00019168242859844903, | |
| "loss": 2.5145, | |
| "theoretical_loss": 3.4245437267150214, | |
| "tokens_seen": 2041446400 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.00019085492717987515, | |
| "loss": 2.5595, | |
| "theoretical_loss": 3.4240877252664834, | |
| "tokens_seen": 2044723200 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.00019002742576130133, | |
| "loss": 2.5328, | |
| "theoretical_loss": 3.4236326582474716, | |
| "tokens_seen": 2048000000 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.00018919992434272744, | |
| "loss": 2.525, | |
| "theoretical_loss": 3.423178522252672, | |
| "tokens_seen": 2051276800 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.00018837242292415356, | |
| "loss": 2.5283, | |
| "theoretical_loss": 3.4227253138945946, | |
| "tokens_seen": 2054553600 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 0.00018754492150557973, | |
| "loss": 2.532, | |
| "theoretical_loss": 3.4222730298034545, | |
| "tokens_seen": 2057830400 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 0.00018671742008700588, | |
| "loss": 2.5283, | |
| "theoretical_loss": 3.421821666627046, | |
| "tokens_seen": 2061107200 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 0.000185889918668432, | |
| "loss": 2.5447, | |
| "theoretical_loss": 3.4213712210306286, | |
| "tokens_seen": 2064384000 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 0.00018506241724985812, | |
| "loss": 2.5558, | |
| "theoretical_loss": 3.4209216896968044, | |
| "tokens_seen": 2067660800 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 0.0001842349158312843, | |
| "loss": 2.5039, | |
| "theoretical_loss": 3.4204730693254035, | |
| "tokens_seen": 2070937600 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 0.0001834074144127104, | |
| "loss": 2.5758, | |
| "theoretical_loss": 3.4200253566333645, | |
| "tokens_seen": 2074214400 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 0.00018257991299413655, | |
| "loss": 2.6052, | |
| "theoretical_loss": 3.4195785483546226, | |
| "tokens_seen": 2077491200 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 0.0001817524115755627, | |
| "loss": 2.6032, | |
| "theoretical_loss": 3.419132641239991, | |
| "tokens_seen": 2080768000 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "learning_rate": 0.00018092491015698884, | |
| "loss": 2.5538, | |
| "theoretical_loss": 3.4186876320570505, | |
| "tokens_seen": 2084044800 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.00018009740873841496, | |
| "loss": 2.581, | |
| "theoretical_loss": 3.418243517590036, | |
| "tokens_seen": 2087321600 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.00017926990731984113, | |
| "loss": 2.557, | |
| "theoretical_loss": 3.4178002946397217, | |
| "tokens_seen": 2090598400 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.00017844240590126725, | |
| "loss": 2.5924, | |
| "theoretical_loss": 3.4173579600233155, | |
| "tokens_seen": 2093875200 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.0001776149044826934, | |
| "loss": 2.5692, | |
| "theoretical_loss": 3.416916510574345, | |
| "tokens_seen": 2097152000 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.00017678740306411954, | |
| "loss": 2.5512, | |
| "theoretical_loss": 3.4164759431425495, | |
| "tokens_seen": 2100428800 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.0001759599016455457, | |
| "loss": 2.5781, | |
| "theoretical_loss": 3.4160362545937732, | |
| "tokens_seen": 2103705600 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.0001751324002269718, | |
| "loss": 2.5732, | |
| "theoretical_loss": 3.4155974418098562, | |
| "tokens_seen": 2106982400 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.00017430489880839795, | |
| "loss": 2.5704, | |
| "theoretical_loss": 3.415159501688528, | |
| "tokens_seen": 2110259200 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 0.0001734773973898241, | |
| "loss": 2.6042, | |
| "theoretical_loss": 3.4147224311433053, | |
| "tokens_seen": 2113536000 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "learning_rate": 0.00017264989597125022, | |
| "loss": 2.612, | |
| "theoretical_loss": 3.414286227103381, | |
| "tokens_seen": 2116812800 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "learning_rate": 0.00017182239455267636, | |
| "loss": 2.5871, | |
| "theoretical_loss": 3.413850886513529, | |
| "tokens_seen": 2120089600 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "learning_rate": 0.00017099489313410253, | |
| "loss": 2.5607, | |
| "theoretical_loss": 3.413416406333992, | |
| "tokens_seen": 2123366400 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "learning_rate": 0.00017016739171552865, | |
| "loss": 2.5734, | |
| "theoretical_loss": 3.412982783540387, | |
| "tokens_seen": 2126643200 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "objective/train/avg_token_score": 0.011589955538511276, | |
| "objective/train/avg_weight": 0.9894662499427795, | |
| "objective/train/docs_used": 1206282, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.4739794731140137, | |
| "objective/train/std_weight": 0.045400042086839676, | |
| "objective/train/theoretical_loss": 3.4125500151236, | |
| "objective/train/tokens_used": 2150380000, | |
| "theoretical_loss": 3.4125500151236, | |
| "tokens_seen": 2129920000 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "learning_rate": 0.0001693398902969548, | |
| "loss": 2.5238, | |
| "theoretical_loss": 3.4125500151236, | |
| "tokens_seen": 2129920000 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "learning_rate": 0.00016851238887838094, | |
| "loss": 2.5471, | |
| "theoretical_loss": 3.412118098089686, | |
| "tokens_seen": 2133196800 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "learning_rate": 0.00016768488745980706, | |
| "loss": 2.5653, | |
| "theoretical_loss": 3.411687029459771, | |
| "tokens_seen": 2136473600 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "learning_rate": 0.0001668573860412332, | |
| "loss": 2.5337, | |
| "theoretical_loss": 3.411256806269951, | |
| "tokens_seen": 2139750400 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.00016602988462265935, | |
| "loss": 2.5359, | |
| "theoretical_loss": 3.4108274255711954, | |
| "tokens_seen": 2143027200 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.0001652023832040855, | |
| "loss": 2.556, | |
| "theoretical_loss": 3.4103988844292497, | |
| "tokens_seen": 2146304000 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.00016437488178551164, | |
| "loss": 2.5809, | |
| "theoretical_loss": 3.4099711799245385, | |
| "tokens_seen": 2149580800 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.00016354738036693776, | |
| "loss": 2.5927, | |
| "theoretical_loss": 3.4095443091520705, | |
| "tokens_seen": 2152857600 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.0001627198789483639, | |
| "loss": 2.5588, | |
| "theoretical_loss": 3.4091182692213415, | |
| "tokens_seen": 2156134400 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.00016189237752979005, | |
| "loss": 2.5175, | |
| "theoretical_loss": 3.4086930572562446, | |
| "tokens_seen": 2159411200 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.0001610648761112162, | |
| "loss": 2.5451, | |
| "theoretical_loss": 3.4082686703949716, | |
| "tokens_seen": 2162688000 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.00016023737469264234, | |
| "loss": 2.5787, | |
| "theoretical_loss": 3.4078451057899244, | |
| "tokens_seen": 2165964800 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 0.0001594098732740685, | |
| "loss": 2.6015, | |
| "theoretical_loss": 3.4074223606076206, | |
| "tokens_seen": 2169241600 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "learning_rate": 0.0001585823718554946, | |
| "loss": 2.5976, | |
| "theoretical_loss": 3.4070004320286036, | |
| "tokens_seen": 2172518400 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "learning_rate": 0.00015775487043692075, | |
| "loss": 2.5559, | |
| "theoretical_loss": 3.4065793172473526, | |
| "tokens_seen": 2175795200 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "learning_rate": 0.00015692736901834687, | |
| "loss": 2.5379, | |
| "theoretical_loss": 3.40615901347219, | |
| "tokens_seen": 2179072000 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "learning_rate": 0.00015609986759977301, | |
| "loss": 2.5734, | |
| "theoretical_loss": 3.405739517925197, | |
| "tokens_seen": 2182348800 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "learning_rate": 0.00015527236618119916, | |
| "loss": 2.5733, | |
| "theoretical_loss": 3.4053208278421208, | |
| "tokens_seen": 2185625600 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "learning_rate": 0.0001544448647626253, | |
| "loss": 2.5461, | |
| "theoretical_loss": 3.4049029404722897, | |
| "tokens_seen": 2188902400 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "learning_rate": 0.00015361736334405145, | |
| "loss": 2.5263, | |
| "theoretical_loss": 3.4044858530785254, | |
| "tokens_seen": 2192179200 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "learning_rate": 0.0001527898619254776, | |
| "loss": 2.5814, | |
| "theoretical_loss": 3.4040695629370563, | |
| "tokens_seen": 2195456000 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.00015196236050690371, | |
| "loss": 2.5766, | |
| "theoretical_loss": 3.403654067337433, | |
| "tokens_seen": 2198732800 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.00015113485908832986, | |
| "loss": 2.5392, | |
| "theoretical_loss": 3.4032393635824416, | |
| "tokens_seen": 2202009600 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.000150307357669756, | |
| "loss": 2.5908, | |
| "theoretical_loss": 3.4028254489880227, | |
| "tokens_seen": 2205286400 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.00014947985625118215, | |
| "loss": 2.7262, | |
| "theoretical_loss": 3.4024123208831836, | |
| "tokens_seen": 2208563200 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.0001486523548326083, | |
| "loss": 2.782, | |
| "theoretical_loss": 3.4019999766099187, | |
| "tokens_seen": 2211840000 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.00014782485341403441, | |
| "loss": 2.8268, | |
| "theoretical_loss": 3.4015884135231262, | |
| "tokens_seen": 2215116800 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.00014699735199546056, | |
| "loss": 2.7471, | |
| "theoretical_loss": 3.401177628990526, | |
| "tokens_seen": 2218393600 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.0001461698505768867, | |
| "loss": 2.7503, | |
| "theoretical_loss": 3.400767620392579, | |
| "tokens_seen": 2221670400 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 0.00014534234915831282, | |
| "loss": 2.7165, | |
| "theoretical_loss": 3.4003583851224075, | |
| "tokens_seen": 2224947200 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 0.00014451484773973897, | |
| "loss": 2.7019, | |
| "theoretical_loss": 3.3999499205857138, | |
| "tokens_seen": 2228224000 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 0.00014368734632116514, | |
| "loss": 2.6365, | |
| "theoretical_loss": 3.3995422242007036, | |
| "tokens_seen": 2231500800 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 0.00014285984490259126, | |
| "loss": 2.6595, | |
| "theoretical_loss": 3.399135293398004, | |
| "tokens_seen": 2234777600 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 0.0001420323434840174, | |
| "loss": 2.6163, | |
| "theoretical_loss": 3.39872912562059, | |
| "tokens_seen": 2238054400 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 0.000141221392093815, | |
| "loss": 2.6067, | |
| "theoretical_loss": 3.3983237183237023, | |
| "tokens_seen": 2241331200 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 0.00014039389067524116, | |
| "loss": 2.5757, | |
| "theoretical_loss": 3.397919068974775, | |
| "tokens_seen": 2244608000 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 0.0001395663892566673, | |
| "loss": 2.5545, | |
| "theoretical_loss": 3.3975151750533574, | |
| "tokens_seen": 2247884800 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "learning_rate": 0.00013873888783809342, | |
| "loss": 2.526, | |
| "theoretical_loss": 3.3971120340510375, | |
| "tokens_seen": 2251161600 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 0.00013791138641951957, | |
| "loss": 2.5446, | |
| "theoretical_loss": 3.39670964347137, | |
| "tokens_seen": 2254438400 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 0.0001370838850009457, | |
| "loss": 2.5508, | |
| "theoretical_loss": 3.396308000829799, | |
| "tokens_seen": 2257715200 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 0.00013625638358237186, | |
| "loss": 2.5261, | |
| "theoretical_loss": 3.395907103653585, | |
| "tokens_seen": 2260992000 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 0.000135428882163798, | |
| "loss": 2.5239, | |
| "theoretical_loss": 3.395506949481734, | |
| "tokens_seen": 2264268800 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 0.00013460138074522415, | |
| "loss": 2.533, | |
| "theoretical_loss": 3.3951075358649208, | |
| "tokens_seen": 2267545600 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 0.00013377387932665027, | |
| "loss": 2.509, | |
| "theoretical_loss": 3.3947088603654203, | |
| "tokens_seen": 2270822400 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 0.0001329463779080764, | |
| "loss": 2.5623, | |
| "theoretical_loss": 3.394310920557034, | |
| "tokens_seen": 2274099200 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 0.00013213542651787405, | |
| "loss": 2.5419, | |
| "theoretical_loss": 3.393913714025021, | |
| "tokens_seen": 2277376000 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "learning_rate": 0.00013130792509930016, | |
| "loss": 2.5505, | |
| "theoretical_loss": 3.393517238366023, | |
| "tokens_seen": 2280652800 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 0.0001304804236807263, | |
| "loss": 2.5774, | |
| "theoretical_loss": 3.3931214911880008, | |
| "tokens_seen": 2283929600 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 0.00012965292226215246, | |
| "loss": 2.5607, | |
| "theoretical_loss": 3.3927264701101594, | |
| "tokens_seen": 2287206400 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 0.00012882542084357857, | |
| "loss": 2.5907, | |
| "theoretical_loss": 3.392332172762883, | |
| "tokens_seen": 2290483200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.515096219338814, | |
| "debugging/distinct-1-grams": 0.7923118197529954, | |
| "debugging/distinct-2-grams": 0.9542743672238616, | |
| "debugging/entropy-1-grams": 6.147062465574793, | |
| "debugging/entropy-2-grams": 7.107820582543805, | |
| "debugging/length": 504.1666666666667, | |
| "debugging/num_segments": 18, | |
| "epoch": 0.82, | |
| "objective/train/avg_token_score": 0.012544393539428711, | |
| "objective/train/avg_weight": 0.9885972142219543, | |
| "objective/train/docs_used": 1292636, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.5001182556152344, | |
| "objective/train/std_weight": 0.04136689379811287, | |
| "objective/train/theoretical_loss": 3.3919385967876625, | |
| "objective/train/tokens_used": 2314220000, | |
| "theoretical_loss": 3.3919385967876625, | |
| "tokens_seen": 2293760000 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 0.00012799791942500472, | |
| "loss": 2.568, | |
| "theoretical_loss": 3.3919385967876625, | |
| "tokens_seen": 2293760000 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 0.00012717041800643086, | |
| "loss": 2.5943, | |
| "theoretical_loss": 3.3915457398370323, | |
| "tokens_seen": 2297036800 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 0.000126342916587857, | |
| "loss": 2.6606, | |
| "theoretical_loss": 3.3911535995744995, | |
| "tokens_seen": 2300313600 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 0.00012551541516928316, | |
| "loss": 2.7144, | |
| "theoretical_loss": 3.390762173674479, | |
| "tokens_seen": 2303590400 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 0.00012468791375070927, | |
| "loss": 2.6978, | |
| "theoretical_loss": 3.3903714598222243, | |
| "tokens_seen": 2306867200 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 0.00012386041233213542, | |
| "loss": 2.6477, | |
| "theoretical_loss": 3.389981455713767, | |
| "tokens_seen": 2310144000 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 0.00012303291091356156, | |
| "loss": 2.6676, | |
| "theoretical_loss": 3.389592159055845, | |
| "tokens_seen": 2313420800 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 0.0001222054094949877, | |
| "loss": 2.6261, | |
| "theoretical_loss": 3.389203567565843, | |
| "tokens_seen": 2316697600 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 0.00012137790807641384, | |
| "loss": 2.556, | |
| "theoretical_loss": 3.3888156789717243, | |
| "tokens_seen": 2319974400 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 0.00012055040665783997, | |
| "loss": 2.6187, | |
| "theoretical_loss": 3.3884284910119704, | |
| "tokens_seen": 2323251200 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 0.00011972290523926612, | |
| "loss": 2.5889, | |
| "theoretical_loss": 3.3880420014355135, | |
| "tokens_seen": 2326528000 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 0.00011889540382069226, | |
| "loss": 2.5669, | |
| "theoretical_loss": 3.387656208001678, | |
| "tokens_seen": 2329804800 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 0.0001180679024021184, | |
| "loss": 2.5754, | |
| "theoretical_loss": 3.387271108480113, | |
| "tokens_seen": 2333081600 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "learning_rate": 0.00011724040098354454, | |
| "loss": 2.5648, | |
| "theoretical_loss": 3.386886700650736, | |
| "tokens_seen": 2336358400 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 0.00011641289956497069, | |
| "loss": 2.5755, | |
| "theoretical_loss": 3.386502982303667, | |
| "tokens_seen": 2339635200 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 0.00011558539814639682, | |
| "loss": 2.5348, | |
| "theoretical_loss": 3.3861199512391673, | |
| "tokens_seen": 2342912000 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 0.00011475789672782296, | |
| "loss": 2.5981, | |
| "theoretical_loss": 3.385737605267583, | |
| "tokens_seen": 2346188800 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 0.00011393039530924911, | |
| "loss": 2.5657, | |
| "theoretical_loss": 3.38535594220928, | |
| "tokens_seen": 2349465600 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 0.00011310289389067524, | |
| "loss": 2.5559, | |
| "theoretical_loss": 3.3849749598945875, | |
| "tokens_seen": 2352742400 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 0.00011227539247210139, | |
| "loss": 2.5472, | |
| "theoretical_loss": 3.3845946561637366, | |
| "tokens_seen": 2356019200 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 0.0001114478910535275, | |
| "loss": 2.6395, | |
| "theoretical_loss": 3.384215028866803, | |
| "tokens_seen": 2359296000 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 0.00011062038963495366, | |
| "loss": 2.6597, | |
| "theoretical_loss": 3.3838360758636474, | |
| "tokens_seen": 2362572800 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 0.00010979288821637981, | |
| "loss": 2.5951, | |
| "theoretical_loss": 3.3834577950238582, | |
| "tokens_seen": 2365849600 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "learning_rate": 0.00010896538679780593, | |
| "loss": 2.6092, | |
| "theoretical_loss": 3.383080184226693, | |
| "tokens_seen": 2369126400 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "learning_rate": 0.00010813788537923207, | |
| "loss": 2.6244, | |
| "theoretical_loss": 3.3827032413610225, | |
| "tokens_seen": 2372403200 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "learning_rate": 0.00010731038396065823, | |
| "loss": 2.5959, | |
| "theoretical_loss": 3.382326964325273, | |
| "tokens_seen": 2375680000 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "learning_rate": 0.00010648288254208435, | |
| "loss": 2.5713, | |
| "theoretical_loss": 3.3819513510273698, | |
| "tokens_seen": 2378956800 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "learning_rate": 0.0001056553811235105, | |
| "loss": 2.5767, | |
| "theoretical_loss": 3.381576399384682, | |
| "tokens_seen": 2382233600 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "learning_rate": 0.00010482787970493663, | |
| "loss": 2.6158, | |
| "theoretical_loss": 3.3812021073239675, | |
| "tokens_seen": 2385510400 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "learning_rate": 0.00010400037828636277, | |
| "loss": 2.6135, | |
| "theoretical_loss": 3.380828472781316, | |
| "tokens_seen": 2388787200 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "learning_rate": 0.00010317287686778892, | |
| "loss": 2.6155, | |
| "theoretical_loss": 3.380455493702095, | |
| "tokens_seen": 2392064000 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 0.00010234537544921505, | |
| "loss": 2.5991, | |
| "theoretical_loss": 3.3800831680408963, | |
| "tokens_seen": 2395340800 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 0.00010153442405901267, | |
| "loss": 2.5776, | |
| "theoretical_loss": 3.3797114937614814, | |
| "tokens_seen": 2398617600 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 0.00010070692264043882, | |
| "loss": 2.5957, | |
| "theoretical_loss": 3.3793404688367277, | |
| "tokens_seen": 2401894400 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 9.987942122186495e-05, | |
| "loss": 2.579, | |
| "theoretical_loss": 3.378970091248576, | |
| "tokens_seen": 2405171200 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 9.90519198032911e-05, | |
| "loss": 2.5646, | |
| "theoretical_loss": 3.3786003589879763, | |
| "tokens_seen": 2408448000 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 9.822441838471724e-05, | |
| "loss": 2.5251, | |
| "theoretical_loss": 3.3782312700548376, | |
| "tokens_seen": 2411724800 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 9.739691696614337e-05, | |
| "loss": 2.5219, | |
| "theoretical_loss": 3.377862822457974, | |
| "tokens_seen": 2415001600 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 9.656941554756952e-05, | |
| "loss": 2.5311, | |
| "theoretical_loss": 3.3774950142150537, | |
| "tokens_seen": 2418278400 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 9.574191412899566e-05, | |
| "loss": 2.5407, | |
| "theoretical_loss": 3.3771278433525476, | |
| "tokens_seen": 2421555200 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "learning_rate": 9.491441271042178e-05, | |
| "loss": 2.5535, | |
| "theoretical_loss": 3.376761307905678, | |
| "tokens_seen": 2424832000 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "learning_rate": 9.408691129184794e-05, | |
| "loss": 2.5607, | |
| "theoretical_loss": 3.376395405918369, | |
| "tokens_seen": 2428108800 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "learning_rate": 9.325940987327406e-05, | |
| "loss": 2.5425, | |
| "theoretical_loss": 3.3760301354431963, | |
| "tokens_seen": 2431385600 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "learning_rate": 9.24319084547002e-05, | |
| "loss": 2.5471, | |
| "theoretical_loss": 3.375665494541335, | |
| "tokens_seen": 2434662400 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "learning_rate": 9.160440703612635e-05, | |
| "loss": 2.5007, | |
| "theoretical_loss": 3.3753014812825133, | |
| "tokens_seen": 2437939200 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "learning_rate": 9.077690561755248e-05, | |
| "loss": 2.4879, | |
| "theoretical_loss": 3.374938093744962, | |
| "tokens_seen": 2441216000 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "learning_rate": 8.994940419897862e-05, | |
| "loss": 2.519, | |
| "theoretical_loss": 3.3745753300153636, | |
| "tokens_seen": 2444492800 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "learning_rate": 8.912190278040477e-05, | |
| "loss": 2.514, | |
| "theoretical_loss": 3.374213188188808, | |
| "tokens_seen": 2447769600 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 8.82944013618309e-05, | |
| "loss": 2.5187, | |
| "theoretical_loss": 3.373851666368742, | |
| "tokens_seen": 2451046400 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 8.746689994325705e-05, | |
| "loss": 2.5147, | |
| "theoretical_loss": 3.373490762666921, | |
| "tokens_seen": 2454323200 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "objective/train/avg_token_score": 0.009017497301101685, | |
| "objective/train/avg_weight": 0.9917955994606018, | |
| "objective/train/docs_used": 1380428, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.5374679565429688, | |
| "objective/train/std_weight": 0.06343280524015427, | |
| "objective/train/theoretical_loss": 3.3731304752033617, | |
| "objective/train/tokens_used": 2478060000, | |
| "theoretical_loss": 3.3731304752033617, | |
| "tokens_seen": 2457600000 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 8.663939852468318e-05, | |
| "loss": 2.5545, | |
| "theoretical_loss": 3.3731304752033617, | |
| "tokens_seen": 2457600000 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 8.581189710610932e-05, | |
| "loss": 2.549, | |
| "theoretical_loss": 3.372770802106297, | |
| "tokens_seen": 2460876800 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 8.498439568753547e-05, | |
| "loss": 2.5349, | |
| "theoretical_loss": 3.3724117415121273, | |
| "tokens_seen": 2464153600 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 8.41568942689616e-05, | |
| "loss": 2.5417, | |
| "theoretical_loss": 3.372053291565374, | |
| "tokens_seen": 2467430400 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 8.332939285038775e-05, | |
| "loss": 2.561, | |
| "theoretical_loss": 3.3716954504186347, | |
| "tokens_seen": 2470707200 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 8.250189143181388e-05, | |
| "loss": 2.525, | |
| "theoretical_loss": 3.371338216232536, | |
| "tokens_seen": 2473984000 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 8.167439001324002e-05, | |
| "loss": 2.5362, | |
| "theoretical_loss": 3.370981587175689, | |
| "tokens_seen": 2477260800 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "learning_rate": 8.084688859466617e-05, | |
| "loss": 2.5187, | |
| "theoretical_loss": 3.370625561424644, | |
| "tokens_seen": 2480537600 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "learning_rate": 8.00193871760923e-05, | |
| "loss": 2.4826, | |
| "theoretical_loss": 3.3702701371638453, | |
| "tokens_seen": 2483814400 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "learning_rate": 7.919188575751843e-05, | |
| "loss": 2.4837, | |
| "theoretical_loss": 3.369915312585586, | |
| "tokens_seen": 2487091200 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "learning_rate": 7.836438433894458e-05, | |
| "loss": 2.485, | |
| "theoretical_loss": 3.3695610858899663, | |
| "tokens_seen": 2490368000 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "learning_rate": 7.753688292037072e-05, | |
| "loss": 2.4565, | |
| "theoretical_loss": 3.3692074552848457, | |
| "tokens_seen": 2493644800 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "learning_rate": 7.670938150179686e-05, | |
| "loss": 2.5061, | |
| "theoretical_loss": 3.368854418985804, | |
| "tokens_seen": 2496921600 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "learning_rate": 7.5881880083223e-05, | |
| "loss": 2.4649, | |
| "theoretical_loss": 3.368501975216093, | |
| "tokens_seen": 2500198400 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "learning_rate": 7.505437866464915e-05, | |
| "loss": 2.4924, | |
| "theoretical_loss": 3.3681501222065977, | |
| "tokens_seen": 2503475200 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 7.422687724607528e-05, | |
| "loss": 2.4882, | |
| "theoretical_loss": 3.367798858195792, | |
| "tokens_seen": 2506752000 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 7.339937582750141e-05, | |
| "loss": 2.5234, | |
| "theoretical_loss": 3.367448181429695, | |
| "tokens_seen": 2510028800 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 7.257187440892756e-05, | |
| "loss": 2.5222, | |
| "theoretical_loss": 3.36709809016183, | |
| "tokens_seen": 2513305600 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 7.17443729903537e-05, | |
| "loss": 2.542, | |
| "theoretical_loss": 3.3667485826531847, | |
| "tokens_seen": 2516582400 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 7.091687157177983e-05, | |
| "loss": 2.5081, | |
| "theoretical_loss": 3.366399657172166, | |
| "tokens_seen": 2519859200 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 7.008937015320598e-05, | |
| "loss": 2.4884, | |
| "theoretical_loss": 3.36605131199456, | |
| "tokens_seen": 2523136000 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 6.926186873463212e-05, | |
| "loss": 2.4404, | |
| "theoretical_loss": 3.365703545403493, | |
| "tokens_seen": 2526412800 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 6.843436731605826e-05, | |
| "loss": 2.4269, | |
| "theoretical_loss": 3.3653563556893875, | |
| "tokens_seen": 2529689600 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 6.76068658974844e-05, | |
| "loss": 2.4554, | |
| "theoretical_loss": 3.365009741149925, | |
| "tokens_seen": 2532966400 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "learning_rate": 6.677936447891053e-05, | |
| "loss": 2.4884, | |
| "theoretical_loss": 3.364663700090004, | |
| "tokens_seen": 2536243200 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "learning_rate": 6.595186306033668e-05, | |
| "loss": 2.461, | |
| "theoretical_loss": 3.3643182308216986, | |
| "tokens_seen": 2539520000 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "learning_rate": 6.512436164176281e-05, | |
| "loss": 2.4469, | |
| "theoretical_loss": 3.3639733316642233, | |
| "tokens_seen": 2542796800 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "learning_rate": 6.429686022318895e-05, | |
| "loss": 2.4611, | |
| "theoretical_loss": 3.36362900094389, | |
| "tokens_seen": 2546073600 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "learning_rate": 6.346935880461509e-05, | |
| "loss": 2.4757, | |
| "theoretical_loss": 3.3632852369940696, | |
| "tokens_seen": 2549350400 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "learning_rate": 6.264185738604123e-05, | |
| "loss": 2.4886, | |
| "theoretical_loss": 3.362942038155154, | |
| "tokens_seen": 2552627200 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "learning_rate": 6.181435596746738e-05, | |
| "loss": 2.4206, | |
| "theoretical_loss": 3.362599402774518, | |
| "tokens_seen": 2555904000 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "learning_rate": 6.098685454889351e-05, | |
| "loss": 2.4852, | |
| "theoretical_loss": 3.362257329206479, | |
| "tokens_seen": 2559180800 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 6.0159353130319655e-05, | |
| "loss": 2.4987, | |
| "theoretical_loss": 3.36191581581226, | |
| "tokens_seen": 2562457600 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 5.933185171174579e-05, | |
| "loss": 2.4837, | |
| "theoretical_loss": 3.3615748609599545, | |
| "tokens_seen": 2565734400 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 5.850435029317193e-05, | |
| "loss": 2.4585, | |
| "theoretical_loss": 3.361234463024483, | |
| "tokens_seen": 2569011200 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 5.767684887459807e-05, | |
| "loss": 2.4455, | |
| "theoretical_loss": 3.3608946203875614, | |
| "tokens_seen": 2572288000 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 5.6849347456024216e-05, | |
| "loss": 2.4754, | |
| "theoretical_loss": 3.360555331437662, | |
| "tokens_seen": 2575564800 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 5.602184603745035e-05, | |
| "loss": 2.5134, | |
| "theoretical_loss": 3.360216594569975, | |
| "tokens_seen": 2578841600 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 5.5194344618876486e-05, | |
| "loss": 2.4456, | |
| "theoretical_loss": 3.3598784081863755, | |
| "tokens_seen": 2582118400 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 5.4366843200302625e-05, | |
| "loss": 2.4831, | |
| "theoretical_loss": 3.3595407706953835, | |
| "tokens_seen": 2585395200 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 5.353934178172877e-05, | |
| "loss": 2.477, | |
| "theoretical_loss": 3.359203680512131, | |
| "tokens_seen": 2588672000 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 5.271184036315491e-05, | |
| "loss": 2.4893, | |
| "theoretical_loss": 3.3588671360583247, | |
| "tokens_seen": 2591948800 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 5.188433894458105e-05, | |
| "loss": 2.4997, | |
| "theoretical_loss": 3.3585311357622096, | |
| "tokens_seen": 2595225600 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 5.1056837526007186e-05, | |
| "loss": 2.5057, | |
| "theoretical_loss": 3.358195678058536, | |
| "tokens_seen": 2598502400 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 5.022933610743333e-05, | |
| "loss": 2.5474, | |
| "theoretical_loss": 3.3578607613885225, | |
| "tokens_seen": 2601779200 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 4.940183468885946e-05, | |
| "loss": 2.5399, | |
| "theoretical_loss": 3.357526384199822, | |
| "tokens_seen": 2605056000 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 4.85743332702856e-05, | |
| "loss": 2.5357, | |
| "theoretical_loss": 3.3571925449464874, | |
| "tokens_seen": 2608332800 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 4.774683185171175e-05, | |
| "loss": 2.5203, | |
| "theoretical_loss": 3.356859242088935, | |
| "tokens_seen": 2611609600 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "learning_rate": 4.6919330433137886e-05, | |
| "loss": 2.5076, | |
| "theoretical_loss": 3.356526474093914, | |
| "tokens_seen": 2614886400 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 4.6091829014564024e-05, | |
| "loss": 2.5442, | |
| "theoretical_loss": 3.35619423943447, | |
| "tokens_seen": 2618163200 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.449855913696806, | |
| "debugging/distinct-1-grams": 0.769778305351557, | |
| "debugging/distinct-2-grams": 0.9467712904639874, | |
| "debugging/entropy-1-grams": 5.998633443012235, | |
| "debugging/entropy-2-grams": 6.951908950814323, | |
| "debugging/length": 469.11764705882354, | |
| "debugging/num_segments": 17, | |
| "epoch": 0.94, | |
| "objective/train/avg_token_score": 0.039095163345336914, | |
| "objective/train/avg_weight": 0.9644700288772583, | |
| "objective/train/docs_used": 1474904, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.570134162902832, | |
| "objective/train/std_weight": 0.1007777601480484, | |
| "objective/train/theoretical_loss": 3.3558625365899117, | |
| "objective/train/tokens_used": 2641900000, | |
| "theoretical_loss": 3.3558625365899117, | |
| "tokens_seen": 2621440000 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 4.526432759599016e-05, | |
| "loss": 2.5674, | |
| "theoretical_loss": 3.3558625365899117, | |
| "tokens_seen": 2621440000 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 4.443682617741631e-05, | |
| "loss": 2.5636, | |
| "theoretical_loss": 3.3555313640457776, | |
| "tokens_seen": 2624716800 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 4.360932475884245e-05, | |
| "loss": 2.5055, | |
| "theoretical_loss": 3.355200720293803, | |
| "tokens_seen": 2627993600 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 4.278182334026858e-05, | |
| "loss": 2.5273, | |
| "theoretical_loss": 3.3548706038318867, | |
| "tokens_seen": 2631270400 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 4.1954321921694724e-05, | |
| "loss": 2.4866, | |
| "theoretical_loss": 3.3545410131640585, | |
| "tokens_seen": 2634547200 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 4.112682050312086e-05, | |
| "loss": 2.5063, | |
| "theoretical_loss": 3.354211946800446, | |
| "tokens_seen": 2637824000 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 4.0299319084547e-05, | |
| "loss": 2.5056, | |
| "theoretical_loss": 3.3538834032572424, | |
| "tokens_seen": 2641100800 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 3.947181766597314e-05, | |
| "loss": 2.528, | |
| "theoretical_loss": 3.3535553810566743, | |
| "tokens_seen": 2644377600 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 3.864431624739928e-05, | |
| "loss": 2.5232, | |
| "theoretical_loss": 3.3532278787269707, | |
| "tokens_seen": 2647654400 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 3.781681482882542e-05, | |
| "loss": 2.4993, | |
| "theoretical_loss": 3.35290089480233, | |
| "tokens_seen": 2650931200 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 3.698931341025156e-05, | |
| "loss": 2.5075, | |
| "theoretical_loss": 3.352574427822888, | |
| "tokens_seen": 2654208000 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 3.61618119916777e-05, | |
| "loss": 2.5071, | |
| "theoretical_loss": 3.3522484763346885, | |
| "tokens_seen": 2657484800 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 3.533431057310384e-05, | |
| "loss": 2.5007, | |
| "theoretical_loss": 3.3519230388896504, | |
| "tokens_seen": 2660761600 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 3.450680915452998e-05, | |
| "loss": 2.5096, | |
| "theoretical_loss": 3.3515981140455375, | |
| "tokens_seen": 2664038400 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 3.3679307735956117e-05, | |
| "loss": 2.5152, | |
| "theoretical_loss": 3.3512737003659288, | |
| "tokens_seen": 2667315200 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 3.285180631738226e-05, | |
| "loss": 2.5159, | |
| "theoretical_loss": 3.350949796420184, | |
| "tokens_seen": 2670592000 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "learning_rate": 3.2024304898808394e-05, | |
| "loss": 2.522, | |
| "theoretical_loss": 3.3506264007834186, | |
| "tokens_seen": 2673868800 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 3.119680348023454e-05, | |
| "loss": 2.5375, | |
| "theoretical_loss": 3.3503035120364704, | |
| "tokens_seen": 2677145600 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 3.0369302061660674e-05, | |
| "loss": 2.523, | |
| "theoretical_loss": 3.349981128765869, | |
| "tokens_seen": 2680422400 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 2.9541800643086816e-05, | |
| "loss": 2.5204, | |
| "theoretical_loss": 3.3496592495638096, | |
| "tokens_seen": 2683699200 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 2.8714299224512955e-05, | |
| "loss": 2.4853, | |
| "theoretical_loss": 3.349337873028119, | |
| "tokens_seen": 2686976000 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 2.7886797805939097e-05, | |
| "loss": 2.5156, | |
| "theoretical_loss": 3.349016997762229, | |
| "tokens_seen": 2690252800 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 2.7059296387365232e-05, | |
| "loss": 2.4785, | |
| "theoretical_loss": 3.348696622375147, | |
| "tokens_seen": 2693529600 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 2.6231794968791374e-05, | |
| "loss": 2.4962, | |
| "theoretical_loss": 3.3483767454814255, | |
| "tokens_seen": 2696806400 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 2.5404293550217513e-05, | |
| "loss": 2.5055, | |
| "theoretical_loss": 3.3480573657011354, | |
| "tokens_seen": 2700083200 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 2.4576792131643655e-05, | |
| "loss": 2.4814, | |
| "theoretical_loss": 3.347738481659836, | |
| "tokens_seen": 2703360000 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 2.3749290713069797e-05, | |
| "loss": 2.4681, | |
| "theoretical_loss": 3.347420091988546, | |
| "tokens_seen": 2706636800 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 2.2921789294495932e-05, | |
| "loss": 2.4964, | |
| "theoretical_loss": 3.3471021953237177, | |
| "tokens_seen": 2709913600 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 2.2094287875922074e-05, | |
| "loss": 2.5121, | |
| "theoretical_loss": 3.346784790307205, | |
| "tokens_seen": 2713190400 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 2.1266786457348212e-05, | |
| "loss": 2.499, | |
| "theoretical_loss": 3.346467875586239, | |
| "tokens_seen": 2716467200 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 2.043928503877435e-05, | |
| "loss": 2.5797, | |
| "theoretical_loss": 3.3461514498133997, | |
| "tokens_seen": 2719744000 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 1.961178362020049e-05, | |
| "loss": 2.5741, | |
| "theoretical_loss": 3.345835511646587, | |
| "tokens_seen": 2723020800 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 1.878428220162663e-05, | |
| "loss": 2.5581, | |
| "theoretical_loss": 3.345520059748993, | |
| "tokens_seen": 2726297600 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "learning_rate": 1.795678078305277e-05, | |
| "loss": 2.5442, | |
| "theoretical_loss": 3.3452050927890777, | |
| "tokens_seen": 2729574400 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 1.7129279364478912e-05, | |
| "loss": 2.5597, | |
| "theoretical_loss": 3.3448906094405393, | |
| "tokens_seen": 2732851200 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 1.630177794590505e-05, | |
| "loss": 2.5501, | |
| "theoretical_loss": 3.344576608382288, | |
| "tokens_seen": 2736128000 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 1.547427652733119e-05, | |
| "loss": 2.5704, | |
| "theoretical_loss": 3.34426308829842, | |
| "tokens_seen": 2739404800 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 1.464677510875733e-05, | |
| "loss": 2.57, | |
| "theoretical_loss": 3.343950047878189, | |
| "tokens_seen": 2742681600 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 1.381927369018347e-05, | |
| "loss": 2.5797, | |
| "theoretical_loss": 3.343637485815983, | |
| "tokens_seen": 2745958400 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 1.2991772271609609e-05, | |
| "loss": 2.5643, | |
| "theoretical_loss": 3.3433254008112963, | |
| "tokens_seen": 2749235200 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 1.2164270853035749e-05, | |
| "loss": 2.5687, | |
| "theoretical_loss": 3.343013791568702, | |
| "tokens_seen": 2752512000 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 1.1336769434461887e-05, | |
| "loss": 2.5393, | |
| "theoretical_loss": 3.342702656797829, | |
| "tokens_seen": 2755788800 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 1.0509268015888028e-05, | |
| "loss": 2.5178, | |
| "theoretical_loss": 3.342391995213335, | |
| "tokens_seen": 2759065600 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 9.681766597314166e-06, | |
| "loss": 2.5289, | |
| "theoretical_loss": 3.34208180553488, | |
| "tokens_seen": 2762342400 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 8.854265178740307e-06, | |
| "loss": 2.5114, | |
| "theoretical_loss": 3.341772086487102, | |
| "tokens_seen": 2765619200 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 8.026763760166447e-06, | |
| "loss": 2.5689, | |
| "theoretical_loss": 3.341462836799593, | |
| "tokens_seen": 2768896000 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 7.199262341592586e-06, | |
| "loss": 2.5621, | |
| "theoretical_loss": 3.341154055206871, | |
| "tokens_seen": 2772172800 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 6.371760923018726e-06, | |
| "loss": 2.6112, | |
| "theoretical_loss": 3.3408457404483576, | |
| "tokens_seen": 2775449600 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 5.544259504444865e-06, | |
| "loss": 2.5833, | |
| "theoretical_loss": 3.3405378912683514, | |
| "tokens_seen": 2778726400 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 4.716758085871005e-06, | |
| "loss": 2.6055, | |
| "theoretical_loss": 3.340230506416005, | |
| "tokens_seen": 2782003200 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "objective/train/avg_token_score": 0.025402138009667397, | |
| "objective/train/avg_weight": 0.9769189953804016, | |
| "objective/train/docs_used": 1566749, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.588498592376709, | |
| "objective/train/std_weight": 0.1057284027338028, | |
| "objective/train/theoretical_loss": 3.3399235846453004, | |
| "objective/train/tokens_used": 2805740000, | |
| "theoretical_loss": 3.3399235846453004, | |
| "tokens_seen": 2785280000 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "learning_rate": 3.889256667297144e-06, | |
| "loss": 2.618, | |
| "theoretical_loss": 3.3399235846453004, | |
| "tokens_seen": 2785280000 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "learning_rate": 3.061755248723284e-06, | |
| "loss": 2.5737, | |
| "theoretical_loss": 3.3396171247150224, | |
| "tokens_seen": 2788556800 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "learning_rate": 2.2342538301494233e-06, | |
| "loss": 2.5223, | |
| "theoretical_loss": 3.3393111253887375, | |
| "tokens_seen": 2791833600 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "learning_rate": 1.4067524115755626e-06, | |
| "loss": 2.5394, | |
| "theoretical_loss": 3.339005585434769, | |
| "tokens_seen": 2795110400 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "learning_rate": 5.792509930017022e-07, | |
| "loss": 2.5056, | |
| "theoretical_loss": 3.338700503626171, | |
| "tokens_seen": 2798387200 | |
| } | |
| ], | |
| "max_steps": 42724, | |
| "num_train_epochs": 9223372036854775807, | |
| "total_flos": 1.428920419221504e+18, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |