|
{ |
|
"best_metric": null, |
|
"best_model_checkpoint": null, |
|
"epoch": 0.5, |
|
"global_step": 21362, |
|
"is_hyper_param_search": false, |
|
"is_local_process_zero": true, |
|
"is_world_process_zero": true, |
|
"log_history": [ |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 1.6355140186915887e-06, |
|
"loss": 10.7543, |
|
"theoretical_loss": 20.81281780154715, |
|
"tokens_seen": 65536 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 8.177570093457944e-05, |
|
"loss": 8.6636, |
|
"theoretical_loss": 8.563482664611069, |
|
"tokens_seen": 3276800 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 0.0001635514018691589, |
|
"loss": 6.6432, |
|
"theoretical_loss": 7.4777587180480305, |
|
"tokens_seen": 6553600 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 0.0002453271028037383, |
|
"loss": 5.9475, |
|
"theoretical_loss": 6.9337544888949, |
|
"tokens_seen": 9830400 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 0.0003271028037383178, |
|
"loss": 5.5534, |
|
"theoretical_loss": 6.583566228426414, |
|
"tokens_seen": 13107200 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0004088785046728972, |
|
"loss": 5.3664, |
|
"theoretical_loss": 6.330713565116083, |
|
"tokens_seen": 16384000 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0004906542056074766, |
|
"loss": 5.1899, |
|
"theoretical_loss": 6.135529231940326, |
|
"tokens_seen": 19660800 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0005724299065420561, |
|
"loss": 5.0234, |
|
"theoretical_loss": 5.978101583869607, |
|
"tokens_seen": 22937600 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0006542056074766356, |
|
"loss": 4.9249, |
|
"theoretical_loss": 5.8471173262659235, |
|
"tokens_seen": 26214400 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0006996358993758275, |
|
"loss": 4.7711, |
|
"theoretical_loss": 5.7355768158821245, |
|
"tokens_seen": 29491200 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0006988083979572536, |
|
"loss": 4.7076, |
|
"theoretical_loss": 5.638870144071353, |
|
"tokens_seen": 32768000 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0006979808965386797, |
|
"loss": 4.5841, |
|
"theoretical_loss": 5.553812381844907, |
|
"tokens_seen": 36044800 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.000697153395120106, |
|
"loss": 4.5051, |
|
"theoretical_loss": 5.478118080556438, |
|
"tokens_seen": 39321600 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0006963258937015321, |
|
"loss": 4.4155, |
|
"theoretical_loss": 5.410095959579362, |
|
"tokens_seen": 42598400 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0006954983922829582, |
|
"loss": 4.3751, |
|
"theoretical_loss": 5.348462083735834, |
|
"tokens_seen": 45875200 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0006946708908643843, |
|
"loss": 4.2199, |
|
"theoretical_loss": 5.292220566937567, |
|
"tokens_seen": 49152000 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0006938433894458105, |
|
"loss": 4.189, |
|
"theoretical_loss": 5.240584625769978, |
|
"tokens_seen": 52428800 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0006930158880272367, |
|
"loss": 4.1234, |
|
"theoretical_loss": 5.192922724525789, |
|
"tokens_seen": 55705600 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0006921883866086628, |
|
"loss": 4.051, |
|
"theoretical_loss": 5.1487208633564405, |
|
"tokens_seen": 58982400 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0006913608851900889, |
|
"loss": 3.9171, |
|
"theoretical_loss": 5.107555562405102, |
|
"tokens_seen": 62259200 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.000690533383771515, |
|
"loss": 3.8106, |
|
"theoretical_loss": 5.069074117143246, |
|
"tokens_seen": 65536000 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0006897058823529412, |
|
"loss": 3.7893, |
|
"theoretical_loss": 5.032979909838007, |
|
"tokens_seen": 68812800 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0006888783809343674, |
|
"loss": 3.7301, |
|
"theoretical_loss": 4.999021308224664, |
|
"tokens_seen": 72089600 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0006880508795157935, |
|
"loss": 3.6744, |
|
"theoretical_loss": 4.966983155351962, |
|
"tokens_seen": 75366400 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0006872233780972196, |
|
"loss": 3.5873, |
|
"theoretical_loss": 4.9366801616251355, |
|
"tokens_seen": 78643200 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0006863958766786457, |
|
"loss": 3.5755, |
|
"theoretical_loss": 4.907951713830082, |
|
"tokens_seen": 81920000 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0006855683752600718, |
|
"loss": 3.5132, |
|
"theoretical_loss": 4.880657753812926, |
|
"tokens_seen": 85196800 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.000684740873841498, |
|
"loss": 3.5089, |
|
"theoretical_loss": 4.854675474481779, |
|
"tokens_seen": 88473600 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0006839133724229242, |
|
"loss": 3.4237, |
|
"theoretical_loss": 4.8298966473088125, |
|
"tokens_seen": 91750400 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0006830858710043503, |
|
"loss": 3.4596, |
|
"theoretical_loss": 4.8062254427779205, |
|
"tokens_seen": 95027200 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0006822583695857764, |
|
"loss": 3.4046, |
|
"theoretical_loss": 4.783576639276257, |
|
"tokens_seen": 98304000 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0006814308681672025, |
|
"loss": 3.4409, |
|
"theoretical_loss": 4.761874140772408, |
|
"tokens_seen": 101580800 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0006806033667486286, |
|
"loss": 3.4431, |
|
"theoretical_loss": 4.741049741962473, |
|
"tokens_seen": 104857600 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0006797758653300548, |
|
"loss": 3.4137, |
|
"theoretical_loss": 4.721042093249051, |
|
"tokens_seen": 108134400 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.000678948363911481, |
|
"loss": 3.3936, |
|
"theoretical_loss": 4.701795828231866, |
|
"tokens_seen": 111411200 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0006781208624929071, |
|
"loss": 3.3917, |
|
"theoretical_loss": 4.68326082423593, |
|
"tokens_seen": 114688000 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0006772933610743332, |
|
"loss": 3.3676, |
|
"theoretical_loss": 4.665391572426282, |
|
"tokens_seen": 117964800 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0006764658596557593, |
|
"loss": 3.3325, |
|
"theoretical_loss": 4.648146638719739, |
|
"tokens_seen": 121241600 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0006756383582371856, |
|
"loss": 3.345, |
|
"theoretical_loss": 4.631488200339643, |
|
"tokens_seen": 124518400 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0006748108568186117, |
|
"loss": 3.371, |
|
"theoretical_loss": 4.615381645715717, |
|
"tokens_seen": 127795200 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0006739833554000378, |
|
"loss": 3.2953, |
|
"theoretical_loss": 4.599795227690505, |
|
"tokens_seen": 131072000 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.000673155853981464, |
|
"loss": 3.3211, |
|
"theoretical_loss": 4.584699761792674, |
|
"tokens_seen": 134348800 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0006723283525628902, |
|
"loss": 3.2745, |
|
"theoretical_loss": 4.570068362778516, |
|
"tokens_seen": 137625600 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0006715008511443163, |
|
"loss": 3.289, |
|
"theoretical_loss": 4.555876213804037, |
|
"tokens_seen": 140902400 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0006706733497257424, |
|
"loss": 3.2753, |
|
"theoretical_loss": 4.542100363530799, |
|
"tokens_seen": 144179200 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0006698458483071685, |
|
"loss": 3.2546, |
|
"theoretical_loss": 4.528719547234816, |
|
"tokens_seen": 147456000 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0006690183468885946, |
|
"loss": 3.2145, |
|
"theoretical_loss": 4.515714028614996, |
|
"tokens_seen": 150732800 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0006681908454700209, |
|
"loss": 3.2252, |
|
"theoretical_loss": 4.503065459513339, |
|
"tokens_seen": 154009600 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.000667363344051447, |
|
"loss": 3.2379, |
|
"theoretical_loss": 4.4907567551852665, |
|
"tokens_seen": 157286400 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0006665358426328731, |
|
"loss": 3.2175, |
|
"theoretical_loss": 4.478771983111967, |
|
"tokens_seen": 160563200 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"objective/train/avg_token_score": 0.027871694415807724, |
|
"objective/train/avg_weight": 0.9700481295585632, |
|
"objective/train/docs_used": 104000, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 3.27850341796875, |
|
"objective/train/std_weight": 0.08482376486063004, |
|
"objective/train/theoretical_loss": 4.467096263641219, |
|
"objective/train/tokens_used": 184300000, |
|
"theoretical_loss": 4.467096263641219, |
|
"tokens_seen": 163840000 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0006657083412142992, |
|
"loss": 3.2065, |
|
"theoretical_loss": 4.467096263641219, |
|
"tokens_seen": 163840000 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0006648808397957253, |
|
"loss": 3.2165, |
|
"theoretical_loss": 4.455715680989545, |
|
"tokens_seen": 167116800 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0006640533383771514, |
|
"loss": 3.168, |
|
"theoretical_loss": 4.44461720334543, |
|
"tokens_seen": 170393600 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0006632258369585777, |
|
"loss": 3.118, |
|
"theoretical_loss": 4.433788610987646, |
|
"tokens_seen": 173670400 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0006623983355400038, |
|
"loss": 3.1702, |
|
"theoretical_loss": 4.42321843148016, |
|
"tokens_seen": 176947200 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0006615708341214299, |
|
"loss": 3.1235, |
|
"theoretical_loss": 4.412895881130142, |
|
"tokens_seen": 180224000 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.000660743332702856, |
|
"loss": 3.1592, |
|
"theoretical_loss": 4.4028108120020795, |
|
"tokens_seen": 183500800 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0006599158312842821, |
|
"loss": 3.1533, |
|
"theoretical_loss": 4.392953663871862, |
|
"tokens_seen": 186777600 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0006590883298657083, |
|
"loss": 3.1398, |
|
"theoretical_loss": 4.383315420582533, |
|
"tokens_seen": 190054400 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0006582608284471345, |
|
"loss": 3.1768, |
|
"theoretical_loss": 4.373887570330275, |
|
"tokens_seen": 193331200 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0006574333270285606, |
|
"loss": 3.136, |
|
"theoretical_loss": 4.364662069466704, |
|
"tokens_seen": 196608000 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0006566058256099867, |
|
"loss": 3.134, |
|
"theoretical_loss": 4.355631309453283, |
|
"tokens_seen": 199884800 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0006557783241914128, |
|
"loss": 3.0886, |
|
"theoretical_loss": 4.346788086646671, |
|
"tokens_seen": 203161600 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0006549508227728391, |
|
"loss": 3.1173, |
|
"theoretical_loss": 4.33812557463116, |
|
"tokens_seen": 206438400 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0006541233213542652, |
|
"loss": 3.1278, |
|
"theoretical_loss": 4.329637298846812, |
|
"tokens_seen": 209715200 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0006532958199356913, |
|
"loss": 3.0931, |
|
"theoretical_loss": 4.321317113290252, |
|
"tokens_seen": 212992000 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0006524683185171174, |
|
"loss": 3.0638, |
|
"theoretical_loss": 4.3131591790897925, |
|
"tokens_seen": 216268800 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0006516408170985437, |
|
"loss": 3.1152, |
|
"theoretical_loss": 4.305157944778228, |
|
"tokens_seen": 219545600 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0006508133156799698, |
|
"loss": 3.1342, |
|
"theoretical_loss": 4.297308128105687, |
|
"tokens_seen": 222822400 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0006499858142613959, |
|
"loss": 3.1163, |
|
"theoretical_loss": 4.2896046992515995, |
|
"tokens_seen": 226099200 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.000649158312842822, |
|
"loss": 3.152, |
|
"theoretical_loss": 4.282042865309616, |
|
"tokens_seen": 229376000 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0006483308114242481, |
|
"loss": 3.1198, |
|
"theoretical_loss": 4.274618055932298, |
|
"tokens_seen": 232652800 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0006475033100056744, |
|
"loss": 3.0873, |
|
"theoretical_loss": 4.267325910033897, |
|
"tokens_seen": 235929600 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0006466758085871005, |
|
"loss": 3.0382, |
|
"theoretical_loss": 4.260162263459744, |
|
"tokens_seen": 239206400 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0006458483071685266, |
|
"loss": 3.0672, |
|
"theoretical_loss": 4.253123137539814, |
|
"tokens_seen": 242483200 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0006450208057499527, |
|
"loss": 3.0837, |
|
"theoretical_loss": 4.246204728452055, |
|
"tokens_seen": 245760000 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0006441933043313788, |
|
"loss": 3.1016, |
|
"theoretical_loss": 4.239403397328261, |
|
"tokens_seen": 249036800 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0006433658029128049, |
|
"loss": 3.0698, |
|
"theoretical_loss": 4.232715661041632, |
|
"tokens_seen": 252313600 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0006425383014942312, |
|
"loss": 3.0794, |
|
"theoretical_loss": 4.226138183620867, |
|
"tokens_seen": 255590400 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0006417108000756573, |
|
"loss": 3.0643, |
|
"theoretical_loss": 4.219667768240775, |
|
"tokens_seen": 258867200 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0006408832986570834, |
|
"loss": 3.0542, |
|
"theoretical_loss": 4.213301349743924, |
|
"tokens_seen": 262144000 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0006400557972385095, |
|
"loss": 2.9911, |
|
"theoretical_loss": 4.20703598765197, |
|
"tokens_seen": 265420800 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0006392282958199356, |
|
"loss": 2.9473, |
|
"theoretical_loss": 4.2008688596290025, |
|
"tokens_seen": 268697600 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0006384007944013618, |
|
"loss": 2.9018, |
|
"theoretical_loss": 4.194797255362549, |
|
"tokens_seen": 271974400 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.000637573292982788, |
|
"loss": 2.9774, |
|
"theoretical_loss": 4.188818570830883, |
|
"tokens_seen": 275251200 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0006367457915642141, |
|
"loss": 2.9891, |
|
"theoretical_loss": 4.182930302927963, |
|
"tokens_seen": 278528000 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0006359182901456402, |
|
"loss": 2.952, |
|
"theoretical_loss": 4.17713004441978, |
|
"tokens_seen": 281804800 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0006350907887270663, |
|
"loss": 2.9969, |
|
"theoretical_loss": 4.1714154792080915, |
|
"tokens_seen": 285081600 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0006342632873084925, |
|
"loss": 3.0117, |
|
"theoretical_loss": 4.165784377879517, |
|
"tokens_seen": 288358400 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0006334357858899187, |
|
"loss": 2.9764, |
|
"theoretical_loss": 4.160234593519768, |
|
"tokens_seen": 291635200 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0006326082844713448, |
|
"loss": 2.997, |
|
"theoretical_loss": 4.15476405777444, |
|
"tokens_seen": 294912000 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0006317807830527709, |
|
"loss": 2.9709, |
|
"theoretical_loss": 4.149370777139286, |
|
"tokens_seen": 298188800 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0006309532816341972, |
|
"loss": 2.968, |
|
"theoretical_loss": 4.144052829464249, |
|
"tokens_seen": 301465600 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0006301257802156233, |
|
"loss": 2.9237, |
|
"theoretical_loss": 4.138808360656742, |
|
"tokens_seen": 304742400 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0006292982787970494, |
|
"loss": 2.9387, |
|
"theoretical_loss": 4.133635581570836, |
|
"tokens_seen": 308019200 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0006284707773784755, |
|
"loss": 2.9275, |
|
"theoretical_loss": 4.128532765070004, |
|
"tokens_seen": 311296000 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0006276432759599016, |
|
"loss": 2.9465, |
|
"theoretical_loss": 4.123498243252032, |
|
"tokens_seen": 314572800 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0006268157745413279, |
|
"loss": 2.9295, |
|
"theoretical_loss": 4.118530404825556, |
|
"tokens_seen": 317849600 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.000625988273122754, |
|
"loss": 2.9456, |
|
"theoretical_loss": 4.113627692628464, |
|
"tokens_seen": 321126400 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0006251607717041801, |
|
"loss": 2.949, |
|
"theoretical_loss": 4.108788601279149, |
|
"tokens_seen": 324403200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.5365128506817183, |
|
"debugging/distinct-1-grams": 0.7612814402327299, |
|
"debugging/distinct-2-grams": 0.9694583753853511, |
|
"debugging/entropy-1-grams": 6.003629944255698, |
|
"debugging/entropy-2-grams": 7.054987089269872, |
|
"debugging/length": 495.25, |
|
"debugging/num_segments": 16, |
|
"epoch": 0.12, |
|
"objective/train/avg_token_score": 0.04385810345411301, |
|
"objective/train/avg_weight": 0.9528664350509644, |
|
"objective/train/docs_used": 197327, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 3.1396584510803223, |
|
"objective/train/std_weight": 0.1686321198940277, |
|
"objective/train/theoretical_loss": 4.10401167495222, |
|
"objective/train/tokens_used": 348140000, |
|
"theoretical_loss": 4.10401167495222, |
|
"tokens_seen": 327680000 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0006243332702856062, |
|
"loss": 2.9772, |
|
"theoretical_loss": 4.10401167495222, |
|
"tokens_seen": 327680000 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0006235057688670323, |
|
"loss": 2.9764, |
|
"theoretical_loss": 4.099295505270921, |
|
"tokens_seen": 330956800 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0006226782674484584, |
|
"loss": 2.9224, |
|
"theoretical_loss": 4.094638729309031, |
|
"tokens_seen": 334233600 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0006218507660298847, |
|
"loss": 2.9348, |
|
"theoretical_loss": 4.090040027695556, |
|
"tokens_seen": 337510400 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0006210232646113108, |
|
"loss": 2.8915, |
|
"theoretical_loss": 4.085498122815992, |
|
"tokens_seen": 340787200 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0006201957631927369, |
|
"loss": 2.9147, |
|
"theoretical_loss": 4.081011777104333, |
|
"tokens_seen": 344064000 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.000619368261774163, |
|
"loss": 2.9209, |
|
"theoretical_loss": 4.076579791420469, |
|
"tokens_seen": 347340800 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.0006185407603555891, |
|
"loss": 2.9303, |
|
"theoretical_loss": 4.0722010035079155, |
|
"tokens_seen": 350617600 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.0006177132589370153, |
|
"loss": 2.9261, |
|
"theoretical_loss": 4.067874286527197, |
|
"tokens_seen": 353894400 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.0006168857575184414, |
|
"loss": 2.924, |
|
"theoretical_loss": 4.063598547660519, |
|
"tokens_seen": 357171200 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.0006160582560998676, |
|
"loss": 2.9546, |
|
"theoretical_loss": 4.05937272678363, |
|
"tokens_seen": 360448000 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.0006152307546812937, |
|
"loss": 2.9394, |
|
"theoretical_loss": 4.055195795201069, |
|
"tokens_seen": 363724800 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.0006144032532627198, |
|
"loss": 2.8913, |
|
"theoretical_loss": 4.051066754441235, |
|
"tokens_seen": 367001600 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.000613575751844146, |
|
"loss": 2.8973, |
|
"theoretical_loss": 4.04698463510794, |
|
"tokens_seen": 370278400 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.0006127482504255721, |
|
"loss": 2.901, |
|
"theoretical_loss": 4.042948495785312, |
|
"tokens_seen": 373555200 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.0006119207490069983, |
|
"loss": 2.8647, |
|
"theoretical_loss": 4.038957421993153, |
|
"tokens_seen": 376832000 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0006110932475884244, |
|
"loss": 2.8707, |
|
"theoretical_loss": 4.035010525189982, |
|
"tokens_seen": 380108800 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0006102657461698505, |
|
"loss": 2.904, |
|
"theoretical_loss": 4.031106941821218, |
|
"tokens_seen": 383385600 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0006094382447512768, |
|
"loss": 2.8677, |
|
"theoretical_loss": 4.027245832410079, |
|
"tokens_seen": 386662400 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0006086107433327029, |
|
"loss": 2.8421, |
|
"theoretical_loss": 4.023426380688943, |
|
"tokens_seen": 389939200 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.000607783241914129, |
|
"loss": 2.8849, |
|
"theoretical_loss": 4.019647792769048, |
|
"tokens_seen": 393216000 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0006069557404955551, |
|
"loss": 2.9447, |
|
"theoretical_loss": 4.015909296346521, |
|
"tokens_seen": 396492800 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0006061447891053528, |
|
"loss": 2.9014, |
|
"theoretical_loss": 4.012210139942894, |
|
"tokens_seen": 399769600 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0006053172876867789, |
|
"loss": 2.9463, |
|
"theoretical_loss": 4.008549592178291, |
|
"tokens_seen": 403046400 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.000604489786268205, |
|
"loss": 2.8978, |
|
"theoretical_loss": 4.004926941075674, |
|
"tokens_seen": 406323200 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0006036622848496311, |
|
"loss": 2.9521, |
|
"theoretical_loss": 4.001341493394558, |
|
"tokens_seen": 409600000 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0006028347834310573, |
|
"loss": 2.9652, |
|
"theoretical_loss": 3.997792573992726, |
|
"tokens_seen": 412876800 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0006020072820124835, |
|
"loss": 2.9098, |
|
"theoretical_loss": 3.994279525214554, |
|
"tokens_seen": 416153600 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0006011797805939096, |
|
"loss": 2.9572, |
|
"theoretical_loss": 3.990801706304647, |
|
"tokens_seen": 419430400 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0006003522791753358, |
|
"loss": 2.9129, |
|
"theoretical_loss": 3.987358492845532, |
|
"tokens_seen": 422707200 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0005995247777567619, |
|
"loss": 2.9439, |
|
"theoretical_loss": 3.9839492762182647, |
|
"tokens_seen": 425984000 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.000598697276338188, |
|
"loss": 2.9134, |
|
"theoretical_loss": 3.9805734630848306, |
|
"tokens_seen": 429260800 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0005978697749196142, |
|
"loss": 2.8965, |
|
"theoretical_loss": 3.9772304748913054, |
|
"tokens_seen": 432537600 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.0005970422735010403, |
|
"loss": 2.8898, |
|
"theoretical_loss": 3.973919747390801, |
|
"tokens_seen": 435814400 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.0005962147720824664, |
|
"loss": 2.9002, |
|
"theoretical_loss": 3.9706407301852487, |
|
"tokens_seen": 439091200 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.0005953872706638926, |
|
"loss": 2.8783, |
|
"theoretical_loss": 3.9673928862851655, |
|
"tokens_seen": 442368000 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.0005945597692453187, |
|
"loss": 2.8682, |
|
"theoretical_loss": 3.9641756916865463, |
|
"tokens_seen": 445644800 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.0005937322678267449, |
|
"loss": 2.8783, |
|
"theoretical_loss": 3.960988634964113, |
|
"tokens_seen": 448921600 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.000592904766408171, |
|
"loss": 2.8511, |
|
"theoretical_loss": 3.9578312168801597, |
|
"tokens_seen": 452198400 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.0005920772649895971, |
|
"loss": 2.851, |
|
"theoretical_loss": 3.954702950008308, |
|
"tokens_seen": 455475200 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.0005912497635710232, |
|
"loss": 2.835, |
|
"theoretical_loss": 3.9516033583714734, |
|
"tokens_seen": 458752000 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0005904222621524494, |
|
"loss": 2.8607, |
|
"theoretical_loss": 3.9485319770934355, |
|
"tokens_seen": 462028800 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0005895947607338756, |
|
"loss": 2.8461, |
|
"theoretical_loss": 3.945488352063391, |
|
"tokens_seen": 465305600 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0005887672593153017, |
|
"loss": 2.8307, |
|
"theoretical_loss": 3.942472039612926, |
|
"tokens_seen": 468582400 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0005879397578967278, |
|
"loss": 2.8269, |
|
"theoretical_loss": 3.939482606204863, |
|
"tokens_seen": 471859200 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0005871122564781539, |
|
"loss": 2.8407, |
|
"theoretical_loss": 3.936519628133466, |
|
"tokens_seen": 475136000 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.00058628475505958, |
|
"loss": 2.8754, |
|
"theoretical_loss": 3.9335826912355114, |
|
"tokens_seen": 478412800 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0005854572536410061, |
|
"loss": 2.9168, |
|
"theoretical_loss": 3.93067139061177, |
|
"tokens_seen": 481689600 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0005846297522224324, |
|
"loss": 2.8452, |
|
"theoretical_loss": 3.927785330358441, |
|
"tokens_seen": 484966400 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0005838022508038585, |
|
"loss": 2.8359, |
|
"theoretical_loss": 3.9249241233081333, |
|
"tokens_seen": 488243200 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"objective/train/avg_token_score": 0.004644907079637051, |
|
"objective/train/avg_weight": 0.995009183883667, |
|
"objective/train/docs_used": 287192, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 3.014026165008545, |
|
"objective/train/std_weight": 0.02483248896896839, |
|
"objective/train/theoretical_loss": 3.92208739077998, |
|
"objective/train/tokens_used": 511980000, |
|
"theoretical_loss": 3.92208739077998, |
|
"tokens_seen": 491520000 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0005829747493852846, |
|
"loss": 2.8155, |
|
"theoretical_loss": 3.92208739077998, |
|
"tokens_seen": 491520000 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0005821472479667107, |
|
"loss": 2.8248, |
|
"theoretical_loss": 3.919274762338519, |
|
"tokens_seen": 494796800 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.000581319746548137, |
|
"loss": 2.8481, |
|
"theoretical_loss": 3.9164858755609613, |
|
"tokens_seen": 498073600 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0005804922451295631, |
|
"loss": 2.8296, |
|
"theoretical_loss": 3.9137203758125176, |
|
"tokens_seen": 501350400 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0005796647437109892, |
|
"loss": 2.7929, |
|
"theoretical_loss": 3.910977916029439, |
|
"tokens_seen": 504627200 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0005788372422924154, |
|
"loss": 2.8029, |
|
"theoretical_loss": 3.908258156509472, |
|
"tokens_seen": 507904000 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0005780097408738415, |
|
"loss": 2.7949, |
|
"theoretical_loss": 3.905560764709417, |
|
"tokens_seen": 511180800 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0005771822394552677, |
|
"loss": 2.7728, |
|
"theoretical_loss": 3.9028854150495143, |
|
"tokens_seen": 514457600 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0005763547380366938, |
|
"loss": 2.7991, |
|
"theoretical_loss": 3.9002317887243834, |
|
"tokens_seen": 517734400 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0005755272366181199, |
|
"loss": 2.7956, |
|
"theoretical_loss": 3.897599573520247, |
|
"tokens_seen": 521011200 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0005746997351995461, |
|
"loss": 2.8261, |
|
"theoretical_loss": 3.8949884636382106, |
|
"tokens_seen": 524288000 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0005738722337809722, |
|
"loss": 2.8118, |
|
"theoretical_loss": 3.892398159523345, |
|
"tokens_seen": 527564800 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0005730447323623984, |
|
"loss": 2.8643, |
|
"theoretical_loss": 3.889828367699349, |
|
"tokens_seen": 530841600 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0005722172309438245, |
|
"loss": 2.8021, |
|
"theoretical_loss": 3.8872788006085894, |
|
"tokens_seen": 534118400 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0005713897295252506, |
|
"loss": 2.8285, |
|
"theoretical_loss": 3.8847491764572926, |
|
"tokens_seen": 537395200 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0005705622281066767, |
|
"loss": 2.8119, |
|
"theoretical_loss": 3.882239219065708, |
|
"tokens_seen": 540672000 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0005697347266881029, |
|
"loss": 2.8372, |
|
"theoretical_loss": 3.879748657723039, |
|
"tokens_seen": 543948800 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0005689072252695291, |
|
"loss": 2.8645, |
|
"theoretical_loss": 3.8772772270469824, |
|
"tokens_seen": 547225600 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0005680797238509552, |
|
"loss": 2.8276, |
|
"theoretical_loss": 3.8748246668476827, |
|
"tokens_seen": 550502400 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0005672522224323813, |
|
"loss": 2.8475, |
|
"theoretical_loss": 3.8723907219959486, |
|
"tokens_seen": 553779200 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0005664247210138074, |
|
"loss": 2.8644, |
|
"theoretical_loss": 3.869975142295573, |
|
"tokens_seen": 557056000 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0005655972195952335, |
|
"loss": 2.9145, |
|
"theoretical_loss": 3.8675776823595998, |
|
"tokens_seen": 560332800 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0005647697181766596, |
|
"loss": 2.8816, |
|
"theoretical_loss": 3.8651981014904027, |
|
"tokens_seen": 563609600 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0005639422167580859, |
|
"loss": 2.8532, |
|
"theoretical_loss": 3.8628361635634265, |
|
"tokens_seen": 566886400 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0005631312653678835, |
|
"loss": 2.8788, |
|
"theoretical_loss": 3.8604916369144666, |
|
"tokens_seen": 570163200 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0005623037639493096, |
|
"loss": 2.8551, |
|
"theoretical_loss": 3.858164294230354, |
|
"tokens_seen": 573440000 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0005614762625307357, |
|
"loss": 2.8309, |
|
"theoretical_loss": 3.85585391244293, |
|
"tokens_seen": 576716800 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0005606487611121619, |
|
"loss": 2.8414, |
|
"theoretical_loss": 3.8535602726261864, |
|
"tokens_seen": 579993600 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.000559821259693588, |
|
"loss": 2.8235, |
|
"theoretical_loss": 3.851283159896468, |
|
"tokens_seen": 583270400 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0005589937582750142, |
|
"loss": 2.8092, |
|
"theoretical_loss": 3.8490223633156173, |
|
"tokens_seen": 586547200 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0005581662568564403, |
|
"loss": 2.7915, |
|
"theoretical_loss": 3.846777675796974, |
|
"tokens_seen": 589824000 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0005573387554378664, |
|
"loss": 2.7981, |
|
"theoretical_loss": 3.844548894014116, |
|
"tokens_seen": 593100800 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0005565112540192926, |
|
"loss": 2.8022, |
|
"theoretical_loss": 3.8423358183122582, |
|
"tokens_seen": 596377600 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0005556837526007187, |
|
"loss": 2.7754, |
|
"theoretical_loss": 3.840138252622208, |
|
"tokens_seen": 599654400 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0005548562511821448, |
|
"loss": 2.8037, |
|
"theoretical_loss": 3.837956004376799, |
|
"tokens_seen": 602931200 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.000554028749763571, |
|
"loss": 2.7709, |
|
"theoretical_loss": 3.8357888844297094, |
|
"tokens_seen": 606208000 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0005532012483449971, |
|
"loss": 2.7927, |
|
"theoretical_loss": 3.8336367069765958, |
|
"tokens_seen": 609484800 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0005523737469264233, |
|
"loss": 2.7451, |
|
"theoretical_loss": 3.8314992894784536, |
|
"tokens_seen": 612761600 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0005515462455078494, |
|
"loss": 2.7896, |
|
"theoretical_loss": 3.829376452587134, |
|
"tokens_seen": 616038400 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0005507187440892756, |
|
"loss": 2.8047, |
|
"theoretical_loss": 3.827268020072948, |
|
"tokens_seen": 619315200 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0005498912426707017, |
|
"loss": 2.8391, |
|
"theoretical_loss": 3.8251738187542843, |
|
"tokens_seen": 622592000 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0005490637412521278, |
|
"loss": 2.8195, |
|
"theoretical_loss": 3.8230936784291787, |
|
"tokens_seen": 625868800 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0005482362398335541, |
|
"loss": 2.8531, |
|
"theoretical_loss": 3.8210274318087656, |
|
"tokens_seen": 629145600 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0005474087384149802, |
|
"loss": 2.8001, |
|
"theoretical_loss": 3.818974914452557, |
|
"tokens_seen": 632422400 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0005465812369964063, |
|
"loss": 2.7996, |
|
"theoretical_loss": 3.8169359647054835, |
|
"tokens_seen": 635699200 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0005457537355778324, |
|
"loss": 2.7817, |
|
"theoretical_loss": 3.8149104236366433, |
|
"tokens_seen": 638976000 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0005449262341592585, |
|
"loss": 2.7613, |
|
"theoretical_loss": 3.8128981349797098, |
|
"tokens_seen": 642252800 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0005440987327406847, |
|
"loss": 2.7851, |
|
"theoretical_loss": 3.8108989450749293, |
|
"tokens_seen": 645529600 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0005432712313221109, |
|
"loss": 2.7738, |
|
"theoretical_loss": 3.8089127028126764, |
|
"tokens_seen": 648806400 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.000542443729903537, |
|
"loss": 2.8114, |
|
"theoretical_loss": 3.8069392595785083, |
|
"tokens_seen": 652083200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.5265375629586004, |
|
"debugging/distinct-1-grams": 0.7435820408094715, |
|
"debugging/distinct-2-grams": 0.9558103821233092, |
|
"debugging/entropy-1-grams": 5.931434510687563, |
|
"debugging/entropy-2-grams": 6.886416755326388, |
|
"debugging/length": 521.9230769230769, |
|
"debugging/num_segments": 13, |
|
"epoch": 0.23, |
|
"objective/train/avg_token_score": 0.022742915898561478, |
|
"objective/train/avg_weight": 0.9755392670631409, |
|
"objective/train/docs_used": 379091, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 3.0632386207580566, |
|
"objective/train/std_weight": 0.08433963358402252, |
|
"objective/train/theoretical_loss": 3.804978469199669, |
|
"objective/train/tokens_used": 675820000, |
|
"theoretical_loss": 3.804978469199669, |
|
"tokens_seen": 655360000 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0005416162284849631, |
|
"loss": 2.805, |
|
"theoretical_loss": 3.804978469199669, |
|
"tokens_seen": 655360000 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0005407887270663892, |
|
"loss": 2.8212, |
|
"theoretical_loss": 3.803030187893005, |
|
"tokens_seen": 658636800 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0005399612256478154, |
|
"loss": 2.82, |
|
"theoretical_loss": 3.8010942742142415, |
|
"tokens_seen": 661913600 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0005391337242292415, |
|
"loss": 2.8474, |
|
"theoretical_loss": 3.799170589008585, |
|
"tokens_seen": 665190400 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0005383062228106677, |
|
"loss": 2.8322, |
|
"theoretical_loss": 3.7972589953626006, |
|
"tokens_seen": 668467200 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0005374787213920938, |
|
"loss": 2.8432, |
|
"theoretical_loss": 3.795359358557337, |
|
"tokens_seen": 671744000 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0005366512199735199, |
|
"loss": 2.7784, |
|
"theoretical_loss": 3.79347154602265, |
|
"tokens_seen": 675020800 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0005358237185549461, |
|
"loss": 2.824, |
|
"theoretical_loss": 3.7915954272926955, |
|
"tokens_seen": 678297600 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0005349962171363722, |
|
"loss": 2.7824, |
|
"theoretical_loss": 3.789730873962557, |
|
"tokens_seen": 681574400 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0005341687157177983, |
|
"loss": 2.7185, |
|
"theoretical_loss": 3.787877759645963, |
|
"tokens_seen": 684851200 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0005333412142992244, |
|
"loss": 2.7511, |
|
"theoretical_loss": 3.7860359599340776, |
|
"tokens_seen": 688128000 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0005325137128806506, |
|
"loss": 2.791, |
|
"theoretical_loss": 3.784205352355321, |
|
"tokens_seen": 691404800 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0005316862114620768, |
|
"loss": 2.8214, |
|
"theoretical_loss": 3.782385816336189, |
|
"tokens_seen": 694681600 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0005308587100435029, |
|
"loss": 2.7788, |
|
"theoretical_loss": 3.7805772331630516, |
|
"tokens_seen": 697958400 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0005300312086249291, |
|
"loss": 2.7553, |
|
"theoretical_loss": 3.7787794859448898, |
|
"tokens_seen": 701235200 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0005292037072063552, |
|
"loss": 2.7382, |
|
"theoretical_loss": 3.7769924595769546, |
|
"tokens_seen": 704512000 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0005283762057877813, |
|
"loss": 2.7255, |
|
"theoretical_loss": 3.7752160407053115, |
|
"tokens_seen": 707788800 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0005275487043692075, |
|
"loss": 2.7101, |
|
"theoretical_loss": 3.7734501176922493, |
|
"tokens_seen": 711065600 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0005267212029506337, |
|
"loss": 2.7514, |
|
"theoretical_loss": 3.7716945805825337, |
|
"tokens_seen": 714342400 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0005258937015320598, |
|
"loss": 2.7886, |
|
"theoretical_loss": 3.7699493210704667, |
|
"tokens_seen": 717619200 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0005250662001134859, |
|
"loss": 2.7644, |
|
"theoretical_loss": 3.7682142324677455, |
|
"tokens_seen": 720896000 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.000524238698694912, |
|
"loss": 2.8249, |
|
"theoretical_loss": 3.7664892096720886, |
|
"tokens_seen": 724172800 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0005234111972763382, |
|
"loss": 2.7615, |
|
"theoretical_loss": 3.7647741491366067, |
|
"tokens_seen": 727449600 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0005225836958577644, |
|
"loss": 2.7764, |
|
"theoretical_loss": 3.7630689488399027, |
|
"tokens_seen": 730726400 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0005217561944391905, |
|
"loss": 2.7431, |
|
"theoretical_loss": 3.7613735082568764, |
|
"tokens_seen": 734003200 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0005209286930206166, |
|
"loss": 2.7793, |
|
"theoretical_loss": 3.759687728330217, |
|
"tokens_seen": 737280000 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0005201011916020427, |
|
"loss": 2.7728, |
|
"theoretical_loss": 3.75801151144256, |
|
"tokens_seen": 740556800 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0005192736901834689, |
|
"loss": 2.7565, |
|
"theoretical_loss": 3.756344761389295, |
|
"tokens_seen": 743833600 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.000518446188764895, |
|
"loss": 2.7275, |
|
"theoretical_loss": 3.754687383352003, |
|
"tokens_seen": 747110400 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0005176186873463211, |
|
"loss": 2.7306, |
|
"theoretical_loss": 3.7530392838725097, |
|
"tokens_seen": 750387200 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0005167911859277473, |
|
"loss": 2.7465, |
|
"theoretical_loss": 3.751400370827529, |
|
"tokens_seen": 753664000 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0005159636845091734, |
|
"loss": 2.7684, |
|
"theoretical_loss": 3.749770553403895, |
|
"tokens_seen": 756940800 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0005151361830905996, |
|
"loss": 2.7327, |
|
"theoretical_loss": 3.748149742074355, |
|
"tokens_seen": 760217600 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0005143086816720257, |
|
"loss": 2.7618, |
|
"theoretical_loss": 3.746537848573908, |
|
"tokens_seen": 763494400 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0005134811802534518, |
|
"loss": 2.7765, |
|
"theoretical_loss": 3.744934785876686, |
|
"tokens_seen": 766771200 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0005126536788348779, |
|
"loss": 2.7845, |
|
"theoretical_loss": 3.7433404681733475, |
|
"tokens_seen": 770048000 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0005118261774163041, |
|
"loss": 2.7901, |
|
"theoretical_loss": 3.7417548108489846, |
|
"tokens_seen": 773324800 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0005109986759977303, |
|
"loss": 2.7636, |
|
"theoretical_loss": 3.740177730461517, |
|
"tokens_seen": 776601600 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0005101711745791564, |
|
"loss": 2.8129, |
|
"theoretical_loss": 3.73860914472057, |
|
"tokens_seen": 779878400 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0005093436731605825, |
|
"loss": 2.7786, |
|
"theoretical_loss": 3.7370489724668197, |
|
"tokens_seen": 783155200 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0005085161717420087, |
|
"loss": 2.7891, |
|
"theoretical_loss": 3.735497133651788, |
|
"tokens_seen": 786432000 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0005076886703234348, |
|
"loss": 2.7787, |
|
"theoretical_loss": 3.733953549318091, |
|
"tokens_seen": 789708800 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.000506861168904861, |
|
"loss": 2.7923, |
|
"theoretical_loss": 3.7324181415801094, |
|
"tokens_seen": 792985600 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0005060336674862872, |
|
"loss": 2.7873, |
|
"theoretical_loss": 3.7308908336050814, |
|
"tokens_seen": 796262400 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0005052061660677133, |
|
"loss": 2.772, |
|
"theoretical_loss": 3.729371549594614, |
|
"tokens_seen": 799539200 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0005043786646491394, |
|
"loss": 2.7862, |
|
"theoretical_loss": 3.7278602147665776, |
|
"tokens_seen": 802816000 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0005035511632305655, |
|
"loss": 2.7841, |
|
"theoretical_loss": 3.726356755337407, |
|
"tokens_seen": 806092800 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0005027236618119917, |
|
"loss": 2.7735, |
|
"theoretical_loss": 3.724861098504767, |
|
"tokens_seen": 809369600 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0005018961603934179, |
|
"loss": 2.7389, |
|
"theoretical_loss": 3.7233731724305974, |
|
"tokens_seen": 812646400 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.000501068658974844, |
|
"loss": 2.7629, |
|
"theoretical_loss": 3.7218929062245105, |
|
"tokens_seen": 815923200 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"objective/train/avg_token_score": 0.009068925864994526, |
|
"objective/train/avg_weight": 0.9902427196502686, |
|
"objective/train/docs_used": 471128, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.6760623455047607, |
|
"objective/train/std_weight": 0.06257461756467819, |
|
"objective/train/theoretical_loss": 3.7204202299275475, |
|
"objective/train/tokens_used": 839660000, |
|
"theoretical_loss": 3.7204202299275475, |
|
"tokens_seen": 819200000 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0005002411575562701, |
|
"loss": 2.7057, |
|
"theoretical_loss": 3.7204202299275475, |
|
"tokens_seen": 819200000 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0004994136561376962, |
|
"loss": 2.7306, |
|
"theoretical_loss": 3.7189550744962707, |
|
"tokens_seen": 822476800 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0004985861547191224, |
|
"loss": 2.7091, |
|
"theoretical_loss": 3.717497371787192, |
|
"tokens_seen": 825753600 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0004977586533005485, |
|
"loss": 2.7075, |
|
"theoretical_loss": 3.7160470545415274, |
|
"tokens_seen": 829030400 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0004969311518819746, |
|
"loss": 2.7036, |
|
"theoretical_loss": 3.714604056370267, |
|
"tokens_seen": 832307200 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0004961036504634008, |
|
"loss": 2.716, |
|
"theoretical_loss": 3.713168311739558, |
|
"tokens_seen": 835584000 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0004952761490448269, |
|
"loss": 2.7011, |
|
"theoretical_loss": 3.7117397559563843, |
|
"tokens_seen": 838860800 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0004944486476262531, |
|
"loss": 2.6838, |
|
"theoretical_loss": 3.710318325154545, |
|
"tokens_seen": 842137600 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0004936211462076792, |
|
"loss": 2.7304, |
|
"theoretical_loss": 3.7089039562809223, |
|
"tokens_seen": 845414400 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0004927936447891053, |
|
"loss": 2.7027, |
|
"theoretical_loss": 3.7074965870820193, |
|
"tokens_seen": 848691200 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0004919661433705314, |
|
"loss": 2.7073, |
|
"theoretical_loss": 3.7060961560907857, |
|
"tokens_seen": 851968000 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0004911386419519576, |
|
"loss": 2.7664, |
|
"theoretical_loss": 3.7047026026137, |
|
"tokens_seen": 855244800 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0004903111405333838, |
|
"loss": 2.7282, |
|
"theoretical_loss": 3.7033158667181154, |
|
"tokens_seen": 858521600 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0004894836391148099, |
|
"loss": 2.7546, |
|
"theoretical_loss": 3.701935889219863, |
|
"tokens_seen": 861798400 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.000488656137696236, |
|
"loss": 2.7233, |
|
"theoretical_loss": 3.7005626116710966, |
|
"tokens_seen": 865075200 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00048782863627766217, |
|
"loss": 2.7292, |
|
"theoretical_loss": 3.69919597634839, |
|
"tokens_seen": 868352000 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0004870011348590883, |
|
"loss": 2.7075, |
|
"theoretical_loss": 3.6978359262410603, |
|
"tokens_seen": 871628800 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0004861736334405145, |
|
"loss": 2.7358, |
|
"theoretical_loss": 3.6964824050397276, |
|
"tokens_seen": 874905600 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00048534613202194063, |
|
"loss": 2.676, |
|
"theoretical_loss": 3.6951353571251015, |
|
"tokens_seen": 878182400 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00048451863060336675, |
|
"loss": 2.6779, |
|
"theoretical_loss": 3.693794727556988, |
|
"tokens_seen": 881459200 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.00048369112918479287, |
|
"loss": 2.6825, |
|
"theoretical_loss": 3.692460462063506, |
|
"tokens_seen": 884736000 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.000482863627766219, |
|
"loss": 2.7369, |
|
"theoretical_loss": 3.691132507030521, |
|
"tokens_seen": 888012800 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.00048203612634764516, |
|
"loss": 2.7514, |
|
"theoretical_loss": 3.6898108094912816, |
|
"tokens_seen": 891289600 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0004812086249290713, |
|
"loss": 2.7082, |
|
"theoretical_loss": 3.6884953171162556, |
|
"tokens_seen": 894566400 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.00048038112351049745, |
|
"loss": 2.7144, |
|
"theoretical_loss": 3.6871859782031624, |
|
"tokens_seen": 897843200 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0004795536220919236, |
|
"loss": 2.7252, |
|
"theoretical_loss": 3.685882741667202, |
|
"tokens_seen": 901120000 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.00047872612067334974, |
|
"loss": 2.7262, |
|
"theoretical_loss": 3.684585557031461, |
|
"tokens_seen": 904396800 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.00047789861925477586, |
|
"loss": 2.7381, |
|
"theoretical_loss": 3.6832943744175126, |
|
"tokens_seen": 907673600 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.000477071117836202, |
|
"loss": 2.7141, |
|
"theoretical_loss": 3.682009144536188, |
|
"tokens_seen": 910950400 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.0004762436164176281, |
|
"loss": 2.7418, |
|
"theoretical_loss": 3.680729818678526, |
|
"tokens_seen": 914227200 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.0004754161149990543, |
|
"loss": 2.7204, |
|
"theoretical_loss": 3.6794563487068936, |
|
"tokens_seen": 917504000 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00047458861358048044, |
|
"loss": 2.7297, |
|
"theoretical_loss": 3.6781886870462692, |
|
"tokens_seen": 920780800 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00047376111216190656, |
|
"loss": 2.7465, |
|
"theoretical_loss": 3.676926786675698, |
|
"tokens_seen": 924057600 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.0004729336107433327, |
|
"loss": 2.6863, |
|
"theoretical_loss": 3.6756706011198963, |
|
"tokens_seen": 927334400 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.0004721061093247588, |
|
"loss": 2.6538, |
|
"theoretical_loss": 3.6744200844410217, |
|
"tokens_seen": 930611200 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00047127860790618497, |
|
"loss": 2.6555, |
|
"theoretical_loss": 3.6731751912305914, |
|
"tokens_seen": 933888000 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00047045110648761114, |
|
"loss": 2.688, |
|
"theoretical_loss": 3.671935876601547, |
|
"tokens_seen": 937164800 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.00046962360506903726, |
|
"loss": 2.6286, |
|
"theoretical_loss": 3.6707020961804715, |
|
"tokens_seen": 940441600 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.00046879610365046343, |
|
"loss": 2.6828, |
|
"theoretical_loss": 3.6694738060999468, |
|
"tokens_seen": 943718400 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.00046796860223188955, |
|
"loss": 2.7023, |
|
"theoretical_loss": 3.668250962991049, |
|
"tokens_seen": 946995200 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.00046714110081331567, |
|
"loss": 2.7018, |
|
"theoretical_loss": 3.667033523975983, |
|
"tokens_seen": 950272000 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0004663135993947418, |
|
"loss": 2.6876, |
|
"theoretical_loss": 3.66582144666085, |
|
"tokens_seen": 953548800 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0004654860979761679, |
|
"loss": 2.687, |
|
"theoretical_loss": 3.664614689128546, |
|
"tokens_seen": 956825600 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.00046465859655759413, |
|
"loss": 2.6865, |
|
"theoretical_loss": 3.6634132099317886, |
|
"tokens_seen": 960102400 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.00046383109513902025, |
|
"loss": 2.6943, |
|
"theoretical_loss": 3.662216968086267, |
|
"tokens_seen": 963379200 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00046300359372044637, |
|
"loss": 2.7232, |
|
"theoretical_loss": 3.6610259230639217, |
|
"tokens_seen": 966656000 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0004621760923018725, |
|
"loss": 2.7559, |
|
"theoretical_loss": 3.659840034786333, |
|
"tokens_seen": 969932800 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00046134859088329866, |
|
"loss": 2.8156, |
|
"theoretical_loss": 3.6586592636182376, |
|
"tokens_seen": 973209600 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0004605210894647248, |
|
"loss": 2.8084, |
|
"theoretical_loss": 3.6574835703611566, |
|
"tokens_seen": 976486400 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00045969358804615095, |
|
"loss": 2.793, |
|
"theoretical_loss": 3.6563129162471313, |
|
"tokens_seen": 979763200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.4286046663919377, |
|
"debugging/distinct-1-grams": 0.8147567798871364, |
|
"debugging/distinct-2-grams": 0.9823269374342457, |
|
"debugging/entropy-1-grams": 6.1671920556004824, |
|
"debugging/entropy-2-grams": 6.947028138756313, |
|
"debugging/length": 477.53333333333336, |
|
"debugging/num_segments": 15, |
|
"epoch": 0.35, |
|
"objective/train/avg_token_score": 0.020611366257071495, |
|
"objective/train/avg_weight": 0.9778223633766174, |
|
"objective/train/docs_used": 560408, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.926975965499878, |
|
"objective/train/std_weight": 0.09140268713235855, |
|
"objective/train/theoretical_loss": 3.6551472629325787, |
|
"objective/train/tokens_used": 1003500000, |
|
"theoretical_loss": 3.6551472629325787, |
|
"tokens_seen": 983040000 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00045886608662757707, |
|
"loss": 2.7908, |
|
"theoretical_loss": 3.6551472629325787, |
|
"tokens_seen": 983040000 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00045803858520900324, |
|
"loss": 2.7934, |
|
"theoretical_loss": 3.653986572492247, |
|
"tokens_seen": 986316800 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00045721108379042936, |
|
"loss": 2.7583, |
|
"theoretical_loss": 3.65283080741328, |
|
"tokens_seen": 989593600 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00045640013240022694, |
|
"loss": 2.7744, |
|
"theoretical_loss": 3.6516799305893866, |
|
"tokens_seen": 992870400 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.00045557263098165306, |
|
"loss": 2.7683, |
|
"theoretical_loss": 3.6505339053151076, |
|
"tokens_seen": 996147200 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0004547451295630793, |
|
"loss": 2.7755, |
|
"theoretical_loss": 3.649392695280186, |
|
"tokens_seen": 999424000 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0004539176281445054, |
|
"loss": 2.7407, |
|
"theoretical_loss": 3.6482562645640337, |
|
"tokens_seen": 1002700800 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0004530901267259315, |
|
"loss": 2.7456, |
|
"theoretical_loss": 3.6471245776302883, |
|
"tokens_seen": 1005977600 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.00045226262530735764, |
|
"loss": 2.7827, |
|
"theoretical_loss": 3.6459975993214724, |
|
"tokens_seen": 1009254400 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0004514351238887838, |
|
"loss": 2.7414, |
|
"theoretical_loss": 3.6448752948537377, |
|
"tokens_seen": 1012531200 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.00045060762247020993, |
|
"loss": 2.7774, |
|
"theoretical_loss": 3.6437576298116996, |
|
"tokens_seen": 1015808000 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.00044978012105163605, |
|
"loss": 2.7705, |
|
"theoretical_loss": 3.6426445701433607, |
|
"tokens_seen": 1019084800 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0004489526196330623, |
|
"loss": 2.79, |
|
"theoretical_loss": 3.6415360821551226, |
|
"tokens_seen": 1022361600 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0004481251182144884, |
|
"loss": 2.7458, |
|
"theoretical_loss": 3.6404321325068754, |
|
"tokens_seen": 1025638400 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0004472976167959145, |
|
"loss": 2.7668, |
|
"theoretical_loss": 3.639332688207178, |
|
"tokens_seen": 1028915200 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00044647011537734063, |
|
"loss": 2.7474, |
|
"theoretical_loss": 3.6382377166085096, |
|
"tokens_seen": 1032192000 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00044564261395876675, |
|
"loss": 2.7098, |
|
"theoretical_loss": 3.6371471854026147, |
|
"tokens_seen": 1035468800 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00044481511254019287, |
|
"loss": 2.7161, |
|
"theoretical_loss": 3.6360610626159087, |
|
"tokens_seen": 1038745600 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0004439876111216191, |
|
"loss": 2.6831, |
|
"theoretical_loss": 3.634979316604973, |
|
"tokens_seen": 1042022400 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0004431601097030452, |
|
"loss": 2.6983, |
|
"theoretical_loss": 3.6339019160521198, |
|
"tokens_seen": 1045299200 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00044233260828447133, |
|
"loss": 2.6901, |
|
"theoretical_loss": 3.632828829961029, |
|
"tokens_seen": 1048576000 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.00044150510686589745, |
|
"loss": 2.7065, |
|
"theoretical_loss": 3.631760027652461, |
|
"tokens_seen": 1051852800 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0004406776054473236, |
|
"loss": 2.7373, |
|
"theoretical_loss": 3.630695478760034, |
|
"tokens_seen": 1055129600 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.00043985010402874974, |
|
"loss": 2.709, |
|
"theoretical_loss": 3.6296351532260767, |
|
"tokens_seen": 1058406400 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0004390226026101759, |
|
"loss": 2.7033, |
|
"theoretical_loss": 3.6285790212975435, |
|
"tokens_seen": 1061683200 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0004381951011916021, |
|
"loss": 2.6803, |
|
"theoretical_loss": 3.6275270535220008, |
|
"tokens_seen": 1064960000 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0004373675997730282, |
|
"loss": 2.7044, |
|
"theoretical_loss": 3.626479220743673, |
|
"tokens_seen": 1068236800 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0004365400983544543, |
|
"loss": 2.7078, |
|
"theoretical_loss": 3.6254354940995586, |
|
"tokens_seen": 1071513600 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.00043571259693588044, |
|
"loss": 2.7014, |
|
"theoretical_loss": 3.624395845015602, |
|
"tokens_seen": 1074790400 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.00043488509551730656, |
|
"loss": 2.6599, |
|
"theoretical_loss": 3.6233602452029348, |
|
"tokens_seen": 1078067200 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0004340575940987327, |
|
"loss": 2.6797, |
|
"theoretical_loss": 3.6223286666541683, |
|
"tokens_seen": 1081344000 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0004332300926801589, |
|
"loss": 2.7048, |
|
"theoretical_loss": 3.621301081639753, |
|
"tokens_seen": 1084620800 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.000432402591261585, |
|
"loss": 2.7439, |
|
"theoretical_loss": 3.6202774627043923, |
|
"tokens_seen": 1087897600 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.00043157508984301114, |
|
"loss": 2.6985, |
|
"theoretical_loss": 3.619257782663513, |
|
"tokens_seen": 1091174400 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0004307475884244373, |
|
"loss": 2.7095, |
|
"theoretical_loss": 3.618242014599793, |
|
"tokens_seen": 1094451200 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.00042992008700586343, |
|
"loss": 2.6983, |
|
"theoretical_loss": 3.617230131859743, |
|
"tokens_seen": 1097728000 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.00042909258558728955, |
|
"loss": 2.6823, |
|
"theoretical_loss": 3.6162221080503416, |
|
"tokens_seen": 1101004800 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0004282650841687157, |
|
"loss": 2.6665, |
|
"theoretical_loss": 3.615217917035726, |
|
"tokens_seen": 1104281600 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0004274375827501419, |
|
"loss": 2.6981, |
|
"theoretical_loss": 3.614217532933929, |
|
"tokens_seen": 1107558400 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.000426610081331568, |
|
"loss": 2.6861, |
|
"theoretical_loss": 3.6132209301136715, |
|
"tokens_seen": 1110835200 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00042578257991299413, |
|
"loss": 2.6717, |
|
"theoretical_loss": 3.612228083191205, |
|
"tokens_seen": 1114112000 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00042495507849442025, |
|
"loss": 2.7291, |
|
"theoretical_loss": 3.611238967027199, |
|
"tokens_seen": 1117388800 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00042412757707584636, |
|
"loss": 2.7692, |
|
"theoretical_loss": 3.610253556723679, |
|
"tokens_seen": 1120665600 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0004233000756572726, |
|
"loss": 2.7318, |
|
"theoretical_loss": 3.609271827621014, |
|
"tokens_seen": 1123942400 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0004224725742386987, |
|
"loss": 2.7372, |
|
"theoretical_loss": 3.6082937552949463, |
|
"tokens_seen": 1127219200 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00042164507282012483, |
|
"loss": 2.7052, |
|
"theoretical_loss": 3.607319315553669, |
|
"tokens_seen": 1130496000 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00042081757140155095, |
|
"loss": 2.668, |
|
"theoretical_loss": 3.6063484844349456, |
|
"tokens_seen": 1133772800 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.0004199900699829771, |
|
"loss": 2.6911, |
|
"theoretical_loss": 3.605381238203279, |
|
"tokens_seen": 1137049600 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00041916256856440324, |
|
"loss": 2.7014, |
|
"theoretical_loss": 3.604417553347117, |
|
"tokens_seen": 1140326400 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.0004183350671458294, |
|
"loss": 2.6926, |
|
"theoretical_loss": 3.603457406576106, |
|
"tokens_seen": 1143603200 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"objective/train/avg_token_score": 0.022877871990203857, |
|
"objective/train/avg_weight": 0.97539883852005, |
|
"objective/train/docs_used": 649861, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.6508138179779053, |
|
"objective/train/std_weight": 0.10265526175498962, |
|
"objective/train/theoretical_loss": 3.602500774818379, |
|
"objective/train/tokens_used": 1167340000, |
|
"theoretical_loss": 3.602500774818379, |
|
"tokens_seen": 1146880000 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00041750756572725553, |
|
"loss": 2.6792, |
|
"theoretical_loss": 3.602500774818379, |
|
"tokens_seen": 1146880000 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.0004166800643086817, |
|
"loss": 2.7197, |
|
"theoretical_loss": 3.601547635217892, |
|
"tokens_seen": 1150156800 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.0004158525628901078, |
|
"loss": 2.7201, |
|
"theoretical_loss": 3.6005979651317976, |
|
"tokens_seen": 1153433600 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00041502506147153394, |
|
"loss": 2.7286, |
|
"theoretical_loss": 3.599651742127855, |
|
"tokens_seen": 1156710400 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00041419756005296005, |
|
"loss": 2.681, |
|
"theoretical_loss": 3.5987089439818805, |
|
"tokens_seen": 1159987200 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00041337005863438617, |
|
"loss": 2.6621, |
|
"theoretical_loss": 3.5977695486752426, |
|
"tokens_seen": 1163264000 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0004125425572158124, |
|
"loss": 2.6926, |
|
"theoretical_loss": 3.596833534392379, |
|
"tokens_seen": 1166540800 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0004117150557972385, |
|
"loss": 2.7096, |
|
"theoretical_loss": 3.595900879518368, |
|
"tokens_seen": 1169817600 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00041088755437866464, |
|
"loss": 2.7241, |
|
"theoretical_loss": 3.594971562636521, |
|
"tokens_seen": 1173094400 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00041006005296009075, |
|
"loss": 2.6816, |
|
"theoretical_loss": 3.5940455625260226, |
|
"tokens_seen": 1176371200 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0004092325515415169, |
|
"loss": 2.6928, |
|
"theoretical_loss": 3.5931228581595938, |
|
"tokens_seen": 1179648000 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00040840505012294305, |
|
"loss": 2.6899, |
|
"theoretical_loss": 3.5922034287011995, |
|
"tokens_seen": 1182924800 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0004075775487043692, |
|
"loss": 2.6567, |
|
"theoretical_loss": 3.5912872535037828, |
|
"tokens_seen": 1186201600 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0004067500472857954, |
|
"loss": 2.739, |
|
"theoretical_loss": 3.590374312107035, |
|
"tokens_seen": 1189478400 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.0004059225458672215, |
|
"loss": 2.724, |
|
"theoretical_loss": 3.5894645842351993, |
|
"tokens_seen": 1192755200 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.0004050950444486476, |
|
"loss": 2.7097, |
|
"theoretical_loss": 3.588558049794902, |
|
"tokens_seen": 1196032000 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.00040426754303007374, |
|
"loss": 2.689, |
|
"theoretical_loss": 3.5876546888730187, |
|
"tokens_seen": 1199308800 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.00040344004161149986, |
|
"loss": 2.7083, |
|
"theoretical_loss": 3.5867544817345713, |
|
"tokens_seen": 1202585600 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.0004026125401929261, |
|
"loss": 2.7336, |
|
"theoretical_loss": 3.585857408820652, |
|
"tokens_seen": 1205862400 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.00040180158880272367, |
|
"loss": 2.7425, |
|
"theoretical_loss": 3.58496345074638, |
|
"tokens_seen": 1209139200 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.0004009740873841498, |
|
"loss": 2.7536, |
|
"theoretical_loss": 3.5840725882988873, |
|
"tokens_seen": 1212416000 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.00040014658596557596, |
|
"loss": 2.7189, |
|
"theoretical_loss": 3.5831848024353317, |
|
"tokens_seen": 1215692800 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0003993190845470021, |
|
"loss": 2.7442, |
|
"theoretical_loss": 3.5823000742809374, |
|
"tokens_seen": 1218969600 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0003984915831284282, |
|
"loss": 2.7158, |
|
"theoretical_loss": 3.5814183851270673, |
|
"tokens_seen": 1222246400 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0003976640817098543, |
|
"loss": 2.7064, |
|
"theoretical_loss": 3.5805397164293167, |
|
"tokens_seen": 1225523200 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00039683658029128054, |
|
"loss": 2.6876, |
|
"theoretical_loss": 3.5796640498056407, |
|
"tokens_seen": 1228800000 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00039600907887270666, |
|
"loss": 2.696, |
|
"theoretical_loss": 3.5787913670345013, |
|
"tokens_seen": 1232076800 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00039519812748250424, |
|
"loss": 2.6741, |
|
"theoretical_loss": 3.577921650053045, |
|
"tokens_seen": 1235353600 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00039437062606393036, |
|
"loss": 2.6706, |
|
"theoretical_loss": 3.577054880955303, |
|
"tokens_seen": 1238630400 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00039354312464535653, |
|
"loss": 2.657, |
|
"theoretical_loss": 3.5761910419904193, |
|
"tokens_seen": 1241907200 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00039271562322678265, |
|
"loss": 2.7193, |
|
"theoretical_loss": 3.5753301155609014, |
|
"tokens_seen": 1245184000 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.0003918881218082088, |
|
"loss": 2.6606, |
|
"theoretical_loss": 3.574472084220896, |
|
"tokens_seen": 1248460800 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00039106062038963494, |
|
"loss": 2.6497, |
|
"theoretical_loss": 3.5736169306744885, |
|
"tokens_seen": 1251737600 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.0003902331189710611, |
|
"loss": 2.6164, |
|
"theoretical_loss": 3.572764637774024, |
|
"tokens_seen": 1255014400 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00038940561755248723, |
|
"loss": 2.6288, |
|
"theoretical_loss": 3.571915188518457, |
|
"tokens_seen": 1258291200 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00038857811613391335, |
|
"loss": 2.6475, |
|
"theoretical_loss": 3.571068566051716, |
|
"tokens_seen": 1261568000 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00038775061471533947, |
|
"loss": 2.6866, |
|
"theoretical_loss": 3.5702247536610976, |
|
"tokens_seen": 1264844800 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.0003869231132967657, |
|
"loss": 2.667, |
|
"theoretical_loss": 3.5693837347756783, |
|
"tokens_seen": 1268121600 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.0003860956118781918, |
|
"loss": 2.6401, |
|
"theoretical_loss": 3.5685454929647475, |
|
"tokens_seen": 1271398400 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00038526811045961793, |
|
"loss": 2.6509, |
|
"theoretical_loss": 3.5677100119362675, |
|
"tokens_seen": 1274675200 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00038444060904104405, |
|
"loss": 2.6335, |
|
"theoretical_loss": 3.566877275535345, |
|
"tokens_seen": 1277952000 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00038361310762247017, |
|
"loss": 2.6285, |
|
"theoretical_loss": 3.566047267742733, |
|
"tokens_seen": 1281228800 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00038278560620389634, |
|
"loss": 2.6087, |
|
"theoretical_loss": 3.5652199726733453, |
|
"tokens_seen": 1284505600 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0003819581047853225, |
|
"loss": 2.6032, |
|
"theoretical_loss": 3.564395374574796, |
|
"tokens_seen": 1287782400 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00038113060336674863, |
|
"loss": 2.6624, |
|
"theoretical_loss": 3.5635734578259557, |
|
"tokens_seen": 1291059200 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0003803031019481748, |
|
"loss": 2.6763, |
|
"theoretical_loss": 3.5627542069355282, |
|
"tokens_seen": 1294336000 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0003794756005296009, |
|
"loss": 2.6398, |
|
"theoretical_loss": 3.5619376065406474, |
|
"tokens_seen": 1297612800 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00037864809911102704, |
|
"loss": 2.6559, |
|
"theoretical_loss": 3.5611236414054868, |
|
"tokens_seen": 1300889600 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00037782059769245316, |
|
"loss": 2.6598, |
|
"theoretical_loss": 3.560312296419899, |
|
"tokens_seen": 1304166400 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.0003769930962738793, |
|
"loss": 2.6584, |
|
"theoretical_loss": 3.55950355659806, |
|
"tokens_seen": 1307443200 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.49020908264157476, |
|
"debugging/distinct-1-grams": 0.768901113497886, |
|
"debugging/distinct-2-grams": 0.9428782333551957, |
|
"debugging/entropy-1-grams": 6.085999550681761, |
|
"debugging/entropy-2-grams": 7.0033060167714964, |
|
"debugging/length": 490.2352941176471, |
|
"debugging/num_segments": 17, |
|
"epoch": 0.47, |
|
"objective/train/avg_token_score": 0.02056093141436577, |
|
"objective/train/avg_weight": 0.977873682975769, |
|
"objective/train/docs_used": 741674, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 3.746752977371216, |
|
"objective/train/std_weight": 0.11813095211982727, |
|
"objective/train/theoretical_loss": 3.558697407077142, |
|
"objective/train/tokens_used": 1331180000, |
|
"theoretical_loss": 3.558697407077142, |
|
"tokens_seen": 1310720000 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.0003761655948553055, |
|
"loss": 2.6483, |
|
"theoretical_loss": 3.558697407077142, |
|
"tokens_seen": 1310720000 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.0003753380934367316, |
|
"loss": 2.6646, |
|
"theoretical_loss": 3.5578938331159975, |
|
"tokens_seen": 1313996800 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00037451059201815774, |
|
"loss": 2.6382, |
|
"theoretical_loss": 3.557092820093863, |
|
"tokens_seen": 1317273600 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00037368309059958386, |
|
"loss": 2.6577, |
|
"theoretical_loss": 3.556294353509079, |
|
"tokens_seen": 1320550400 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00037285558918101, |
|
"loss": 2.6493, |
|
"theoretical_loss": 3.555498418977828, |
|
"tokens_seen": 1323827200 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00037202808776243615, |
|
"loss": 2.6434, |
|
"theoretical_loss": 3.5547050022328874, |
|
"tokens_seen": 1327104000 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.0003712005863438623, |
|
"loss": 2.639, |
|
"theoretical_loss": 3.553914089122399, |
|
"tokens_seen": 1330380800 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00037037308492528844, |
|
"loss": 2.6797, |
|
"theoretical_loss": 3.553125665608655, |
|
"tokens_seen": 1333657600 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.0003695455835067146, |
|
"loss": 2.6617, |
|
"theoretical_loss": 3.5523397177669005, |
|
"tokens_seen": 1336934400 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00036871808208814073, |
|
"loss": 2.6747, |
|
"theoretical_loss": 3.551556231784149, |
|
"tokens_seen": 1340211200 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00036789058066956685, |
|
"loss": 2.6276, |
|
"theoretical_loss": 3.5507751939580148, |
|
"tokens_seen": 1343488000 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00036706307925099297, |
|
"loss": 2.6325, |
|
"theoretical_loss": 3.5499965906955606, |
|
"tokens_seen": 1346764800 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.0003662355778324192, |
|
"loss": 2.6789, |
|
"theoretical_loss": 3.549220408512161, |
|
"tokens_seen": 1350041600 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.0003654080764138453, |
|
"loss": 2.6544, |
|
"theoretical_loss": 3.5484466340303755, |
|
"tokens_seen": 1353318400 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00036458057499527143, |
|
"loss": 2.6411, |
|
"theoretical_loss": 3.547675253978843, |
|
"tokens_seen": 1356595200 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00036375307357669755, |
|
"loss": 2.6751, |
|
"theoretical_loss": 3.5469062551911854, |
|
"tokens_seen": 1359872000 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00036292557215812367, |
|
"loss": 2.6842, |
|
"theoretical_loss": 3.5461396246049244, |
|
"tokens_seen": 1363148800 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.0003620980707395498, |
|
"loss": 2.6504, |
|
"theoretical_loss": 3.545375349260419, |
|
"tokens_seen": 1366425600 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00036127056932097596, |
|
"loss": 2.6894, |
|
"theoretical_loss": 3.544613416299808, |
|
"tokens_seen": 1369702400 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00036044306790240213, |
|
"loss": 2.6629, |
|
"theoretical_loss": 3.5438538129659687, |
|
"tokens_seen": 1372979200 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00035961556648382825, |
|
"loss": 2.6823, |
|
"theoretical_loss": 3.5430965266014933, |
|
"tokens_seen": 1376256000 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.0003587880650652544, |
|
"loss": 2.6879, |
|
"theoretical_loss": 3.5423415446476705, |
|
"tokens_seen": 1379532800 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00035796056364668054, |
|
"loss": 2.6256, |
|
"theoretical_loss": 3.541588854643487, |
|
"tokens_seen": 1382809600 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.00035713306222810666, |
|
"loss": 2.6521, |
|
"theoretical_loss": 3.5408384442246343, |
|
"tokens_seen": 1386086400 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.00035632211083790424, |
|
"loss": 2.6065, |
|
"theoretical_loss": 3.540090301122535, |
|
"tokens_seen": 1389363200 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.00035549460941933046, |
|
"loss": 2.6299, |
|
"theoretical_loss": 3.5393444131633762, |
|
"tokens_seen": 1392640000 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.0003546671080007566, |
|
"loss": 2.676, |
|
"theoretical_loss": 3.5386007682671576, |
|
"tokens_seen": 1395916800 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.0003538396065821827, |
|
"loss": 2.6589, |
|
"theoretical_loss": 3.5378593544467494, |
|
"tokens_seen": 1399193600 |
|
} |
|
], |
|
"max_steps": 42724, |
|
"num_train_epochs": 9223372036854775807, |
|
"total_flos": 7.14460209610752e+17, |
|
"trial_name": null, |
|
"trial_params": null |
|
} |
|
|