{ "best_metric": null, "best_model_checkpoint": null, "epoch": 0.5, "global_step": 21362, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0.0, "learning_rate": 1.1682242990654204e-06, "loss": 10.7926, "theoretical_loss": 20.81281176760504, "tokens_seen": 65536 }, { "epoch": 0.0, "learning_rate": 5.841121495327103e-05, "loss": 8.8129, "theoretical_loss": 8.563476630668958, "tokens_seen": 3276800 }, { "epoch": 0.0, "learning_rate": 0.00011682242990654206, "loss": 6.5669, "theoretical_loss": 7.477752684105921, "tokens_seen": 6553600 }, { "epoch": 0.0, "learning_rate": 0.00017523364485981307, "loss": 5.6216, "theoretical_loss": 6.9337484549527915, "tokens_seen": 9830400 }, { "epoch": 0.0, "learning_rate": 0.00023364485981308412, "loss": 5.2369, "theoretical_loss": 6.5835601944843045, "tokens_seen": 13107200 }, { "epoch": 0.01, "learning_rate": 0.00029205607476635517, "loss": 5.0152, "theoretical_loss": 6.3307075311739744, "tokens_seen": 16384000 }, { "epoch": 0.01, "learning_rate": 0.00035046728971962614, "loss": 4.854, "theoretical_loss": 6.135523197998216, "tokens_seen": 19660800 }, { "epoch": 0.01, "learning_rate": 0.0004088785046728972, "loss": 4.7404, "theoretical_loss": 5.978095549927499, "tokens_seen": 22937600 }, { "epoch": 0.01, "learning_rate": 0.00046728971962616824, "loss": 4.6057, "theoretical_loss": 5.847111292323815, "tokens_seen": 26214400 }, { "epoch": 0.01, "learning_rate": 0.000499739928125591, "loss": 4.4738, "theoretical_loss": 5.735570781940016, "tokens_seen": 29491200 }, { "epoch": 0.01, "learning_rate": 0.0004991488556837526, "loss": 4.3968, "theoretical_loss": 5.638864110129244, "tokens_seen": 32768000 }, { "epoch": 0.01, "learning_rate": 0.0004985577832419141, "loss": 4.2438, "theoretical_loss": 5.553806347902798, "tokens_seen": 36044800 }, { "epoch": 0.01, "learning_rate": 0.0004979667108000757, "loss": 4.1836, "theoretical_loss": 5.478112046614329, "tokens_seen": 39321600 }, { "epoch": 0.02, "learning_rate": 0.0004973756383582371, "loss": 4.1064, "theoretical_loss": 5.410089925637252, "tokens_seen": 42598400 }, { "epoch": 0.02, "learning_rate": 0.0004967845659163987, "loss": 4.0626, "theoretical_loss": 5.348456049793725, "tokens_seen": 45875200 }, { "epoch": 0.02, "learning_rate": 0.0004961934934745603, "loss": 3.9378, "theoretical_loss": 5.292214532995457, "tokens_seen": 49152000 }, { "epoch": 0.02, "learning_rate": 0.0004956024210327218, "loss": 3.9244, "theoretical_loss": 5.240578591827869, "tokens_seen": 52428800 }, { "epoch": 0.02, "learning_rate": 0.0004950113485908833, "loss": 3.8394, "theoretical_loss": 5.192916690583679, "tokens_seen": 55705600 }, { "epoch": 0.02, "learning_rate": 0.0004944202761490448, "loss": 3.761, "theoretical_loss": 5.148714829414331, "tokens_seen": 58982400 }, { "epoch": 0.02, "learning_rate": 0.0004938292037072064, "loss": 3.6902, "theoretical_loss": 5.107549528462992, "tokens_seen": 62259200 }, { "epoch": 0.02, "learning_rate": 0.0004932381312653678, "loss": 3.649, "theoretical_loss": 5.069068083201136, "tokens_seen": 65536000 }, { "epoch": 0.02, "learning_rate": 0.0004926470588235294, "loss": 3.5367, "theoretical_loss": 5.032973875895897, "tokens_seen": 68812800 }, { "epoch": 0.03, "learning_rate": 0.000492055986381691, "loss": 3.473, "theoretical_loss": 4.999015274282555, "tokens_seen": 72089600 }, { "epoch": 0.03, "learning_rate": 0.0004914649139398525, "loss": 3.3792, "theoretical_loss": 4.966977121409853, "tokens_seen": 75366400 }, { "epoch": 0.03, "learning_rate": 0.000490873841498014, "loss": 3.3265, "theoretical_loss": 4.936674127683027, "tokens_seen": 78643200 }, { "epoch": 0.03, "learning_rate": 0.0004902827690561755, "loss": 3.3293, "theoretical_loss": 4.907945679887972, "tokens_seen": 81920000 }, { "epoch": 0.03, "learning_rate": 0.0004896916966143371, "loss": 3.321, "theoretical_loss": 4.8806517198708175, "tokens_seen": 85196800 }, { "epoch": 0.03, "learning_rate": 0.0004891006241724985, "loss": 3.2217, "theoretical_loss": 4.85466944053967, "tokens_seen": 88473600 }, { "epoch": 0.03, "learning_rate": 0.0004885095517306601, "loss": 3.2277, "theoretical_loss": 4.829890613366704, "tokens_seen": 91750400 }, { "epoch": 0.03, "learning_rate": 0.0004879184792888217, "loss": 3.2068, "theoretical_loss": 4.806219408835812, "tokens_seen": 95027200 }, { "epoch": 0.04, "learning_rate": 0.0004873274068469832, "loss": 3.1622, "theoretical_loss": 4.783570605334148, "tokens_seen": 98304000 }, { "epoch": 0.04, "learning_rate": 0.00048673633440514467, "loss": 3.1879, "theoretical_loss": 4.761868106830299, "tokens_seen": 101580800 }, { "epoch": 0.04, "learning_rate": 0.0004861452619633062, "loss": 3.1352, "theoretical_loss": 4.741043708020364, "tokens_seen": 104857600 }, { "epoch": 0.04, "learning_rate": 0.0004855541895214677, "loss": 3.1048, "theoretical_loss": 4.721036059306941, "tokens_seen": 108134400 }, { "epoch": 0.04, "learning_rate": 0.0004849631170796293, "loss": 3.1318, "theoretical_loss": 4.701789794289756, "tokens_seen": 111411200 }, { "epoch": 0.04, "learning_rate": 0.0004843720446377908, "loss": 3.1115, "theoretical_loss": 4.68325479029382, "tokens_seen": 114688000 }, { "epoch": 0.04, "learning_rate": 0.00048378097219595233, "loss": 3.1102, "theoretical_loss": 4.6653855384841725, "tokens_seen": 117964800 }, { "epoch": 0.04, "learning_rate": 0.00048318989975411385, "loss": 3.0769, "theoretical_loss": 4.6481406047776295, "tokens_seen": 121241600 }, { "epoch": 0.04, "learning_rate": 0.00048259882731227537, "loss": 3.0882, "theoretical_loss": 4.631482166397534, "tokens_seen": 124518400 }, { "epoch": 0.05, "learning_rate": 0.0004820077548704369, "loss": 3.084, "theoretical_loss": 4.615375611773608, "tokens_seen": 127795200 }, { "epoch": 0.05, "learning_rate": 0.00048141668242859847, "loss": 3.0752, "theoretical_loss": 4.5997891937483955, "tokens_seen": 131072000 }, { "epoch": 0.05, "learning_rate": 0.00048082560998676, "loss": 3.0295, "theoretical_loss": 4.584693727850565, "tokens_seen": 134348800 }, { "epoch": 0.05, "learning_rate": 0.0004802345375449215, "loss": 3.0131, "theoretical_loss": 4.570062328836407, "tokens_seen": 137625600 }, { "epoch": 0.05, "learning_rate": 0.00047964346510308303, "loss": 3.0004, "theoretical_loss": 4.5558701798619285, "tokens_seen": 140902400 }, { "epoch": 0.05, "learning_rate": 0.00047905239266124455, "loss": 3.0324, "theoretical_loss": 4.542094329588689, "tokens_seen": 144179200 }, { "epoch": 0.05, "learning_rate": 0.00047846132021940607, "loss": 3.0272, "theoretical_loss": 4.528713513292708, "tokens_seen": 147456000 }, { "epoch": 0.05, "learning_rate": 0.0004778702477775676, "loss": 2.9858, "theoretical_loss": 4.515707994672887, "tokens_seen": 150732800 }, { "epoch": 0.06, "learning_rate": 0.00047727917533572917, "loss": 2.9764, "theoretical_loss": 4.503059425571229, "tokens_seen": 154009600 }, { "epoch": 0.06, "learning_rate": 0.0004766881028938907, "loss": 2.9751, "theoretical_loss": 4.490750721243157, "tokens_seen": 157286400 }, { "epoch": 0.06, "learning_rate": 0.0004760970304520522, "loss": 3.005, "theoretical_loss": 4.478765949169858, "tokens_seen": 160563200 }, { "epoch": 0.06, "objective/train/docs_used": 91708, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 2.834399700164795, "objective/train/theoretical_loss": 4.46709022969911, "objective/train/tokens_used": 184300000, "theoretical_loss": 4.46709022969911, "tokens_seen": 163840000 }, { "epoch": 0.06, "learning_rate": 0.00047550595801021373, "loss": 2.9681, "theoretical_loss": 4.46709022969911, "tokens_seen": 163840000 }, { "epoch": 0.06, "learning_rate": 0.00047491488556837525, "loss": 2.9537, "theoretical_loss": 4.455709647047437, "tokens_seen": 167116800 }, { "epoch": 0.06, "learning_rate": 0.00047432381312653677, "loss": 2.9301, "theoretical_loss": 4.444611169403321, "tokens_seen": 170393600 }, { "epoch": 0.06, "learning_rate": 0.00047373274068469835, "loss": 2.9362, "theoretical_loss": 4.4337825770455375, "tokens_seen": 173670400 }, { "epoch": 0.06, "learning_rate": 0.00047314166824285987, "loss": 2.9216, "theoretical_loss": 4.423212397538051, "tokens_seen": 176947200 }, { "epoch": 0.06, "learning_rate": 0.0004725505958010214, "loss": 2.9052, "theoretical_loss": 4.4128898471880325, "tokens_seen": 180224000 }, { "epoch": 0.07, "learning_rate": 0.0004719595233591829, "loss": 2.9085, "theoretical_loss": 4.40280477805997, "tokens_seen": 183500800 }, { "epoch": 0.07, "learning_rate": 0.00047136845091734443, "loss": 2.929, "theoretical_loss": 4.392947629929752, "tokens_seen": 186777600 }, { "epoch": 0.07, "learning_rate": 0.00047077737847550595, "loss": 2.934, "theoretical_loss": 4.383309386640423, "tokens_seen": 190054400 }, { "epoch": 0.07, "learning_rate": 0.0004701863060336675, "loss": 2.909, "theoretical_loss": 4.373881536388167, "tokens_seen": 193331200 }, { "epoch": 0.07, "learning_rate": 0.00046959523359182905, "loss": 2.8757, "theoretical_loss": 4.364656035524595, "tokens_seen": 196608000 }, { "epoch": 0.07, "learning_rate": 0.00046900416114999057, "loss": 2.8927, "theoretical_loss": 4.355625275511174, "tokens_seen": 199884800 }, { "epoch": 0.07, "learning_rate": 0.0004684130887081521, "loss": 2.8711, "theoretical_loss": 4.346782052704563, "tokens_seen": 203161600 }, { "epoch": 0.07, "learning_rate": 0.0004678220162663136, "loss": 2.8508, "theoretical_loss": 4.338119540689052, "tokens_seen": 206438400 }, { "epoch": 0.07, "learning_rate": 0.00046723094382447513, "loss": 2.8683, "theoretical_loss": 4.329631264904703, "tokens_seen": 209715200 }, { "epoch": 0.08, "learning_rate": 0.00046663987138263665, "loss": 2.8571, "theoretical_loss": 4.321311079348144, "tokens_seen": 212992000 }, { "epoch": 0.08, "learning_rate": 0.0004660487989407982, "loss": 2.888, "theoretical_loss": 4.313153145147683, "tokens_seen": 216268800 }, { "epoch": 0.08, "learning_rate": 0.00046545772649895975, "loss": 2.8459, "theoretical_loss": 4.305151910836119, "tokens_seen": 219545600 }, { "epoch": 0.08, "learning_rate": 0.00046486665405712127, "loss": 2.8228, "theoretical_loss": 4.2973020941635784, "tokens_seen": 222822400 }, { "epoch": 0.08, "learning_rate": 0.0004642755816152828, "loss": 2.8308, "theoretical_loss": 4.28959866530949, "tokens_seen": 226099200 }, { "epoch": 0.08, "learning_rate": 0.0004636845091734443, "loss": 2.8282, "theoretical_loss": 4.282036831367506, "tokens_seen": 229376000 }, { "epoch": 0.08, "learning_rate": 0.00046309343673160583, "loss": 2.8328, "theoretical_loss": 4.274612021990189, "tokens_seen": 232652800 }, { "epoch": 0.08, "learning_rate": 0.0004625023642897674, "loss": 2.859, "theoretical_loss": 4.267319876091788, "tokens_seen": 235929600 }, { "epoch": 0.09, "learning_rate": 0.0004619112918479289, "loss": 2.8342, "theoretical_loss": 4.260156229517635, "tokens_seen": 239206400 }, { "epoch": 0.09, "learning_rate": 0.00046132021940609044, "loss": 2.7766, "theoretical_loss": 4.253117103597704, "tokens_seen": 242483200 }, { "epoch": 0.09, "learning_rate": 0.00046072914696425197, "loss": 2.7575, "theoretical_loss": 4.246198694509945, "tokens_seen": 245760000 }, { "epoch": 0.09, "learning_rate": 0.0004601380745224135, "loss": 2.7936, "theoretical_loss": 4.239397363386152, "tokens_seen": 249036800 }, { "epoch": 0.09, "learning_rate": 0.000459547002080575, "loss": 2.8022, "theoretical_loss": 4.232709627099522, "tokens_seen": 252313600 }, { "epoch": 0.09, "learning_rate": 0.0004589559296387366, "loss": 2.8341, "theoretical_loss": 4.226132149678757, "tokens_seen": 255590400 }, { "epoch": 0.09, "learning_rate": 0.0004583648571968981, "loss": 2.8652, "theoretical_loss": 4.219661734298666, "tokens_seen": 258867200 }, { "epoch": 0.09, "learning_rate": 0.0004577737847550596, "loss": 2.849, "theoretical_loss": 4.213295315801815, "tokens_seen": 262144000 }, { "epoch": 0.09, "learning_rate": 0.0004571827123132211, "loss": 2.8169, "theoretical_loss": 4.207029953709861, "tokens_seen": 265420800 }, { "epoch": 0.1, "learning_rate": 0.0004565916398713826, "loss": 2.8278, "theoretical_loss": 4.200862825686893, "tokens_seen": 268697600 }, { "epoch": 0.1, "learning_rate": 0.00045600056742954413, "loss": 2.7814, "theoretical_loss": 4.19479122142044, "tokens_seen": 271974400 }, { "epoch": 0.1, "learning_rate": 0.0004554094949877057, "loss": 2.7896, "theoretical_loss": 4.188812536888775, "tokens_seen": 275251200 }, { "epoch": 0.1, "learning_rate": 0.00045481842254586723, "loss": 2.7571, "theoretical_loss": 4.182924268985855, "tokens_seen": 278528000 }, { "epoch": 0.1, "learning_rate": 0.00045422735010402875, "loss": 2.7364, "theoretical_loss": 4.177124010477671, "tokens_seen": 281804800 }, { "epoch": 0.1, "learning_rate": 0.00045363627766219027, "loss": 2.7741, "theoretical_loss": 4.171409445265983, "tokens_seen": 285081600 }, { "epoch": 0.1, "learning_rate": 0.0004530452052203518, "loss": 2.7755, "theoretical_loss": 4.165778343937409, "tokens_seen": 288358400 }, { "epoch": 0.1, "learning_rate": 0.0004524541327785133, "loss": 2.801, "theoretical_loss": 4.160228559577659, "tokens_seen": 291635200 }, { "epoch": 0.11, "learning_rate": 0.00045186306033667483, "loss": 2.761, "theoretical_loss": 4.15475802383233, "tokens_seen": 294912000 }, { "epoch": 0.11, "learning_rate": 0.0004512719878948364, "loss": 2.6851, "theoretical_loss": 4.149364743197177, "tokens_seen": 298188800 }, { "epoch": 0.11, "learning_rate": 0.00045068091545299793, "loss": 2.7103, "theoretical_loss": 4.14404679552214, "tokens_seen": 301465600 }, { "epoch": 0.11, "learning_rate": 0.00045008984301115945, "loss": 2.6929, "theoretical_loss": 4.138802326714632, "tokens_seen": 304742400 }, { "epoch": 0.11, "learning_rate": 0.00044949877056932097, "loss": 2.6886, "theoretical_loss": 4.133629547628726, "tokens_seen": 308019200 }, { "epoch": 0.11, "learning_rate": 0.0004489076981274825, "loss": 2.6871, "theoretical_loss": 4.128526731127894, "tokens_seen": 311296000 }, { "epoch": 0.11, "learning_rate": 0.000448316625685644, "loss": 2.7051, "theoretical_loss": 4.123492209309923, "tokens_seen": 314572800 }, { "epoch": 0.11, "learning_rate": 0.0004477255532438056, "loss": 2.7244, "theoretical_loss": 4.118524370883447, "tokens_seen": 317849600 }, { "epoch": 0.11, "learning_rate": 0.0004471344808019671, "loss": 2.7389, "theoretical_loss": 4.113621658686355, "tokens_seen": 321126400 }, { "epoch": 0.12, "learning_rate": 0.00044654340836012863, "loss": 2.7728, "theoretical_loss": 4.108782567337039, "tokens_seen": 324403200 }, { "debugging/Self-BLEU-5": 0.6607348357914675, "debugging/distinct-1-grams": 0.8023495471425544, "debugging/distinct-2-grams": 0.9518089508119704, "debugging/entropy-1-grams": 6.089592994551723, "debugging/entropy-2-grams": 6.90484015923658, "debugging/length": 683.3888888888889, "debugging/num_segments": 18, "epoch": 0.12, "objective/train/docs_used": 173345, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 2.669816255569458, "objective/train/theoretical_loss": 4.104005641010112, "objective/train/tokens_used": 348140000, "theoretical_loss": 4.104005641010112, "tokens_seen": 327680000 }, { "epoch": 0.12, "learning_rate": 0.00044595233591829015, "loss": 2.7658, "theoretical_loss": 4.104005641010112, "tokens_seen": 327680000 }, { "epoch": 0.12, "learning_rate": 0.00044536126347645167, "loss": 2.7501, "theoretical_loss": 4.099289471328812, "tokens_seen": 330956800 }, { "epoch": 0.12, "learning_rate": 0.0004447701910346132, "loss": 2.7384, "theoretical_loss": 4.094632695366921, "tokens_seen": 334233600 }, { "epoch": 0.12, "learning_rate": 0.00044417911859277476, "loss": 2.7302, "theoretical_loss": 4.090033993753448, "tokens_seen": 337510400 }, { "epoch": 0.12, "learning_rate": 0.0004435880461509363, "loss": 2.6847, "theoretical_loss": 4.085492088873883, "tokens_seen": 340787200 }, { "epoch": 0.12, "learning_rate": 0.0004429969737090978, "loss": 2.6919, "theoretical_loss": 4.081005743162224, "tokens_seen": 344064000 }, { "epoch": 0.12, "learning_rate": 0.0004424059012672593, "loss": 2.6785, "theoretical_loss": 4.076573757478361, "tokens_seen": 347340800 }, { "epoch": 0.13, "learning_rate": 0.00044181482882542085, "loss": 2.667, "theoretical_loss": 4.072194969565807, "tokens_seen": 350617600 }, { "epoch": 0.13, "learning_rate": 0.00044122375638358237, "loss": 2.6537, "theoretical_loss": 4.067868252585089, "tokens_seen": 353894400 }, { "epoch": 0.13, "learning_rate": 0.0004406326839417439, "loss": 2.669, "theoretical_loss": 4.063592513718411, "tokens_seen": 357171200 }, { "epoch": 0.13, "learning_rate": 0.00044004161149990546, "loss": 2.6391, "theoretical_loss": 4.059366692841521, "tokens_seen": 360448000 }, { "epoch": 0.13, "learning_rate": 0.000439450539058067, "loss": 2.638, "theoretical_loss": 4.055189761258959, "tokens_seen": 363724800 }, { "epoch": 0.13, "learning_rate": 0.0004388594666162285, "loss": 2.6594, "theoretical_loss": 4.051060720499127, "tokens_seen": 367001600 }, { "epoch": 0.13, "learning_rate": 0.00043826839417439, "loss": 2.6626, "theoretical_loss": 4.046978601165831, "tokens_seen": 370278400 }, { "epoch": 0.13, "learning_rate": 0.00043767732173255155, "loss": 2.6763, "theoretical_loss": 4.042942461843204, "tokens_seen": 373555200 }, { "epoch": 0.13, "learning_rate": 0.00043708624929071307, "loss": 2.6615, "theoretical_loss": 4.038951388051044, "tokens_seen": 376832000 }, { "epoch": 0.14, "learning_rate": 0.00043649517684887464, "loss": 2.6649, "theoretical_loss": 4.035004491247873, "tokens_seen": 380108800 }, { "epoch": 0.14, "learning_rate": 0.00043590410440703616, "loss": 2.6609, "theoretical_loss": 4.031100907879109, "tokens_seen": 383385600 }, { "epoch": 0.14, "learning_rate": 0.0004353130319651977, "loss": 2.6541, "theoretical_loss": 4.02723979846797, "tokens_seen": 386662400 }, { "epoch": 0.14, "learning_rate": 0.0004347219595233592, "loss": 2.6331, "theoretical_loss": 4.023420346746835, "tokens_seen": 389939200 }, { "epoch": 0.14, "learning_rate": 0.0004341308870815207, "loss": 2.6643, "theoretical_loss": 4.019641758826938, "tokens_seen": 393216000 }, { "epoch": 0.14, "learning_rate": 0.00043353981463968225, "loss": 2.7007, "theoretical_loss": 4.015903262404413, "tokens_seen": 396492800 }, { "epoch": 0.14, "learning_rate": 0.0004329487421978438, "loss": 2.6606, "theoretical_loss": 4.012204106000786, "tokens_seen": 399769600 }, { "epoch": 0.14, "learning_rate": 0.00043235766975600534, "loss": 2.6741, "theoretical_loss": 4.008543558236181, "tokens_seen": 403046400 }, { "epoch": 0.15, "learning_rate": 0.00043176659731416686, "loss": 2.6851, "theoretical_loss": 4.004920907133565, "tokens_seen": 406323200 }, { "epoch": 0.15, "learning_rate": 0.0004311755248723284, "loss": 2.6507, "theoretical_loss": 4.001335459452449, "tokens_seen": 409600000 }, { "epoch": 0.15, "learning_rate": 0.0004305844524304899, "loss": 2.6388, "theoretical_loss": 3.997786540050617, "tokens_seen": 412876800 }, { "epoch": 0.15, "learning_rate": 0.0004299933799886514, "loss": 2.6775, "theoretical_loss": 3.9942734912724456, "tokens_seen": 416153600 }, { "epoch": 0.15, "learning_rate": 0.00042940230754681295, "loss": 2.6459, "theoretical_loss": 3.9907956723625375, "tokens_seen": 419430400 }, { "epoch": 0.15, "learning_rate": 0.0004288112351049745, "loss": 2.6205, "theoretical_loss": 3.9873524589034224, "tokens_seen": 422707200 }, { "epoch": 0.15, "learning_rate": 0.000428220162663136, "loss": 2.6119, "theoretical_loss": 3.9839432422761556, "tokens_seen": 425984000 }, { "epoch": 0.15, "learning_rate": 0.0004276290902212975, "loss": 2.5946, "theoretical_loss": 3.980567429142721, "tokens_seen": 429260800 }, { "epoch": 0.15, "learning_rate": 0.00042703801777945903, "loss": 2.6213, "theoretical_loss": 3.977224440949197, "tokens_seen": 432537600 }, { "epoch": 0.16, "learning_rate": 0.00042644694533762055, "loss": 2.6306, "theoretical_loss": 3.9739137134486917, "tokens_seen": 435814400 }, { "epoch": 0.16, "learning_rate": 0.00042585587289578207, "loss": 2.621, "theoretical_loss": 3.9706346962431396, "tokens_seen": 439091200 }, { "epoch": 0.16, "learning_rate": 0.00042526480045394365, "loss": 2.6262, "theoretical_loss": 3.9673868523430564, "tokens_seen": 442368000 }, { "epoch": 0.16, "learning_rate": 0.00042467372801210517, "loss": 2.6389, "theoretical_loss": 3.9641696577444376, "tokens_seen": 445644800 }, { "epoch": 0.16, "learning_rate": 0.0004240826555702667, "loss": 2.6321, "theoretical_loss": 3.9609826010220033, "tokens_seen": 448921600 }, { "epoch": 0.16, "learning_rate": 0.0004234915831284282, "loss": 2.6313, "theoretical_loss": 3.9578251829380506, "tokens_seen": 452198400 }, { "epoch": 0.16, "learning_rate": 0.00042290051068658973, "loss": 2.6319, "theoretical_loss": 3.954696916066199, "tokens_seen": 455475200 }, { "epoch": 0.16, "learning_rate": 0.00042230943824475125, "loss": 2.6369, "theoretical_loss": 3.9515973244293643, "tokens_seen": 458752000 }, { "epoch": 0.17, "learning_rate": 0.0004217183658029128, "loss": 2.648, "theoretical_loss": 3.948525943151326, "tokens_seen": 462028800 }, { "epoch": 0.17, "learning_rate": 0.00042112729336107435, "loss": 2.674, "theoretical_loss": 3.9454823181212815, "tokens_seen": 465305600 }, { "epoch": 0.17, "learning_rate": 0.00042053622091923587, "loss": 2.6931, "theoretical_loss": 3.9424660056708167, "tokens_seen": 468582400 }, { "epoch": 0.17, "learning_rate": 0.0004199451484773974, "loss": 2.6383, "theoretical_loss": 3.939476572262754, "tokens_seen": 471859200 }, { "epoch": 0.17, "learning_rate": 0.0004193540760355589, "loss": 2.6659, "theoretical_loss": 3.9365135941913563, "tokens_seen": 475136000 }, { "epoch": 0.17, "learning_rate": 0.00041876300359372043, "loss": 2.6818, "theoretical_loss": 3.9335766572934023, "tokens_seen": 478412800 }, { "epoch": 0.17, "learning_rate": 0.00041817193115188195, "loss": 2.6567, "theoretical_loss": 3.9306653566696603, "tokens_seen": 481689600 }, { "epoch": 0.17, "learning_rate": 0.0004175808587100435, "loss": 2.6558, "theoretical_loss": 3.927779296416332, "tokens_seen": 484966400 }, { "epoch": 0.17, "learning_rate": 0.00041698978626820505, "loss": 2.6617, "theoretical_loss": 3.924918089366024, "tokens_seen": 488243200 }, { "epoch": 0.18, "objective/train/docs_used": 254462, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 2.4227302074432373, "objective/train/theoretical_loss": 3.9220813568378707, "objective/train/tokens_used": 511980000, "theoretical_loss": 3.9220813568378707, "tokens_seen": 491520000 }, { "epoch": 0.18, "learning_rate": 0.00041639871382636657, "loss": 2.658, "theoretical_loss": 3.9220813568378707, "tokens_seen": 491520000 }, { "epoch": 0.18, "learning_rate": 0.0004158076413845281, "loss": 2.5929, "theoretical_loss": 3.9192687283964096, "tokens_seen": 494796800 }, { "epoch": 0.18, "learning_rate": 0.00041522839039152636, "loss": 2.6373, "theoretical_loss": 3.9164798416188527, "tokens_seen": 498073600 }, { "epoch": 0.18, "learning_rate": 0.0004146373179496879, "loss": 2.6258, "theoretical_loss": 3.913714341870409, "tokens_seen": 501350400 }, { "epoch": 0.18, "learning_rate": 0.00041404624550784946, "loss": 2.6303, "theoretical_loss": 3.9109718820873303, "tokens_seen": 504627200 }, { "epoch": 0.18, "learning_rate": 0.000413455173066011, "loss": 2.6529, "theoretical_loss": 3.9082521225673625, "tokens_seen": 507904000 }, { "epoch": 0.18, "learning_rate": 0.0004128641006241725, "loss": 2.6082, "theoretical_loss": 3.9055547307673075, "tokens_seen": 511180800 }, { "epoch": 0.18, "learning_rate": 0.000412273028182334, "loss": 2.5831, "theoretical_loss": 3.9028793811074056, "tokens_seen": 514457600 }, { "epoch": 0.18, "learning_rate": 0.00041168195574049554, "loss": 2.6007, "theoretical_loss": 3.900225754782274, "tokens_seen": 517734400 }, { "epoch": 0.19, "learning_rate": 0.00041109088329865706, "loss": 2.596, "theoretical_loss": 3.897593539578138, "tokens_seen": 521011200 }, { "epoch": 0.19, "learning_rate": 0.00041049981085681863, "loss": 2.5878, "theoretical_loss": 3.8949824296961015, "tokens_seen": 524288000 }, { "epoch": 0.19, "learning_rate": 0.00040990873841498016, "loss": 2.5884, "theoretical_loss": 3.8923921255812353, "tokens_seen": 527564800 }, { "epoch": 0.19, "learning_rate": 0.0004093176659731417, "loss": 2.5497, "theoretical_loss": 3.8898223337572393, "tokens_seen": 530841600 }, { "epoch": 0.19, "learning_rate": 0.0004087265935313032, "loss": 2.571, "theoretical_loss": 3.88727276666648, "tokens_seen": 534118400 }, { "epoch": 0.19, "learning_rate": 0.0004081355210894647, "loss": 2.5724, "theoretical_loss": 3.884743142515184, "tokens_seen": 537395200 }, { "epoch": 0.19, "learning_rate": 0.00040754444864762624, "loss": 2.5607, "theoretical_loss": 3.8822331851235985, "tokens_seen": 540672000 }, { "epoch": 0.19, "learning_rate": 0.0004069533762057878, "loss": 2.6168, "theoretical_loss": 3.87974262378093, "tokens_seen": 543948800 }, { "epoch": 0.2, "learning_rate": 0.00040636230376394933, "loss": 2.6095, "theoretical_loss": 3.877271193104873, "tokens_seen": 547225600 }, { "epoch": 0.2, "learning_rate": 0.00040577123132211085, "loss": 2.5993, "theoretical_loss": 3.8748186329055736, "tokens_seen": 550502400 }, { "epoch": 0.2, "learning_rate": 0.0004051801588802724, "loss": 2.575, "theoretical_loss": 3.87238468805384, "tokens_seen": 553779200 }, { "epoch": 0.2, "learning_rate": 0.0004045890864384339, "loss": 2.5741, "theoretical_loss": 3.8699691083534633, "tokens_seen": 557056000 }, { "epoch": 0.2, "learning_rate": 0.0004039980139965954, "loss": 2.5787, "theoretical_loss": 3.8675716484174907, "tokens_seen": 560332800 }, { "epoch": 0.2, "learning_rate": 0.00040340694155475694, "loss": 2.5681, "theoretical_loss": 3.8651920675482936, "tokens_seen": 563609600 }, { "epoch": 0.2, "learning_rate": 0.0004028158691129185, "loss": 2.5427, "theoretical_loss": 3.862830129621318, "tokens_seen": 566886400 }, { "epoch": 0.2, "learning_rate": 0.00040222479667108003, "loss": 2.5492, "theoretical_loss": 3.8604856029723575, "tokens_seen": 570163200 }, { "epoch": 0.2, "learning_rate": 0.00040163372422924155, "loss": 2.5492, "theoretical_loss": 3.8581582602882447, "tokens_seen": 573440000 }, { "epoch": 0.21, "learning_rate": 0.0004010426517874031, "loss": 2.526, "theoretical_loss": 3.8558478785008203, "tokens_seen": 576716800 }, { "epoch": 0.21, "learning_rate": 0.0004004515793455646, "loss": 2.5441, "theoretical_loss": 3.8535542386840778, "tokens_seen": 579993600 }, { "epoch": 0.21, "learning_rate": 0.0003998605069037261, "loss": 2.5473, "theoretical_loss": 3.8512771259543586, "tokens_seen": 583270400 }, { "epoch": 0.21, "learning_rate": 0.0003992694344618877, "loss": 2.5235, "theoretical_loss": 3.8490163293735082, "tokens_seen": 586547200 }, { "epoch": 0.21, "learning_rate": 0.0003986783620200492, "loss": 2.5408, "theoretical_loss": 3.8467716418548648, "tokens_seen": 589824000 }, { "epoch": 0.21, "learning_rate": 0.00039808728957821073, "loss": 2.5857, "theoretical_loss": 3.844542860072007, "tokens_seen": 593100800 }, { "epoch": 0.21, "learning_rate": 0.00039749621713637225, "loss": 2.5715, "theoretical_loss": 3.8423297843701496, "tokens_seen": 596377600 }, { "epoch": 0.21, "learning_rate": 0.0003969051446945338, "loss": 2.5891, "theoretical_loss": 3.8401322186800995, "tokens_seen": 599654400 }, { "epoch": 0.22, "learning_rate": 0.0003963140722526953, "loss": 2.5636, "theoretical_loss": 3.83794997043469, "tokens_seen": 602931200 }, { "epoch": 0.22, "learning_rate": 0.00039572299981085687, "loss": 2.5828, "theoretical_loss": 3.8357828504876004, "tokens_seen": 606208000 }, { "epoch": 0.22, "learning_rate": 0.0003951319273690184, "loss": 2.5936, "theoretical_loss": 3.833630673034487, "tokens_seen": 609484800 }, { "epoch": 0.22, "learning_rate": 0.0003945408549271799, "loss": 2.5763, "theoretical_loss": 3.831493255536345, "tokens_seen": 612761600 }, { "epoch": 0.22, "learning_rate": 0.00039394978248534143, "loss": 2.5747, "theoretical_loss": 3.8293704186450253, "tokens_seen": 616038400 }, { "epoch": 0.22, "learning_rate": 0.00039335871004350295, "loss": 2.596, "theoretical_loss": 3.827261986130839, "tokens_seen": 619315200 }, { "epoch": 0.22, "learning_rate": 0.0003927676376016645, "loss": 2.5675, "theoretical_loss": 3.825167784812175, "tokens_seen": 622592000 }, { "epoch": 0.22, "learning_rate": 0.000392176565159826, "loss": 2.5751, "theoretical_loss": 3.823087644487069, "tokens_seen": 625868800 }, { "epoch": 0.22, "learning_rate": 0.00039158549271798757, "loss": 2.6107, "theoretical_loss": 3.8210213978666565, "tokens_seen": 629145600 }, { "epoch": 0.23, "learning_rate": 0.0003909944202761491, "loss": 2.6032, "theoretical_loss": 3.8189688805104476, "tokens_seen": 632422400 }, { "epoch": 0.23, "learning_rate": 0.00039040334783431056, "loss": 2.6353, "theoretical_loss": 3.816929930763374, "tokens_seen": 635699200 }, { "epoch": 0.23, "learning_rate": 0.0003898122753924721, "loss": 2.6048, "theoretical_loss": 3.8149043896945347, "tokens_seen": 638976000 }, { "epoch": 0.23, "learning_rate": 0.0003892212029506336, "loss": 2.5951, "theoretical_loss": 3.812892101037601, "tokens_seen": 642252800 }, { "epoch": 0.23, "learning_rate": 0.0003886301305087951, "loss": 2.6095, "theoretical_loss": 3.81089291113282, "tokens_seen": 645529600 }, { "epoch": 0.23, "learning_rate": 0.0003880390580669567, "loss": 2.5981, "theoretical_loss": 3.8089066688705673, "tokens_seen": 648806400 }, { "epoch": 0.23, "learning_rate": 0.0003874479856251182, "loss": 2.5641, "theoretical_loss": 3.8069332256363992, "tokens_seen": 652083200 }, { "debugging/Self-BLEU-5": 0.5613787023201813, "debugging/distinct-1-grams": 0.8149083264126824, "debugging/distinct-2-grams": 0.9601495159951162, "debugging/entropy-1-grams": 5.947486888431812, "debugging/entropy-2-grams": 6.555422311272144, "debugging/length": 621.5384615384615, "debugging/num_segments": 13, "epoch": 0.23, "objective/train/docs_used": 335300, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 2.533392906188965, "objective/train/theoretical_loss": 3.80497243525756, "objective/train/tokens_used": 675820000, "theoretical_loss": 3.80497243525756, "tokens_seen": 655360000 }, { "epoch": 0.23, "learning_rate": 0.00038685691318327974, "loss": 2.5531, "theoretical_loss": 3.80497243525756, "tokens_seen": 655360000 }, { "epoch": 0.24, "learning_rate": 0.00038626584074144126, "loss": 2.5586, "theoretical_loss": 3.8030241539508958, "tokens_seen": 658636800 }, { "epoch": 0.24, "learning_rate": 0.0003856747682996028, "loss": 2.5483, "theoretical_loss": 3.8010882402721324, "tokens_seen": 661913600 }, { "epoch": 0.24, "learning_rate": 0.0003850836958577643, "loss": 2.5549, "theoretical_loss": 3.7991645550664757, "tokens_seen": 665190400 }, { "epoch": 0.24, "learning_rate": 0.0003844926234159259, "loss": 2.5372, "theoretical_loss": 3.797252961420492, "tokens_seen": 668467200 }, { "epoch": 0.24, "learning_rate": 0.0003839015509740874, "loss": 2.5367, "theoretical_loss": 3.795353324615228, "tokens_seen": 671744000 }, { "epoch": 0.24, "learning_rate": 0.0003833104785322489, "loss": 2.5248, "theoretical_loss": 3.793465512080541, "tokens_seen": 675020800 }, { "epoch": 0.24, "learning_rate": 0.00038271940609041044, "loss": 2.5511, "theoretical_loss": 3.791589393350587, "tokens_seen": 678297600 }, { "epoch": 0.24, "learning_rate": 0.00038212833364857196, "loss": 2.5481, "theoretical_loss": 3.7897248400204475, "tokens_seen": 681574400 }, { "epoch": 0.24, "learning_rate": 0.0003815372612067335, "loss": 2.541, "theoretical_loss": 3.7878717257038534, "tokens_seen": 684851200 }, { "epoch": 0.25, "learning_rate": 0.000380946188764895, "loss": 2.5462, "theoretical_loss": 3.7860299259919685, "tokens_seen": 688128000 }, { "epoch": 0.25, "learning_rate": 0.0003803551163230566, "loss": 2.5154, "theoretical_loss": 3.7841993184132114, "tokens_seen": 691404800 }, { "epoch": 0.25, "learning_rate": 0.0003797640438812181, "loss": 2.5001, "theoretical_loss": 3.78237978239408, "tokens_seen": 694681600 }, { "epoch": 0.25, "learning_rate": 0.0003791729714393796, "loss": 2.5389, "theoretical_loss": 3.780571199220942, "tokens_seen": 697958400 }, { "epoch": 0.25, "learning_rate": 0.00037858189899754114, "loss": 2.5683, "theoretical_loss": 3.7787734520027803, "tokens_seen": 701235200 }, { "epoch": 0.25, "learning_rate": 0.00037799082655570266, "loss": 2.5869, "theoretical_loss": 3.7769864256348455, "tokens_seen": 704512000 }, { "epoch": 0.25, "learning_rate": 0.0003773997541138642, "loss": 2.6073, "theoretical_loss": 3.775210006763202, "tokens_seen": 707788800 }, { "epoch": 0.25, "learning_rate": 0.00037680868167202575, "loss": 2.5991, "theoretical_loss": 3.7734440837501406, "tokens_seen": 711065600 }, { "epoch": 0.26, "learning_rate": 0.0003762176092301873, "loss": 2.5618, "theoretical_loss": 3.7716885466404246, "tokens_seen": 714342400 }, { "epoch": 0.26, "learning_rate": 0.0003756265367883488, "loss": 2.5421, "theoretical_loss": 3.769943287128357, "tokens_seen": 717619200 }, { "epoch": 0.26, "learning_rate": 0.0003750354643465103, "loss": 2.5525, "theoretical_loss": 3.7682081985256364, "tokens_seen": 720896000 }, { "epoch": 0.26, "learning_rate": 0.00037444439190467184, "loss": 2.5112, "theoretical_loss": 3.7664831757299795, "tokens_seen": 724172800 }, { "epoch": 0.26, "learning_rate": 0.00037385331946283336, "loss": 2.5343, "theoretical_loss": 3.7647681151944976, "tokens_seen": 727449600 }, { "epoch": 0.26, "learning_rate": 0.00037326224702099493, "loss": 2.5372, "theoretical_loss": 3.7630629148977937, "tokens_seen": 730726400 }, { "epoch": 0.26, "learning_rate": 0.00037267117457915645, "loss": 2.5665, "theoretical_loss": 3.761367474314768, "tokens_seen": 734003200 }, { "epoch": 0.26, "learning_rate": 0.000372080102137318, "loss": 2.5266, "theoretical_loss": 3.7596816943881084, "tokens_seen": 737280000 }, { "epoch": 0.26, "learning_rate": 0.0003714890296954795, "loss": 2.585, "theoretical_loss": 3.758005477500451, "tokens_seen": 740556800 }, { "epoch": 0.27, "learning_rate": 0.000370897957253641, "loss": 2.5451, "theoretical_loss": 3.756338727447186, "tokens_seen": 743833600 }, { "epoch": 0.27, "learning_rate": 0.00037030688481180254, "loss": 2.5604, "theoretical_loss": 3.7546813494098945, "tokens_seen": 747110400 }, { "epoch": 0.27, "learning_rate": 0.00036971581236996406, "loss": 2.5853, "theoretical_loss": 3.7530332499304007, "tokens_seen": 750387200 }, { "epoch": 0.27, "learning_rate": 0.0003691365613769624, "loss": 2.5897, "theoretical_loss": 3.7513943368854195, "tokens_seen": 753664000 }, { "epoch": 0.27, "learning_rate": 0.0003685454889351239, "loss": 2.599, "theoretical_loss": 3.7497645194617863, "tokens_seen": 756940800 }, { "epoch": 0.27, "learning_rate": 0.0003679544164932854, "loss": 2.5786, "theoretical_loss": 3.748143708132246, "tokens_seen": 760217600 }, { "epoch": 0.27, "learning_rate": 0.00036736334405144695, "loss": 2.5257, "theoretical_loss": 3.7465318146317994, "tokens_seen": 763494400 }, { "epoch": 0.27, "learning_rate": 0.00036677227160960847, "loss": 2.5269, "theoretical_loss": 3.7449287519345766, "tokens_seen": 766771200 }, { "epoch": 0.28, "learning_rate": 0.00036618119916777, "loss": 2.5517, "theoretical_loss": 3.7433344342312385, "tokens_seen": 770048000 }, { "epoch": 0.28, "learning_rate": 0.00036559012672593156, "loss": 2.5221, "theoretical_loss": 3.7417487769068756, "tokens_seen": 773324800 }, { "epoch": 0.28, "learning_rate": 0.0003649990542840931, "loss": 2.4986, "theoretical_loss": 3.7401716965194076, "tokens_seen": 776601600 }, { "epoch": 0.28, "learning_rate": 0.0003644079818422546, "loss": 2.5261, "theoretical_loss": 3.738603110778461, "tokens_seen": 779878400 }, { "epoch": 0.28, "learning_rate": 0.0003638169094004161, "loss": 2.5152, "theoretical_loss": 3.73704293852471, "tokens_seen": 783155200 }, { "epoch": 0.28, "learning_rate": 0.00036322583695857765, "loss": 2.5525, "theoretical_loss": 3.7354910997096793, "tokens_seen": 786432000 }, { "epoch": 0.28, "learning_rate": 0.00036263476451673917, "loss": 2.5747, "theoretical_loss": 3.7339475153759825, "tokens_seen": 789708800 }, { "epoch": 0.28, "learning_rate": 0.00036204369207490074, "loss": 2.5238, "theoretical_loss": 3.732412107638, "tokens_seen": 792985600 }, { "epoch": 0.28, "learning_rate": 0.00036145261963306226, "loss": 2.4792, "theoretical_loss": 3.7308847996629724, "tokens_seen": 796262400 }, { "epoch": 0.29, "learning_rate": 0.0003608615471912238, "loss": 2.4976, "theoretical_loss": 3.7293655156525043, "tokens_seen": 799539200 }, { "epoch": 0.29, "learning_rate": 0.0003602704747493853, "loss": 2.4627, "theoretical_loss": 3.727854180824469, "tokens_seen": 802816000 }, { "epoch": 0.29, "learning_rate": 0.0003596794023075468, "loss": 2.4573, "theoretical_loss": 3.7263507213952978, "tokens_seen": 806092800 }, { "epoch": 0.29, "learning_rate": 0.00035908832986570834, "loss": 2.4972, "theoretical_loss": 3.724855064562658, "tokens_seen": 809369600 }, { "epoch": 0.29, "learning_rate": 0.0003584972574238699, "loss": 2.487, "theoretical_loss": 3.723367138488488, "tokens_seen": 812646400 }, { "epoch": 0.29, "learning_rate": 0.00035790618498203144, "loss": 2.5415, "theoretical_loss": 3.7218868722824014, "tokens_seen": 815923200 }, { "epoch": 0.29, "objective/train/docs_used": 416670, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 2.7136542797088623, "objective/train/theoretical_loss": 3.7204141959854384, "objective/train/tokens_used": 839660000, "theoretical_loss": 3.7204141959854384, "tokens_seen": 819200000 }, { "epoch": 0.29, "learning_rate": 0.00035731511254019296, "loss": 2.512, "theoretical_loss": 3.7204141959854384, "tokens_seen": 819200000 }, { "epoch": 0.29, "learning_rate": 0.0003567240400983545, "loss": 2.5015, "theoretical_loss": 3.718949040554162, "tokens_seen": 822476800 }, { "epoch": 0.29, "learning_rate": 0.000356132967656516, "loss": 2.5342, "theoretical_loss": 3.7174913378450833, "tokens_seen": 825753600 }, { "epoch": 0.3, "learning_rate": 0.0003555418952146775, "loss": 2.5043, "theoretical_loss": 3.7160410205994183, "tokens_seen": 829030400 }, { "epoch": 0.3, "learning_rate": 0.00035495082277283904, "loss": 2.477, "theoretical_loss": 3.7145980224281585, "tokens_seen": 832307200 }, { "epoch": 0.3, "learning_rate": 0.0003543597503310006, "loss": 2.4869, "theoretical_loss": 3.713162277797449, "tokens_seen": 835584000 }, { "epoch": 0.3, "learning_rate": 0.00035376867788916214, "loss": 2.4897, "theoretical_loss": 3.7117337220142748, "tokens_seen": 838860800 }, { "epoch": 0.3, "learning_rate": 0.00035317760544732366, "loss": 2.502, "theoretical_loss": 3.7103122912124364, "tokens_seen": 842137600 }, { "epoch": 0.3, "learning_rate": 0.0003525865330054852, "loss": 2.456, "theoretical_loss": 3.7088979223388128, "tokens_seen": 845414400 }, { "epoch": 0.3, "learning_rate": 0.00035199546056364665, "loss": 2.4775, "theoretical_loss": 3.70749055313991, "tokens_seen": 848691200 }, { "epoch": 0.3, "learning_rate": 0.00035140438812180817, "loss": 2.4525, "theoretical_loss": 3.7060901221486766, "tokens_seen": 851968000 }, { "epoch": 0.31, "learning_rate": 0.00035081331567996974, "loss": 2.4939, "theoretical_loss": 3.704696568671591, "tokens_seen": 855244800 }, { "epoch": 0.31, "learning_rate": 0.00035022224323813127, "loss": 2.4979, "theoretical_loss": 3.7033098327760063, "tokens_seen": 858521600 }, { "epoch": 0.31, "learning_rate": 0.0003496311707962928, "loss": 2.4961, "theoretical_loss": 3.7019298552777533, "tokens_seen": 861798400 }, { "epoch": 0.31, "learning_rate": 0.0003490400983544543, "loss": 2.517, "theoretical_loss": 3.700556577728988, "tokens_seen": 865075200 }, { "epoch": 0.31, "learning_rate": 0.00034844902591261583, "loss": 2.5434, "theoretical_loss": 3.6991899424062815, "tokens_seen": 868352000 }, { "epoch": 0.31, "learning_rate": 0.00034785795347077735, "loss": 2.5203, "theoretical_loss": 3.697829892298951, "tokens_seen": 871628800 }, { "epoch": 0.31, "learning_rate": 0.0003472668810289389, "loss": 2.5336, "theoretical_loss": 3.696476371097618, "tokens_seen": 874905600 }, { "epoch": 0.31, "learning_rate": 0.00034667580858710044, "loss": 2.5414, "theoretical_loss": 3.695129323182993, "tokens_seen": 878182400 }, { "epoch": 0.31, "learning_rate": 0.00034608473614526196, "loss": 2.5387, "theoretical_loss": 3.693788693614879, "tokens_seen": 881459200 }, { "epoch": 0.32, "learning_rate": 0.0003454936637034235, "loss": 2.564, "theoretical_loss": 3.6924544281213967, "tokens_seen": 884736000 }, { "epoch": 0.32, "learning_rate": 0.000344902591261585, "loss": 2.5277, "theoretical_loss": 3.691126473088412, "tokens_seen": 888012800 }, { "epoch": 0.32, "learning_rate": 0.00034431151881974653, "loss": 2.5106, "theoretical_loss": 3.689804775549173, "tokens_seen": 891289600 }, { "epoch": 0.32, "learning_rate": 0.00034372044637790805, "loss": 2.535, "theoretical_loss": 3.688489283174146, "tokens_seen": 894566400 }, { "epoch": 0.32, "learning_rate": 0.0003431293739360696, "loss": 2.5195, "theoretical_loss": 3.6871799442610538, "tokens_seen": 897843200 }, { "epoch": 0.32, "learning_rate": 0.00034253830149423114, "loss": 2.5172, "theoretical_loss": 3.685876707725093, "tokens_seen": 901120000 }, { "epoch": 0.32, "learning_rate": 0.00034194722905239266, "loss": 2.542, "theoretical_loss": 3.6845795230893517, "tokens_seen": 904396800 }, { "epoch": 0.32, "learning_rate": 0.0003413561566105542, "loss": 2.5073, "theoretical_loss": 3.6832883404754035, "tokens_seen": 907673600 }, { "epoch": 0.33, "learning_rate": 0.00034077690561755246, "loss": 2.5154, "theoretical_loss": 3.6820031105940796, "tokens_seen": 910950400 }, { "epoch": 0.33, "learning_rate": 0.000340185833175714, "loss": 2.5224, "theoretical_loss": 3.6807237847364176, "tokens_seen": 914227200 }, { "epoch": 0.33, "learning_rate": 0.00033959476073387555, "loss": 2.5197, "theoretical_loss": 3.6794503147647846, "tokens_seen": 917504000 }, { "epoch": 0.33, "learning_rate": 0.0003390036882920371, "loss": 2.4806, "theoretical_loss": 3.67818265310416, "tokens_seen": 920780800 }, { "epoch": 0.33, "learning_rate": 0.0003384126158501986, "loss": 2.4864, "theoretical_loss": 3.6769207527335888, "tokens_seen": 924057600 }, { "epoch": 0.33, "learning_rate": 0.0003378215434083601, "loss": 2.4467, "theoretical_loss": 3.675664567177787, "tokens_seen": 927334400 }, { "epoch": 0.33, "learning_rate": 0.00033723047096652164, "loss": 2.448, "theoretical_loss": 3.674414050498913, "tokens_seen": 930611200 }, { "epoch": 0.33, "learning_rate": 0.00033663939852468316, "loss": 2.4737, "theoretical_loss": 3.6731691572884824, "tokens_seen": 933888000 }, { "epoch": 0.33, "learning_rate": 0.00033604832608284473, "loss": 2.4584, "theoretical_loss": 3.671929842659438, "tokens_seen": 937164800 }, { "epoch": 0.34, "learning_rate": 0.00033545725364100625, "loss": 2.4622, "theoretical_loss": 3.6706960622383624, "tokens_seen": 940441600 }, { "epoch": 0.34, "learning_rate": 0.0003348661811991678, "loss": 2.4833, "theoretical_loss": 3.6694677721578377, "tokens_seen": 943718400 }, { "epoch": 0.34, "learning_rate": 0.0003342751087573293, "loss": 2.469, "theoretical_loss": 3.66824492904894, "tokens_seen": 946995200 }, { "epoch": 0.34, "learning_rate": 0.0003336840363154908, "loss": 2.4464, "theoretical_loss": 3.667027490033874, "tokens_seen": 950272000 }, { "epoch": 0.34, "learning_rate": 0.00033309296387365234, "loss": 2.4385, "theoretical_loss": 3.6658154127187412, "tokens_seen": 953548800 }, { "epoch": 0.34, "learning_rate": 0.00033250189143181386, "loss": 2.4387, "theoretical_loss": 3.664608655186437, "tokens_seen": 956825600 }, { "epoch": 0.34, "learning_rate": 0.00033191081898997543, "loss": 2.4551, "theoretical_loss": 3.663407175989679, "tokens_seen": 960102400 }, { "epoch": 0.34, "learning_rate": 0.00033131974654813695, "loss": 2.484, "theoretical_loss": 3.662210934144158, "tokens_seen": 963379200 }, { "epoch": 0.35, "learning_rate": 0.0003307286741062985, "loss": 2.4907, "theoretical_loss": 3.661019889121812, "tokens_seen": 966656000 }, { "epoch": 0.35, "learning_rate": 0.00033013760166446, "loss": 2.4687, "theoretical_loss": 3.6598340008442234, "tokens_seen": 969932800 }, { "epoch": 0.35, "learning_rate": 0.0003295465292226215, "loss": 2.5139, "theoretical_loss": 3.6586532296761285, "tokens_seen": 973209600 }, { "epoch": 0.35, "learning_rate": 0.00032895545678078304, "loss": 2.4666, "theoretical_loss": 3.657477536419047, "tokens_seen": 976486400 }, { "epoch": 0.35, "learning_rate": 0.0003283643843389446, "loss": 2.4923, "theoretical_loss": 3.656306882305022, "tokens_seen": 979763200 }, { "debugging/Self-BLEU-5": 0.5808769539451584, "debugging/distinct-1-grams": 0.8092227708966282, "debugging/distinct-2-grams": 0.969278580146789, "debugging/entropy-1-grams": 5.823343222362094, "debugging/entropy-2-grams": 6.624476320908173, "debugging/length": 581.2857142857143, "debugging/num_segments": 14, "epoch": 0.35, "objective/train/docs_used": 498736, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 2.4405508041381836, "objective/train/theoretical_loss": 3.6551412289904697, "objective/train/tokens_used": 1003500000, "theoretical_loss": 3.6551412289904697, "tokens_seen": 983040000 }, { "epoch": 0.35, "learning_rate": 0.00032777331189710613, "loss": 2.4773, "theoretical_loss": 3.6551412289904697, "tokens_seen": 983040000 }, { "epoch": 0.35, "learning_rate": 0.00032718223945526765, "loss": 2.4446, "theoretical_loss": 3.6539805385501376, "tokens_seen": 986316800 }, { "epoch": 0.35, "learning_rate": 0.0003265911670134292, "loss": 2.4604, "theoretical_loss": 3.652824773471171, "tokens_seen": 989593600 }, { "epoch": 0.35, "learning_rate": 0.0003260000945715907, "loss": 2.464, "theoretical_loss": 3.651673896647277, "tokens_seen": 992870400 }, { "epoch": 0.36, "learning_rate": 0.0003254090221297522, "loss": 2.4502, "theoretical_loss": 3.6505278713729985, "tokens_seen": 996147200 }, { "epoch": 0.36, "learning_rate": 0.0003248179496879138, "loss": 2.4302, "theoretical_loss": 3.6493866613380774, "tokens_seen": 999424000 }, { "epoch": 0.36, "learning_rate": 0.0003242268772460753, "loss": 2.4491, "theoretical_loss": 3.648250230621924, "tokens_seen": 1002700800 }, { "epoch": 0.36, "learning_rate": 0.00032363580480423683, "loss": 2.4729, "theoretical_loss": 3.647118543688179, "tokens_seen": 1005977600 }, { "epoch": 0.36, "learning_rate": 0.00032304473236239835, "loss": 2.4767, "theoretical_loss": 3.6459915653793633, "tokens_seen": 1009254400 }, { "epoch": 0.36, "learning_rate": 0.00032245365992055987, "loss": 2.4782, "theoretical_loss": 3.644869260911628, "tokens_seen": 1012531200 }, { "epoch": 0.36, "learning_rate": 0.0003218625874787214, "loss": 2.4723, "theoretical_loss": 3.64375159586959, "tokens_seen": 1015808000 }, { "epoch": 0.36, "learning_rate": 0.0003212715150368829, "loss": 2.4532, "theoretical_loss": 3.642638536201252, "tokens_seen": 1019084800 }, { "epoch": 0.37, "learning_rate": 0.0003206804425950445, "loss": 2.4654, "theoretical_loss": 3.6415300482130135, "tokens_seen": 1022361600 }, { "epoch": 0.37, "learning_rate": 0.000320089370153206, "loss": 2.4787, "theoretical_loss": 3.6404260985647667, "tokens_seen": 1025638400 }, { "epoch": 0.37, "learning_rate": 0.00031949829771136753, "loss": 2.4936, "theoretical_loss": 3.6393266542650684, "tokens_seen": 1028915200 }, { "epoch": 0.37, "learning_rate": 0.00031890722526952905, "loss": 2.4849, "theoretical_loss": 3.638231682666401, "tokens_seen": 1032192000 }, { "epoch": 0.37, "learning_rate": 0.00031831615282769057, "loss": 2.455, "theoretical_loss": 3.637141151460505, "tokens_seen": 1035468800 }, { "epoch": 0.37, "learning_rate": 0.0003177250803858521, "loss": 2.461, "theoretical_loss": 3.636055028673799, "tokens_seen": 1038745600 }, { "epoch": 0.37, "learning_rate": 0.00031713400794401367, "loss": 2.4815, "theoretical_loss": 3.634973282662864, "tokens_seen": 1042022400 }, { "epoch": 0.37, "learning_rate": 0.0003165429355021752, "loss": 2.462, "theoretical_loss": 3.6338958821100107, "tokens_seen": 1045299200 }, { "epoch": 0.37, "learning_rate": 0.0003159518630603367, "loss": 2.4218, "theoretical_loss": 3.63282279601892, "tokens_seen": 1048576000 }, { "epoch": 0.38, "learning_rate": 0.00031536079061849823, "loss": 2.435, "theoretical_loss": 3.631753993710352, "tokens_seen": 1051852800 }, { "epoch": 0.38, "learning_rate": 0.00031476971817665975, "loss": 2.4059, "theoretical_loss": 3.630689444817925, "tokens_seen": 1055129600 }, { "epoch": 0.38, "learning_rate": 0.0003141786457348212, "loss": 2.4132, "theoretical_loss": 3.629629119283967, "tokens_seen": 1058406400 }, { "epoch": 0.38, "learning_rate": 0.0003135875732929828, "loss": 2.4224, "theoretical_loss": 3.628572987355434, "tokens_seen": 1061683200 }, { "epoch": 0.38, "learning_rate": 0.0003129965008511443, "loss": 2.4123, "theoretical_loss": 3.6275210195798913, "tokens_seen": 1064960000 }, { "epoch": 0.38, "learning_rate": 0.00031240542840930583, "loss": 2.4143, "theoretical_loss": 3.626473186801564, "tokens_seen": 1068236800 }, { "epoch": 0.38, "learning_rate": 0.00031181435596746736, "loss": 2.4229, "theoretical_loss": 3.6254294601574495, "tokens_seen": 1071513600 }, { "epoch": 0.38, "learning_rate": 0.0003112232835256289, "loss": 2.4552, "theoretical_loss": 3.624389811073493, "tokens_seen": 1074790400 }, { "epoch": 0.39, "learning_rate": 0.0003106322110837904, "loss": 2.4635, "theoretical_loss": 3.6233542112608257, "tokens_seen": 1078067200 }, { "epoch": 0.39, "learning_rate": 0.0003100529600907888, "loss": 2.4215, "theoretical_loss": 3.6223226327120592, "tokens_seen": 1081344000 }, { "epoch": 0.39, "learning_rate": 0.0003094618876489503, "loss": 2.4313, "theoretical_loss": 3.621295047697644, "tokens_seen": 1084620800 }, { "epoch": 0.39, "learning_rate": 0.0003088708152071118, "loss": 2.4371, "theoretical_loss": 3.6202714287622833, "tokens_seen": 1087897600 }, { "epoch": 0.39, "learning_rate": 0.00030827974276527334, "loss": 2.4532, "theoretical_loss": 3.6192517487214038, "tokens_seen": 1091174400 }, { "epoch": 0.39, "learning_rate": 0.00030768867032343486, "loss": 2.462, "theoretical_loss": 3.6182359806576834, "tokens_seen": 1094451200 }, { "epoch": 0.39, "learning_rate": 0.0003070975978815964, "loss": 2.4922, "theoretical_loss": 3.6172240979176333, "tokens_seen": 1097728000 }, { "epoch": 0.39, "learning_rate": 0.0003065065254397579, "loss": 2.5429, "theoretical_loss": 3.616216074108232, "tokens_seen": 1101004800 }, { "epoch": 0.39, "learning_rate": 0.0003059154529979195, "loss": 2.5284, "theoretical_loss": 3.6152118830936164, "tokens_seen": 1104281600 }, { "epoch": 0.4, "learning_rate": 0.000305324380556081, "loss": 2.5163, "theoretical_loss": 3.6142114989918195, "tokens_seen": 1107558400 }, { "epoch": 0.4, "learning_rate": 0.00030473330811424246, "loss": 2.539, "theoretical_loss": 3.6132148961715624, "tokens_seen": 1110835200 }, { "epoch": 0.4, "learning_rate": 0.000304142235672404, "loss": 2.5355, "theoretical_loss": 3.6122220492490964, "tokens_seen": 1114112000 }, { "epoch": 0.4, "learning_rate": 0.0003035511632305655, "loss": 2.5071, "theoretical_loss": 3.6112329330850894, "tokens_seen": 1117388800 }, { "epoch": 0.4, "learning_rate": 0.000302960090788727, "loss": 2.5236, "theoretical_loss": 3.61024752278157, "tokens_seen": 1120665600 }, { "epoch": 0.4, "learning_rate": 0.0003023690183468886, "loss": 2.5306, "theoretical_loss": 3.6092657936789054, "tokens_seen": 1123942400 }, { "epoch": 0.4, "learning_rate": 0.0003017779459050501, "loss": 2.5418, "theoretical_loss": 3.6082877213528377, "tokens_seen": 1127219200 }, { "epoch": 0.4, "learning_rate": 0.00030118687346321164, "loss": 2.5057, "theoretical_loss": 3.60731328161156, "tokens_seen": 1130496000 }, { "epoch": 0.4, "learning_rate": 0.00030059580102137316, "loss": 2.4979, "theoretical_loss": 3.6063424504928365, "tokens_seen": 1133772800 }, { "epoch": 0.41, "learning_rate": 0.0003000047285795347, "loss": 2.4935, "theoretical_loss": 3.60537520426117, "tokens_seen": 1137049600 }, { "epoch": 0.41, "learning_rate": 0.0002994136561376962, "loss": 2.4755, "theoretical_loss": 3.6044115194050086, "tokens_seen": 1140326400 }, { "epoch": 0.41, "learning_rate": 0.0002988225836958578, "loss": 2.5173, "theoretical_loss": 3.603451372633997, "tokens_seen": 1143603200 }, { "epoch": 0.41, "objective/train/docs_used": 576676, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 2.600228786468506, "objective/train/theoretical_loss": 3.6024947408762698, "objective/train/tokens_used": 1167340000, "theoretical_loss": 3.6024947408762698, "tokens_seen": 1146880000 }, { "epoch": 0.41, "learning_rate": 0.0002982315112540193, "loss": 2.5011, "theoretical_loss": 3.6024947408762698, "tokens_seen": 1146880000 }, { "epoch": 0.41, "learning_rate": 0.0002976404388121808, "loss": 2.4895, "theoretical_loss": 3.601541601275783, "tokens_seen": 1150156800 }, { "epoch": 0.41, "learning_rate": 0.00029704936637034234, "loss": 2.4984, "theoretical_loss": 3.6005919311896886, "tokens_seen": 1153433600 }, { "epoch": 0.41, "learning_rate": 0.00029645829392850386, "loss": 2.4908, "theoretical_loss": 3.5996457081857454, "tokens_seen": 1156710400 }, { "epoch": 0.41, "learning_rate": 0.0002958672214866654, "loss": 2.5063, "theoretical_loss": 3.598702910039772, "tokens_seen": 1159987200 }, { "epoch": 0.42, "learning_rate": 0.0002952761490448269, "loss": 2.4946, "theoretical_loss": 3.597763514733133, "tokens_seen": 1163264000 }, { "epoch": 0.42, "learning_rate": 0.0002946850766029885, "loss": 2.48, "theoretical_loss": 3.59682750045027, "tokens_seen": 1166540800 }, { "epoch": 0.42, "learning_rate": 0.00029409400416115, "loss": 2.46, "theoretical_loss": 3.5958948455762583, "tokens_seen": 1169817600 }, { "epoch": 0.42, "learning_rate": 0.0002935029317193115, "loss": 2.4786, "theoretical_loss": 3.594965528694412, "tokens_seen": 1173094400 }, { "epoch": 0.42, "learning_rate": 0.00029291185927747304, "loss": 2.4519, "theoretical_loss": 3.594039528583913, "tokens_seen": 1176371200 }, { "epoch": 0.42, "learning_rate": 0.00029232078683563456, "loss": 2.437, "theoretical_loss": 3.5931168242174847, "tokens_seen": 1179648000 }, { "epoch": 0.42, "learning_rate": 0.0002917297143937961, "loss": 2.4535, "theoretical_loss": 3.59219739475909, "tokens_seen": 1182924800 }, { "epoch": 0.42, "learning_rate": 0.00029113864195195766, "loss": 2.4221, "theoretical_loss": 3.5912812195616732, "tokens_seen": 1186201600 }, { "epoch": 0.42, "learning_rate": 0.0002905475695101192, "loss": 2.4304, "theoretical_loss": 3.590368278164926, "tokens_seen": 1189478400 }, { "epoch": 0.43, "learning_rate": 0.0002899564970682807, "loss": 2.4601, "theoretical_loss": 3.5894585502930902, "tokens_seen": 1192755200 }, { "epoch": 0.43, "learning_rate": 0.0002893654246264422, "loss": 2.4571, "theoretical_loss": 3.588552015852793, "tokens_seen": 1196032000 }, { "epoch": 0.43, "learning_rate": 0.00028877435218460374, "loss": 2.4433, "theoretical_loss": 3.5876486549309097, "tokens_seen": 1199308800 }, { "epoch": 0.43, "learning_rate": 0.00028818327974276526, "loss": 2.4621, "theoretical_loss": 3.586748447792462, "tokens_seen": 1202585600 }, { "epoch": 0.43, "learning_rate": 0.00028759220730092684, "loss": 2.457, "theoretical_loss": 3.5858513748785423, "tokens_seen": 1205862400 }, { "epoch": 0.43, "learning_rate": 0.00028700113485908836, "loss": 2.4528, "theoretical_loss": 3.5849574168042704, "tokens_seen": 1209139200 }, { "epoch": 0.43, "learning_rate": 0.0002864100624172499, "loss": 2.4433, "theoretical_loss": 3.5840665543567782, "tokens_seen": 1212416000 }, { "epoch": 0.43, "learning_rate": 0.0002858189899754114, "loss": 2.4581, "theoretical_loss": 3.583178768493222, "tokens_seen": 1215692800 }, { "epoch": 0.44, "learning_rate": 0.0002852279175335729, "loss": 2.438, "theoretical_loss": 3.5822940403388284, "tokens_seen": 1218969600 }, { "epoch": 0.44, "learning_rate": 0.00028463684509173444, "loss": 2.4308, "theoretical_loss": 3.581412351184958, "tokens_seen": 1222246400 }, { "epoch": 0.44, "learning_rate": 0.00028404577264989596, "loss": 2.4334, "theoretical_loss": 3.580533682487208, "tokens_seen": 1225523200 }, { "epoch": 0.44, "learning_rate": 0.00028345470020805754, "loss": 2.4651, "theoretical_loss": 3.579658015863532, "tokens_seen": 1228800000 }, { "epoch": 0.44, "learning_rate": 0.00028286362776621906, "loss": 2.4806, "theoretical_loss": 3.5787853330923927, "tokens_seen": 1232076800 }, { "epoch": 0.44, "learning_rate": 0.0002822725553243806, "loss": 2.4682, "theoretical_loss": 3.577915616110936, "tokens_seen": 1235353600 }, { "epoch": 0.44, "learning_rate": 0.0002816814828825421, "loss": 2.4349, "theoretical_loss": 3.577048847013194, "tokens_seen": 1238630400 }, { "epoch": 0.44, "learning_rate": 0.0002810904104407036, "loss": 2.4327, "theoretical_loss": 3.57618500804831, "tokens_seen": 1241907200 }, { "epoch": 0.44, "learning_rate": 0.00028049933799886514, "loss": 2.4281, "theoretical_loss": 3.575324081618793, "tokens_seen": 1245184000 }, { "epoch": 0.45, "learning_rate": 0.0002799082655570267, "loss": 2.4293, "theoretical_loss": 3.5744660502787875, "tokens_seen": 1248460800 }, { "epoch": 0.45, "learning_rate": 0.00027931719311518824, "loss": 2.434, "theoretical_loss": 3.5736108967323794, "tokens_seen": 1251737600 }, { "epoch": 0.45, "learning_rate": 0.00027872612067334976, "loss": 2.4646, "theoretical_loss": 3.5727586038319155, "tokens_seen": 1255014400 }, { "epoch": 0.45, "learning_rate": 0.0002781350482315113, "loss": 2.4234, "theoretical_loss": 3.571909154576348, "tokens_seen": 1258291200 }, { "epoch": 0.45, "learning_rate": 0.0002775439757896728, "loss": 2.4295, "theoretical_loss": 3.5710625321096074, "tokens_seen": 1261568000 }, { "epoch": 0.45, "learning_rate": 0.0002769529033478343, "loss": 2.4476, "theoretical_loss": 3.570218719718989, "tokens_seen": 1264844800 }, { "epoch": 0.45, "learning_rate": 0.0002763618309059959, "loss": 2.4905, "theoretical_loss": 3.569377700833569, "tokens_seen": 1268121600 }, { "epoch": 0.45, "learning_rate": 0.00027577075846415736, "loss": 2.4667, "theoretical_loss": 3.568539459022639, "tokens_seen": 1271398400 }, { "epoch": 0.46, "learning_rate": 0.0002751796860223189, "loss": 2.4788, "theoretical_loss": 3.5677039779941584, "tokens_seen": 1274675200 }, { "epoch": 0.46, "learning_rate": 0.0002745886135804804, "loss": 2.4577, "theoretical_loss": 3.566871241593236, "tokens_seen": 1277952000 }, { "epoch": 0.46, "learning_rate": 0.0002739975411386419, "loss": 2.4777, "theoretical_loss": 3.5660412338006235, "tokens_seen": 1281228800 }, { "epoch": 0.46, "learning_rate": 0.00027340646869680345, "loss": 2.4122, "theoretical_loss": 3.565213938731236, "tokens_seen": 1284505600 }, { "epoch": 0.46, "learning_rate": 0.00027281539625496497, "loss": 2.4321, "theoretical_loss": 3.5643893406326868, "tokens_seen": 1287782400 }, { "epoch": 0.46, "learning_rate": 0.00027222432381312654, "loss": 2.4425, "theoretical_loss": 3.5635674238838466, "tokens_seen": 1291059200 }, { "epoch": 0.46, "learning_rate": 0.00027163325137128806, "loss": 2.4156, "theoretical_loss": 3.5627481729934196, "tokens_seen": 1294336000 }, { "epoch": 0.46, "learning_rate": 0.0002710421789294496, "loss": 2.4324, "theoretical_loss": 3.561931572598538, "tokens_seen": 1297612800 }, { "epoch": 0.46, "learning_rate": 0.0002704511064876111, "loss": 2.4463, "theoretical_loss": 3.5611176074633777, "tokens_seen": 1300889600 }, { "epoch": 0.47, "learning_rate": 0.0002698600340457726, "loss": 2.4452, "theoretical_loss": 3.5603062624777895, "tokens_seen": 1304166400 }, { "epoch": 0.47, "learning_rate": 0.00026926896160393415, "loss": 2.4555, "theoretical_loss": 3.559497522655951, "tokens_seen": 1307443200 }, { "debugging/Self-BLEU-5": 0.6399427451776495, "debugging/distinct-1-grams": 0.8197766779752171, "debugging/distinct-2-grams": 0.9705234588832752, "debugging/entropy-1-grams": 5.882641793405599, "debugging/entropy-2-grams": 6.547451268934809, "debugging/length": 584.7857142857143, "debugging/num_segments": 14, "epoch": 0.47, "objective/train/docs_used": 655957, "objective/train/instantaneous_batch_size": 32, "objective/train/instantaneous_microbatch_size": 32768, "objective/train/original_loss": 2.211557388305664, "objective/train/theoretical_loss": 3.5586913731350327, "objective/train/tokens_used": 1331180000, "theoretical_loss": 3.5586913731350327, "tokens_seen": 1310720000 }, { "epoch": 0.47, "learning_rate": 0.0002686778891620957, "loss": 2.413, "theoretical_loss": 3.5586913731350327, "tokens_seen": 1310720000 }, { "epoch": 0.47, "learning_rate": 0.00026808681672025724, "loss": 2.4279, "theoretical_loss": 3.557887799173889, "tokens_seen": 1313996800 }, { "epoch": 0.47, "learning_rate": 0.00026749574427841876, "loss": 2.4284, "theoretical_loss": 3.557086786151754, "tokens_seen": 1317273600 }, { "epoch": 0.47, "learning_rate": 0.0002669046718365803, "loss": 2.4231, "theoretical_loss": 3.5562883195669697, "tokens_seen": 1320550400 }, { "epoch": 0.47, "learning_rate": 0.0002663135993947418, "loss": 2.4112, "theoretical_loss": 3.555492385035719, "tokens_seen": 1323827200 }, { "epoch": 0.47, "learning_rate": 0.0002657225269529033, "loss": 2.4624, "theoretical_loss": 3.5546989682907784, "tokens_seen": 1327104000 }, { "epoch": 0.48, "learning_rate": 0.0002651314545110649, "loss": 2.4257, "theoretical_loss": 3.5539080551802895, "tokens_seen": 1330380800 }, { "epoch": 0.48, "learning_rate": 0.0002645403820692264, "loss": 2.4078, "theoretical_loss": 3.553119631666546, "tokens_seen": 1333657600 }, { "epoch": 0.48, "learning_rate": 0.00026394930962738794, "loss": 2.4365, "theoretical_loss": 3.5523336838247914, "tokens_seen": 1336934400 }, { "epoch": 0.48, "learning_rate": 0.00026335823718554946, "loss": 2.428, "theoretical_loss": 3.55155019784204, "tokens_seen": 1340211200 }, { "epoch": 0.48, "learning_rate": 0.000262767164743711, "loss": 2.4614, "theoretical_loss": 3.5507691600159053, "tokens_seen": 1343488000 }, { "epoch": 0.48, "learning_rate": 0.0002621760923018725, "loss": 2.4644, "theoretical_loss": 3.5499905567534515, "tokens_seen": 1346764800 }, { "epoch": 0.48, "learning_rate": 0.000261585019860034, "loss": 2.4448, "theoretical_loss": 3.549214374570052, "tokens_seen": 1350041600 }, { "epoch": 0.48, "learning_rate": 0.0002609939474181956, "loss": 2.4452, "theoretical_loss": 3.5484406000882665, "tokens_seen": 1353318400 }, { "epoch": 0.48, "learning_rate": 0.0002604028749763571, "loss": 2.462, "theoretical_loss": 3.5476692200367346, "tokens_seen": 1356595200 }, { "epoch": 0.49, "learning_rate": 0.00025981180253451864, "loss": 2.4537, "theoretical_loss": 3.546900221249076, "tokens_seen": 1359872000 }, { "epoch": 0.49, "learning_rate": 0.00025922073009268016, "loss": 2.4445, "theoretical_loss": 3.5461335906628157, "tokens_seen": 1363148800 }, { "epoch": 0.49, "learning_rate": 0.0002586296576508417, "loss": 2.4372, "theoretical_loss": 3.54536931531831, "tokens_seen": 1366425600 }, { "epoch": 0.49, "learning_rate": 0.0002580385852090032, "loss": 2.4444, "theoretical_loss": 3.5446073823576985, "tokens_seen": 1369702400 }, { "epoch": 0.49, "learning_rate": 0.0002574475127671648, "loss": 2.4888, "theoretical_loss": 3.543847779023859, "tokens_seen": 1372979200 }, { "epoch": 0.49, "learning_rate": 0.0002568564403253263, "loss": 2.4657, "theoretical_loss": 3.543090492659384, "tokens_seen": 1376256000 }, { "epoch": 0.49, "learning_rate": 0.0002562653678834878, "loss": 2.4599, "theoretical_loss": 3.542335510705562, "tokens_seen": 1379532800 }, { "epoch": 0.49, "learning_rate": 0.00025567429544164934, "loss": 2.4652, "theoretical_loss": 3.541582820701378, "tokens_seen": 1382809600 }, { "epoch": 0.5, "learning_rate": 0.00025508322299981086, "loss": 2.4831, "theoretical_loss": 3.5408324102825253, "tokens_seen": 1386086400 }, { "epoch": 0.5, "learning_rate": 0.0002544921505579724, "loss": 2.4716, "theoretical_loss": 3.5400842671804265, "tokens_seen": 1389363200 }, { "epoch": 0.5, "learning_rate": 0.00025390107811613396, "loss": 2.4304, "theoretical_loss": 3.5393383792212676, "tokens_seen": 1392640000 }, { "epoch": 0.5, "learning_rate": 0.0002533100056742955, "loss": 2.4272, "theoretical_loss": 3.5385947343250486, "tokens_seen": 1395916800 }, { "epoch": 0.5, "learning_rate": 0.000252718933232457, "loss": 2.437, "theoretical_loss": 3.53785332050464, "tokens_seen": 1399193600 } ], "max_steps": 42724, "num_train_epochs": 9223372036854775807, "total_flos": 7.14460209610752e+17, "trial_name": null, "trial_params": null }