| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.5, | |
| "global_step": 21362, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 1.1682242990654204e-06, | |
| "loss": 10.814, | |
| "theoretical_loss": 20.812802717066116, | |
| "tokens_seen": 65536 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 2.336448598130841e-05, | |
| "loss": 9.8574, | |
| "theoretical_loss": 10.363548859282295, | |
| "tokens_seen": 1310720 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 4.672897196261682e-05, | |
| "loss": 8.7288, | |
| "theoretical_loss": 8.96027440542832, | |
| "tokens_seen": 2621440 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 7.009345794392523e-05, | |
| "loss": 7.6795, | |
| "theoretical_loss": 8.257160837770828, | |
| "tokens_seen": 3932160 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 9.345794392523364e-05, | |
| "loss": 6.7353, | |
| "theoretical_loss": 7.80455017047313, | |
| "tokens_seen": 5242880 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00011682242990654206, | |
| "loss": 6.1167, | |
| "theoretical_loss": 7.477743633566995, | |
| "tokens_seen": 6553600 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00014018691588785047, | |
| "loss": 5.7605, | |
| "theoretical_loss": 7.225472152031461, | |
| "tokens_seen": 7864320 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00016355140186915886, | |
| "loss": 5.5757, | |
| "theoretical_loss": 7.022000360175642, | |
| "tokens_seen": 9175040 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00018691588785046728, | |
| "loss": 5.4219, | |
| "theoretical_loss": 6.852706072125594, | |
| "tokens_seen": 10485760 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.0002102803738317757, | |
| "loss": 5.2729, | |
| "theoretical_loss": 6.7085424020813775, | |
| "tokens_seen": 11796480 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 0.00023364485981308412, | |
| "loss": 5.1995, | |
| "theoretical_loss": 6.583551143945378, | |
| "tokens_seen": 13107200 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0002570093457943925, | |
| "loss": 5.0953, | |
| "theoretical_loss": 6.4736158466608344, | |
| "tokens_seen": 14417920 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00028037383177570094, | |
| "loss": 4.9747, | |
| "theoretical_loss": 6.375782617839931, | |
| "tokens_seen": 15728640 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00030373831775700936, | |
| "loss": 4.9164, | |
| "theoretical_loss": 6.287865514264695, | |
| "tokens_seen": 17039360 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0003271028037383177, | |
| "loss": 4.8585, | |
| "theoretical_loss": 6.208205077800142, | |
| "tokens_seen": 18350080 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00035046728971962614, | |
| "loss": 4.8139, | |
| "theoretical_loss": 6.13551414745929, | |
| "tokens_seen": 19660800 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00037383177570093456, | |
| "loss": 4.8348, | |
| "theoretical_loss": 6.068775822887412, | |
| "tokens_seen": 20971520 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000397196261682243, | |
| "loss": 4.7559, | |
| "theoretical_loss": 6.007173857995872, | |
| "tokens_seen": 22282240 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004205607476635514, | |
| "loss": 4.582, | |
| "theoretical_loss": 5.950043919222669, | |
| "tokens_seen": 23592960 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004439252336448598, | |
| "loss": 4.6037, | |
| "theoretical_loss": 5.89683866811525, | |
| "tokens_seen": 24903680 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.00046728971962616824, | |
| "loss": 4.5674, | |
| "theoretical_loss": 5.847102241784889, | |
| "tokens_seen": 26214400 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004906542056074767, | |
| "loss": 4.4639, | |
| "theoretical_loss": 5.80045126829951, | |
| "tokens_seen": 27525120 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004998581426139588, | |
| "loss": 4.4639, | |
| "theoretical_loss": 5.756560518460697, | |
| "tokens_seen": 28835840 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004996217136372233, | |
| "loss": 4.4888, | |
| "theoretical_loss": 5.715151906614886, | |
| "tokens_seen": 30146560 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000499385284660488, | |
| "loss": 4.4451, | |
| "theoretical_loss": 5.675985950013518, | |
| "tokens_seen": 31457280 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004991488556837526, | |
| "loss": 4.3658, | |
| "theoretical_loss": 5.638855059590318, | |
| "tokens_seen": 32768000 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004989124267070172, | |
| "loss": 4.2934, | |
| "theoretical_loss": 5.603578213244772, | |
| "tokens_seen": 34078720 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004986759977302818, | |
| "loss": 4.238, | |
| "theoretical_loss": 5.569996685496657, | |
| "tokens_seen": 35389440 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004984395687535465, | |
| "loss": 4.2585, | |
| "theoretical_loss": 5.53797059334095, | |
| "tokens_seen": 36700160 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.000498203139776811, | |
| "loss": 4.2223, | |
| "theoretical_loss": 5.507376079224074, | |
| "tokens_seen": 38010880 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004979667108000757, | |
| "loss": 4.1245, | |
| "theoretical_loss": 5.478102996075402, | |
| "tokens_seen": 39321600 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004977302818233403, | |
| "loss": 4.1117, | |
| "theoretical_loss": 5.450052991438716, | |
| "tokens_seen": 40632320 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "learning_rate": 0.0004974938528466049, | |
| "loss": 4.0718, | |
| "theoretical_loss": 5.4231379114505165, | |
| "tokens_seen": 41943040 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004972574238698695, | |
| "loss": 4.0939, | |
| "theoretical_loss": 5.397278463098006, | |
| "tokens_seen": 43253760 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004970209948931342, | |
| "loss": 4.07, | |
| "theoretical_loss": 5.372403086519811, | |
| "tokens_seen": 44564480 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004967845659163987, | |
| "loss": 4.0263, | |
| "theoretical_loss": 5.348446999254799, | |
| "tokens_seen": 45875200 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004965481369396634, | |
| "loss": 3.9504, | |
| "theoretical_loss": 5.3253513821291545, | |
| "tokens_seen": 47185920 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.000496311707962928, | |
| "loss": 3.9096, | |
| "theoretical_loss": 5.303062682496617, | |
| "tokens_seen": 48496640 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004960752789861926, | |
| "loss": 3.93, | |
| "theoretical_loss": 5.281532015245618, | |
| "tokens_seen": 49807360 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004958388500094572, | |
| "loss": 3.9081, | |
| "theoretical_loss": 5.260714645678606, | |
| "tokens_seen": 51118080 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004956024210327218, | |
| "loss": 3.8783, | |
| "theoretical_loss": 5.240569541288942, | |
| "tokens_seen": 52428800 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004953659920559864, | |
| "loss": 3.8688, | |
| "theoretical_loss": 5.221058981785759, | |
| "tokens_seen": 53739520 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.000495129563079251, | |
| "loss": 3.8196, | |
| "theoretical_loss": 5.202148218579597, | |
| "tokens_seen": 55050240 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004948931341025156, | |
| "loss": 3.794, | |
| "theoretical_loss": 5.1838051764423945, | |
| "tokens_seen": 56360960 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004946567051257802, | |
| "loss": 3.7921, | |
| "theoretical_loss": 5.166000191271213, | |
| "tokens_seen": 57671680 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004944202761490448, | |
| "loss": 3.7145, | |
| "theoretical_loss": 5.148705778875405, | |
| "tokens_seen": 58982400 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004941838471723094, | |
| "loss": 3.7021, | |
| "theoretical_loss": 5.131896430517482, | |
| "tokens_seen": 60293120 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.000493947418195574, | |
| "loss": 3.7415, | |
| "theoretical_loss": 5.1155484316045445, | |
| "tokens_seen": 61603840 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004937109892188386, | |
| "loss": 3.6037, | |
| "theoretical_loss": 5.099639700477905, | |
| "tokens_seen": 62914560 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004934745602421033, | |
| "loss": 3.6308, | |
| "theoretical_loss": 5.084149644705477, | |
| "tokens_seen": 64225280 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "objective/train/docs_used": 42180, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 3.620661973953247, | |
| "objective/train/theoretical_loss": 5.06905903266221, | |
| "objective/train/tokens_used": 85996000, | |
| "theoretical_loss": 5.06905903266221, | |
| "tokens_seen": 65536000 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004932381312653678, | |
| "loss": 3.6053, | |
| "theoretical_loss": 5.06905903266221, | |
| "tokens_seen": 65536000 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004930017022886325, | |
| "loss": 3.5318, | |
| "theoretical_loss": 5.054349878502332, | |
| "tokens_seen": 66846720 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004927652733118971, | |
| "loss": 3.5867, | |
| "theoretical_loss": 5.04000533889449, | |
| "tokens_seen": 68157440 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 0.0004925288443351617, | |
| "loss": 3.4731, | |
| "theoretical_loss": 5.02600962011627, | |
| "tokens_seen": 69468160 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004922924153584263, | |
| "loss": 3.4811, | |
| "theoretical_loss": 5.012347894295022, | |
| "tokens_seen": 70778880 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.000492055986381691, | |
| "loss": 3.4523, | |
| "theoretical_loss": 4.999006223743629, | |
| "tokens_seen": 72089600 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004918195574049555, | |
| "loss": 3.3867, | |
| "theoretical_loss": 4.985971492477399, | |
| "tokens_seen": 73400320 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004915831284282202, | |
| "loss": 3.4144, | |
| "theoretical_loss": 4.973231344115766, | |
| "tokens_seen": 74711040 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004913466994514848, | |
| "loss": 3.3577, | |
| "theoretical_loss": 4.960774125473073, | |
| "tokens_seen": 76021760 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004911102704747494, | |
| "loss": 3.3494, | |
| "theoretical_loss": 4.948588835229052, | |
| "tokens_seen": 77332480 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.000490873841498014, | |
| "loss": 3.3287, | |
| "theoretical_loss": 4.9366650771441005, | |
| "tokens_seen": 78643200 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004906374125212787, | |
| "loss": 3.3758, | |
| "theoretical_loss": 4.924993017348572, | |
| "tokens_seen": 79953920 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004904009835445432, | |
| "loss": 3.4314, | |
| "theoretical_loss": 4.913563345291011, | |
| "tokens_seen": 81264640 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004901645545678079, | |
| "loss": 3.3894, | |
| "theoretical_loss": 4.90236723797842, | |
| "tokens_seen": 82575360 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004899281255910725, | |
| "loss": 3.3281, | |
| "theoretical_loss": 4.891396327183692, | |
| "tokens_seen": 83886080 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004896916966143371, | |
| "loss": 3.3502, | |
| "theoretical_loss": 4.880642669331891, | |
| "tokens_seen": 85196800 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004894552676376017, | |
| "loss": 3.2405, | |
| "theoretical_loss": 4.8700987178090696, | |
| "tokens_seen": 86507520 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004892188386608663, | |
| "loss": 3.2437, | |
| "theoretical_loss": 4.859757297465288, | |
| "tokens_seen": 87818240 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004889824096841309, | |
| "loss": 3.2855, | |
| "theoretical_loss": 4.84961158110812, | |
| "tokens_seen": 89128960 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004887459807073955, | |
| "loss": 3.2293, | |
| "theoretical_loss": 4.839655067804491, | |
| "tokens_seen": 90439680 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004885095517306601, | |
| "loss": 3.2216, | |
| "theoretical_loss": 4.829881562827778, | |
| "tokens_seen": 91750400 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.0004882731227539247, | |
| "loss": 3.2223, | |
| "theoretical_loss": 4.820285159103877, | |
| "tokens_seen": 93061120 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.00048803669377718936, | |
| "loss": 3.2059, | |
| "theoretical_loss": 4.810860220024846, | |
| "tokens_seen": 94371840 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.00048780026480045396, | |
| "loss": 3.1818, | |
| "theoretical_loss": 4.801601363511853, | |
| "tokens_seen": 95682560 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "learning_rate": 0.00048756383582371855, | |
| "loss": 3.2274, | |
| "theoretical_loss": 4.792503447220913, | |
| "tokens_seen": 96993280 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004873274068469832, | |
| "loss": 3.1185, | |
| "theoretical_loss": 4.783561554795222, | |
| "tokens_seen": 98304000 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004870909778702478, | |
| "loss": 3.2279, | |
| "theoretical_loss": 4.774770983077213, | |
| "tokens_seen": 99614720 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004868545488935124, | |
| "loss": 3.2271, | |
| "theoretical_loss": 4.766127230201681, | |
| "tokens_seen": 100925440 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.000486618119916777, | |
| "loss": 3.1355, | |
| "theoretical_loss": 4.757625984498683, | |
| "tokens_seen": 102236160 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.00048638169094004165, | |
| "loss": 3.149, | |
| "theoretical_loss": 4.7492631141415815, | |
| "tokens_seen": 103546880 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004861452619633062, | |
| "loss": 3.1374, | |
| "theoretical_loss": 4.741034657481437, | |
| "tokens_seen": 104857600 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.00048590883298657084, | |
| "loss": 3.1645, | |
| "theoretical_loss": 4.732936814014305, | |
| "tokens_seen": 106168320 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004856724040098355, | |
| "loss": 3.1508, | |
| "theoretical_loss": 4.724965935932751, | |
| "tokens_seen": 107479040 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.00048543597503310004, | |
| "loss": 3.1337, | |
| "theoretical_loss": 4.717118520217186, | |
| "tokens_seen": 108789760 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004851995460563647, | |
| "loss": 3.1445, | |
| "theoretical_loss": 4.709391201226483, | |
| "tokens_seen": 110100480 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004849631170796293, | |
| "loss": 3.1398, | |
| "theoretical_loss": 4.70178074375083, | |
| "tokens_seen": 111411200 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004847266881028939, | |
| "loss": 3.1739, | |
| "theoretical_loss": 4.694284036492922, | |
| "tokens_seen": 112721920 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.00048449025912615854, | |
| "loss": 3.1498, | |
| "theoretical_loss": 4.686898085946441, | |
| "tokens_seen": 114032640 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.00048425383014942313, | |
| "loss": 3.1035, | |
| "theoretical_loss": 4.6796200106433545, | |
| "tokens_seen": 115343360 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.00048401740117268773, | |
| "loss": 3.12, | |
| "theoretical_loss": 4.6724470357438985, | |
| "tokens_seen": 116654080 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.00048378097219595233, | |
| "loss": 3.0993, | |
| "theoretical_loss": 4.665376487945246, | |
| "tokens_seen": 117964800 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.000483544543219217, | |
| "loss": 3.1133, | |
| "theoretical_loss": 4.658405790686763, | |
| "tokens_seen": 119275520 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004833081142424816, | |
| "loss": 3.066, | |
| "theoretical_loss": 4.6515324596315395, | |
| "tokens_seen": 120586240 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.0004830716852657462, | |
| "loss": 3.1408, | |
| "theoretical_loss": 4.6447540984054605, | |
| "tokens_seen": 121896960 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.00048283525628901083, | |
| "loss": 3.0971, | |
| "theoretical_loss": 4.638068394576544, | |
| "tokens_seen": 123207680 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.00048259882731227537, | |
| "loss": 3.0897, | |
| "theoretical_loss": 4.631473115858608, | |
| "tokens_seen": 124518400 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 0.00048236239833554, | |
| "loss": 3.0679, | |
| "theoretical_loss": 4.624966106524518, | |
| "tokens_seen": 125829120 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0004821259693588047, | |
| "loss": 3.1338, | |
| "theoretical_loss": 4.61854528401543, | |
| "tokens_seen": 127139840 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0004818895403820692, | |
| "loss": 3.1244, | |
| "theoretical_loss": 4.6122086357333725, | |
| "tokens_seen": 128450560 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00048165311140533387, | |
| "loss": 3.0844, | |
| "theoretical_loss": 4.605954216005544, | |
| "tokens_seen": 129761280 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "objective/train/docs_used": 74274, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 3.0012848377227783, | |
| "objective/train/theoretical_loss": 4.599780143209469, | |
| "objective/train/tokens_used": 151532000, | |
| "theoretical_loss": 4.599780143209469, | |
| "tokens_seen": 131072000 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00048141668242859847, | |
| "loss": 3.0701, | |
| "theoretical_loss": 4.599780143209469, | |
| "tokens_seen": 131072000 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00048118025345186306, | |
| "loss": 3.0507, | |
| "theoretical_loss": 4.593684597048995, | |
| "tokens_seen": 132382720 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0004809438244751277, | |
| "loss": 3.0452, | |
| "theoretical_loss": 4.587665815971789, | |
| "tokens_seen": 133693440 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0004807073954983923, | |
| "loss": 3.0615, | |
| "theoretical_loss": 4.581722094719703, | |
| "tokens_seen": 135004160 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0004804709665216569, | |
| "loss": 3.0187, | |
| "theoretical_loss": 4.575851782003932, | |
| "tokens_seen": 136314880 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0004802345375449215, | |
| "loss": 3.0266, | |
| "theoretical_loss": 4.570053278297481, | |
| "tokens_seen": 137625600 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00047999810856818616, | |
| "loss": 3.03, | |
| "theoretical_loss": 4.564325033737986, | |
| "tokens_seen": 138936320 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0004797616795914507, | |
| "loss": 3.0362, | |
| "theoretical_loss": 4.558665546134362, | |
| "tokens_seen": 140247040 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00047952525061471536, | |
| "loss": 3.072, | |
| "theoretical_loss": 4.5530733590712344, | |
| "tokens_seen": 141557760 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00047928882163798, | |
| "loss": 3.0484, | |
| "theoretical_loss": 4.547547060105504, | |
| "tokens_seen": 142868480 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00047905239266124455, | |
| "loss": 3.0597, | |
| "theoretical_loss": 4.542085279049763, | |
| "tokens_seen": 144179200 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0004788159636845092, | |
| "loss": 3.081, | |
| "theoretical_loss": 4.53668668633763, | |
| "tokens_seen": 145489920 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00047857953470777375, | |
| "loss": 3.0685, | |
| "theoretical_loss": 4.531349991466399, | |
| "tokens_seen": 146800640 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0004783431057310384, | |
| "loss": 3.0333, | |
| "theoretical_loss": 4.526073941512697, | |
| "tokens_seen": 148111360 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00047810667675430305, | |
| "loss": 3.0074, | |
| "theoretical_loss": 4.520857319717103, | |
| "tokens_seen": 149422080 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.0004778702477775676, | |
| "loss": 3.0174, | |
| "theoretical_loss": 4.515698944133961, | |
| "tokens_seen": 150732800 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00047763381880083224, | |
| "loss": 3.0253, | |
| "theoretical_loss": 4.510597666342842, | |
| "tokens_seen": 152043520 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "learning_rate": 0.00047739738982409684, | |
| "loss": 3.027, | |
| "theoretical_loss": 4.5055523702183375, | |
| "tokens_seen": 153354240 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047716096084736144, | |
| "loss": 3.0164, | |
| "theoretical_loss": 4.500561970755069, | |
| "tokens_seen": 154664960 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0004769245318706261, | |
| "loss": 2.9972, | |
| "theoretical_loss": 4.495625412944987, | |
| "tokens_seen": 155975680 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0004766881028938907, | |
| "loss": 3.031, | |
| "theoretical_loss": 4.490741670704231, | |
| "tokens_seen": 157286400 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0004764516739171553, | |
| "loss": 2.9952, | |
| "theoretical_loss": 4.485909745846939, | |
| "tokens_seen": 158597120 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0004762152449404199, | |
| "loss": 3.0018, | |
| "theoretical_loss": 4.481128667103624, | |
| "tokens_seen": 159907840 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047597881596368453, | |
| "loss": 2.9871, | |
| "theoretical_loss": 4.476397489181794, | |
| "tokens_seen": 161218560 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047574238698694913, | |
| "loss": 2.9838, | |
| "theoretical_loss": 4.471715291866703, | |
| "tokens_seen": 162529280 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047550595801021373, | |
| "loss": 2.9897, | |
| "theoretical_loss": 4.467081179160184, | |
| "tokens_seen": 163840000 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0004752695290334784, | |
| "loss": 2.9338, | |
| "theoretical_loss": 4.462494278455688, | |
| "tokens_seen": 165150720 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0004750331000567429, | |
| "loss": 2.9646, | |
| "theoretical_loss": 4.457953739747708, | |
| "tokens_seen": 166461440 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0004747966710800076, | |
| "loss": 2.9441, | |
| "theoretical_loss": 4.453458734873907, | |
| "tokens_seen": 167772160 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0004745602421032722, | |
| "loss": 2.9944, | |
| "theoretical_loss": 4.449008456788359, | |
| "tokens_seen": 169082880 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047432381312653677, | |
| "loss": 2.9409, | |
| "theoretical_loss": 4.4446021188643945, | |
| "tokens_seen": 170393600 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0004740873841498014, | |
| "loss": 2.9376, | |
| "theoretical_loss": 4.440238954225603, | |
| "tokens_seen": 171704320 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.000473850955173066, | |
| "loss": 2.9203, | |
| "theoretical_loss": 4.435918215103694, | |
| "tokens_seen": 173015040 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0004736145261963306, | |
| "loss": 3.0101, | |
| "theoretical_loss": 4.431639172221903, | |
| "tokens_seen": 174325760 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0004733780972195952, | |
| "loss": 2.9358, | |
| "theoretical_loss": 4.427401114202768, | |
| "tokens_seen": 175636480 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047314166824285987, | |
| "loss": 2.9913, | |
| "theoretical_loss": 4.4232033469991245, | |
| "tokens_seen": 176947200 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047290523926612446, | |
| "loss": 2.932, | |
| "theoretical_loss": 4.419045193347259, | |
| "tokens_seen": 178257920 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.00047266881028938906, | |
| "loss": 2.9443, | |
| "theoretical_loss": 4.414925992241184, | |
| "tokens_seen": 179568640 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 0.0004724323813126537, | |
| "loss": 2.9793, | |
| "theoretical_loss": 4.410845098427085, | |
| "tokens_seen": 180879360 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00047219595233591826, | |
| "loss": 2.9043, | |
| "theoretical_loss": 4.406801881917029, | |
| "tokens_seen": 182190080 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0004719595233591829, | |
| "loss": 2.9424, | |
| "theoretical_loss": 4.402795727521044, | |
| "tokens_seen": 183500800 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00047172309438244756, | |
| "loss": 2.9592, | |
| "theoretical_loss": 4.398826034396784, | |
| "tokens_seen": 184811520 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0004714866654057121, | |
| "loss": 2.959, | |
| "theoretical_loss": 4.394892215615966, | |
| "tokens_seen": 186122240 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00047125023642897675, | |
| "loss": 2.9762, | |
| "theoretical_loss": 4.390993697746873, | |
| "tokens_seen": 187432960 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00047101380745224135, | |
| "loss": 2.8908, | |
| "theoretical_loss": 4.387129920452183, | |
| "tokens_seen": 188743680 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00047077737847550595, | |
| "loss": 2.93, | |
| "theoretical_loss": 4.383300336101497, | |
| "tokens_seen": 190054400 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0004705409494987706, | |
| "loss": 2.9135, | |
| "theoretical_loss": 4.379504409397913, | |
| "tokens_seen": 191365120 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0004703045205220352, | |
| "loss": 2.9337, | |
| "theoretical_loss": 4.375741617018028, | |
| "tokens_seen": 192675840 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0004700680915452998, | |
| "loss": 2.9462, | |
| "theoretical_loss": 4.372011447264842, | |
| "tokens_seen": 193986560 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0004698316625685644, | |
| "loss": 2.8436, | |
| "theoretical_loss": 4.368313399732967, | |
| "tokens_seen": 195297280 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "objective/train/docs_used": 108556, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.698056221008301, | |
| "objective/train/theoretical_loss": 4.364646984985669, | |
| "objective/train/tokens_used": 217068000, | |
| "theoretical_loss": 4.364646984985669, | |
| "tokens_seen": 196608000 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00046959523359182905, | |
| "loss": 2.858, | |
| "theoretical_loss": 4.364646984985669, | |
| "tokens_seen": 196608000 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0004693588046150936, | |
| "loss": 2.8615, | |
| "theoretical_loss": 4.3610117242432125, | |
| "tokens_seen": 197918720 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00046912237563835824, | |
| "loss": 2.9244, | |
| "theoretical_loss": 4.357407149082071, | |
| "tokens_seen": 199229440 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0004688859466616229, | |
| "loss": 2.9059, | |
| "theoretical_loss": 4.353832801144535, | |
| "tokens_seen": 200540160 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00046864951768488744, | |
| "loss": 2.8454, | |
| "theoretical_loss": 4.350288231858301, | |
| "tokens_seen": 201850880 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0004684130887081521, | |
| "loss": 2.9088, | |
| "theoretical_loss": 4.346773002165636, | |
| "tokens_seen": 203161600 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0004681766597314167, | |
| "loss": 2.8966, | |
| "theoretical_loss": 4.343286682261728, | |
| "tokens_seen": 204472320 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.0004679402307546813, | |
| "loss": 2.8624, | |
| "theoretical_loss": 4.339828851341859, | |
| "tokens_seen": 205783040 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00046770380177794593, | |
| "loss": 2.8962, | |
| "theoretical_loss": 4.336399097357047, | |
| "tokens_seen": 207093760 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00046746737280121053, | |
| "loss": 2.8955, | |
| "theoretical_loss": 4.332997016777814, | |
| "tokens_seen": 208404480 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "learning_rate": 0.00046723094382447513, | |
| "loss": 2.911, | |
| "theoretical_loss": 4.329622214365776, | |
| "tokens_seen": 209715200 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0004669945148477397, | |
| "loss": 2.8899, | |
| "theoretical_loss": 4.326274302952741, | |
| "tokens_seen": 211025920 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0004667580858710044, | |
| "loss": 2.8486, | |
| "theoretical_loss": 4.322952903227008, | |
| "tokens_seen": 212336640 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.000466521656894269, | |
| "loss": 2.8928, | |
| "theoretical_loss": 4.319657643526625, | |
| "tokens_seen": 213647360 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.00046628522791753357, | |
| "loss": 2.888, | |
| "theoretical_loss": 4.316388159639304, | |
| "tokens_seen": 214958080 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0004660487989407982, | |
| "loss": 2.935, | |
| "theoretical_loss": 4.313144094608757, | |
| "tokens_seen": 216268800 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.00046581236996406277, | |
| "loss": 2.8594, | |
| "theoretical_loss": 4.3099250985472, | |
| "tokens_seen": 217579520 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0004655759409873274, | |
| "loss": 2.8501, | |
| "theoretical_loss": 4.306730828453813, | |
| "tokens_seen": 218890240 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.00046533951201059207, | |
| "loss": 2.8318, | |
| "theoretical_loss": 4.303560948038896, | |
| "tokens_seen": 220200960 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0004651030830338566, | |
| "loss": 2.8823, | |
| "theoretical_loss": 4.300415127553551, | |
| "tokens_seen": 221511680 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.00046486665405712127, | |
| "loss": 2.8325, | |
| "theoretical_loss": 4.297293043624652, | |
| "tokens_seen": 222822400 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.00046463022508038586, | |
| "loss": 2.8842, | |
| "theoretical_loss": 4.294194379094925, | |
| "tokens_seen": 224133120 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.00046439379610365046, | |
| "loss": 2.8242, | |
| "theoretical_loss": 4.291118822867954, | |
| "tokens_seen": 225443840 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0004641573671269151, | |
| "loss": 2.8608, | |
| "theoretical_loss": 4.288066069757916, | |
| "tokens_seen": 226754560 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0004639209381501797, | |
| "loss": 2.8391, | |
| "theoretical_loss": 4.285035820343889, | |
| "tokens_seen": 228065280 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0004636845091734443, | |
| "loss": 2.8652, | |
| "theoretical_loss": 4.28202778082858, | |
| "tokens_seen": 229376000 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0004634480801967089, | |
| "loss": 2.8379, | |
| "theoretical_loss": 4.27904166290127, | |
| "tokens_seen": 230686720 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.00046321165121997356, | |
| "loss": 2.876, | |
| "theoretical_loss": 4.276077183604883, | |
| "tokens_seen": 231997440 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0004629752222432381, | |
| "loss": 2.901, | |
| "theoretical_loss": 4.2731340652069925, | |
| "tokens_seen": 233308160 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.00046273879326650275, | |
| "loss": 2.8552, | |
| "theoretical_loss": 4.270212035074646, | |
| "tokens_seen": 234618880 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.0004625023642897674, | |
| "loss": 2.8517, | |
| "theoretical_loss": 4.267310825552862, | |
| "tokens_seen": 235929600 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 0.00046226593531303195, | |
| "loss": 2.8531, | |
| "theoretical_loss": 4.264430173846689, | |
| "tokens_seen": 237240320 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004620295063362966, | |
| "loss": 2.8823, | |
| "theoretical_loss": 4.261569821906684, | |
| "tokens_seen": 238551040 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004617930773595612, | |
| "loss": 2.8457, | |
| "theoretical_loss": 4.258729516317703, | |
| "tokens_seen": 239861760 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004615566483828258, | |
| "loss": 2.8403, | |
| "theoretical_loss": 4.255909008190896, | |
| "tokens_seen": 241172480 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.00046132021940609044, | |
| "loss": 2.8388, | |
| "theoretical_loss": 4.253108053058778, | |
| "tokens_seen": 242483200 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.00046108379042935504, | |
| "loss": 2.8376, | |
| "theoretical_loss": 4.250326410773302, | |
| "tokens_seen": 243793920 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.00046084736145261964, | |
| "loss": 2.851, | |
| "theoretical_loss": 4.247563845406793, | |
| "tokens_seen": 245104640 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.00046061093247588424, | |
| "loss": 2.8015, | |
| "theoretical_loss": 4.2448201251556945, | |
| "tokens_seen": 246415360 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004603745034991489, | |
| "loss": 2.8141, | |
| "theoretical_loss": 4.242095022246987, | |
| "tokens_seen": 247726080 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004601380745224135, | |
| "loss": 2.8206, | |
| "theoretical_loss": 4.239388312847225, | |
| "tokens_seen": 249036800 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004599016455456781, | |
| "loss": 2.8453, | |
| "theoretical_loss": 4.236699776974095, | |
| "tokens_seen": 250347520 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.00045966521656894274, | |
| "loss": 2.8398, | |
| "theoretical_loss": 4.234029198410394, | |
| "tokens_seen": 251658240 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004594287875922073, | |
| "loss": 2.8236, | |
| "theoretical_loss": 4.23137636462039, | |
| "tokens_seen": 252968960 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.00045919235861547193, | |
| "loss": 2.8607, | |
| "theoretical_loss": 4.228741066668439, | |
| "tokens_seen": 254279680 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004589559296387366, | |
| "loss": 2.9213, | |
| "theoretical_loss": 4.226123099139831, | |
| "tokens_seen": 255590400 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004587195006620011, | |
| "loss": 2.8963, | |
| "theoretical_loss": 4.223522260063753, | |
| "tokens_seen": 256901120 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004584830716852658, | |
| "loss": 2.9369, | |
| "theoretical_loss": 4.220938350838335, | |
| "tokens_seen": 258211840 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004582466427085304, | |
| "loss": 2.8518, | |
| "theoretical_loss": 4.218371176157692, | |
| "tokens_seen": 259522560 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.00045801021373179497, | |
| "loss": 2.8627, | |
| "theoretical_loss": 4.215820543940906, | |
| "tokens_seen": 260833280 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "objective/train/docs_used": 140861, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 3.0177669525146484, | |
| "objective/train/theoretical_loss": 4.213286265262889, | |
| "objective/train/tokens_used": 282604000, | |
| "theoretical_loss": 4.213286265262889, | |
| "tokens_seen": 262144000 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004577737847550596, | |
| "loss": 2.879, | |
| "theoretical_loss": 4.213286265262889, | |
| "tokens_seen": 262144000 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004575373557783242, | |
| "loss": 2.8798, | |
| "theoretical_loss": 4.210768154287058, | |
| "tokens_seen": 263454720 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "learning_rate": 0.0004573009268015888, | |
| "loss": 2.8486, | |
| "theoretical_loss": 4.208266028199782, | |
| "tokens_seen": 264765440 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0004570644978248534, | |
| "loss": 2.8631, | |
| "theoretical_loss": 4.205779707146535, | |
| "tokens_seen": 266076160 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.00045682806884811807, | |
| "loss": 2.7989, | |
| "theoretical_loss": 4.203309014169696, | |
| "tokens_seen": 267386880 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0004565916398713826, | |
| "loss": 2.8203, | |
| "theoretical_loss": 4.200853775147967, | |
| "tokens_seen": 268697600 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.00045635521089464726, | |
| "loss": 2.7936, | |
| "theoretical_loss": 4.198413818737336, | |
| "tokens_seen": 270008320 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0004561187819179119, | |
| "loss": 2.8047, | |
| "theoretical_loss": 4.195988976313554, | |
| "tokens_seen": 271319040 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.00045588235294117646, | |
| "loss": 2.7823, | |
| "theoretical_loss": 4.193579081916074, | |
| "tokens_seen": 272629760 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0004556459239644411, | |
| "loss": 2.8128, | |
| "theoretical_loss": 4.191183972193405, | |
| "tokens_seen": 273940480 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0004554094949877057, | |
| "loss": 2.8501, | |
| "theoretical_loss": 4.1888034863498484, | |
| "tokens_seen": 275251200 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0004551730660109703, | |
| "loss": 2.7955, | |
| "theoretical_loss": 4.186437466093564, | |
| "tokens_seen": 276561920 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.00045493663703423496, | |
| "loss": 2.7836, | |
| "theoretical_loss": 4.1840857555859365, | |
| "tokens_seen": 277872640 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0004547002080574995, | |
| "loss": 2.7591, | |
| "theoretical_loss": 4.181748201392195, | |
| "tokens_seen": 279183360 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.00045446377908076415, | |
| "loss": 2.8471, | |
| "theoretical_loss": 4.179424652433255, | |
| "tokens_seen": 280494080 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.00045422735010402875, | |
| "loss": 2.7665, | |
| "theoretical_loss": 4.177114959938745, | |
| "tokens_seen": 281804800 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.00045399092112729335, | |
| "loss": 2.843, | |
| "theoretical_loss": 4.174818977401182, | |
| "tokens_seen": 283115520 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.000453754492150558, | |
| "loss": 2.8071, | |
| "theoretical_loss": 4.172536560531271, | |
| "tokens_seen": 284426240 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0004535180631738226, | |
| "loss": 2.7906, | |
| "theoretical_loss": 4.170267567214273, | |
| "tokens_seen": 285736960 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0004532816341970872, | |
| "loss": 2.785, | |
| "theoretical_loss": 4.168011857467442, | |
| "tokens_seen": 287047680 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0004530452052203518, | |
| "loss": 2.7791, | |
| "theoretical_loss": 4.1657692933984825, | |
| "tokens_seen": 288358400 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.00045280877624361644, | |
| "loss": 2.8478, | |
| "theoretical_loss": 4.163539739164992, | |
| "tokens_seen": 289669120 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.00045257234726688104, | |
| "loss": 2.7897, | |
| "theoretical_loss": 4.161323060934883, | |
| "tokens_seen": 290979840 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.00045233591829014564, | |
| "loss": 2.779, | |
| "theoretical_loss": 4.159119126847729, | |
| "tokens_seen": 292290560 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 0.0004520994893134103, | |
| "loss": 2.8162, | |
| "theoretical_loss": 4.1569278069770395, | |
| "tokens_seen": 293601280 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.00045186306033667483, | |
| "loss": 2.7286, | |
| "theoretical_loss": 4.154748973293404, | |
| "tokens_seen": 294912000 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0004516266313599395, | |
| "loss": 2.6994, | |
| "theoretical_loss": 4.152582499628515, | |
| "tokens_seen": 296222720 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0004513902023832041, | |
| "loss": 2.7016, | |
| "theoretical_loss": 4.150428261640019, | |
| "tokens_seen": 297533440 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0004511537734064687, | |
| "loss": 2.7195, | |
| "theoretical_loss": 4.148286136777182, | |
| "tokens_seen": 298844160 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.00045091734442973333, | |
| "loss": 2.7063, | |
| "theoretical_loss": 4.1461560042473575, | |
| "tokens_seen": 300154880 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.00045068091545299793, | |
| "loss": 2.7723, | |
| "theoretical_loss": 4.144037744983214, | |
| "tokens_seen": 301465600 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0004504444864762625, | |
| "loss": 2.7161, | |
| "theoretical_loss": 4.141931241610721, | |
| "tokens_seen": 302776320 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0004502080574995271, | |
| "loss": 2.7333, | |
| "theoretical_loss": 4.139836378417861, | |
| "tokens_seen": 304087040 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0004499716285227918, | |
| "loss": 2.6842, | |
| "theoretical_loss": 4.1377530413240535, | |
| "tokens_seen": 305397760 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.00044973519954605637, | |
| "loss": 2.7274, | |
| "theoretical_loss": 4.135681117850265, | |
| "tokens_seen": 306708480 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.00044949877056932097, | |
| "loss": 2.6965, | |
| "theoretical_loss": 4.1336204970898, | |
| "tokens_seen": 308019200 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0004492623415925856, | |
| "loss": 2.7474, | |
| "theoretical_loss": 4.131571069679739, | |
| "tokens_seen": 309329920 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.00044902591261585016, | |
| "loss": 2.6866, | |
| "theoretical_loss": 4.129532727773022, | |
| "tokens_seen": 310640640 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0004487894836391148, | |
| "loss": 2.7461, | |
| "theoretical_loss": 4.127505365011135, | |
| "tokens_seen": 311951360 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.00044855305466237947, | |
| "loss": 2.7956, | |
| "theoretical_loss": 4.125488876497428, | |
| "tokens_seen": 313262080 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.000448316625685644, | |
| "loss": 2.7507, | |
| "theoretical_loss": 4.123483158770997, | |
| "tokens_seen": 314572800 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.00044808019670890866, | |
| "loss": 2.7164, | |
| "theoretical_loss": 4.121488109781147, | |
| "tokens_seen": 315883520 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.00044784376773217326, | |
| "loss": 2.6818, | |
| "theoretical_loss": 4.11950362886242, | |
| "tokens_seen": 317194240 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.00044760733875543786, | |
| "loss": 2.7681, | |
| "theoretical_loss": 4.1175296167101525, | |
| "tokens_seen": 318504960 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0004473709097787025, | |
| "loss": 2.7417, | |
| "theoretical_loss": 4.115565975356576, | |
| "tokens_seen": 319815680 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "learning_rate": 0.0004471344808019671, | |
| "loss": 2.7462, | |
| "theoretical_loss": 4.113612608147429, | |
| "tokens_seen": 321126400 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0004468980518252317, | |
| "loss": 2.7194, | |
| "theoretical_loss": 4.111669419719063, | |
| "tokens_seen": 322437120 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0004466616228484963, | |
| "loss": 2.7836, | |
| "theoretical_loss": 4.109736315976054, | |
| "tokens_seen": 323747840 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.00044642519387176095, | |
| "loss": 2.7677, | |
| "theoretical_loss": 4.107813204069279, | |
| "tokens_seen": 325058560 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0004461887648950255, | |
| "loss": 2.765, | |
| "theoretical_loss": 4.105899992374464, | |
| "tokens_seen": 326369280 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.5443408244790342, | |
| "debugging/distinct-1-grams": 0.7943066470015702, | |
| "debugging/distinct-2-grams": 0.9495545460213378, | |
| "debugging/entropy-1-grams": 6.188544372388533, | |
| "debugging/entropy-2-grams": 7.017444768234691, | |
| "debugging/length": 488.2, | |
| "debugging/num_segments": 20, | |
| "epoch": 0.12, | |
| "objective/train/docs_used": 173476, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.9443421363830566, | |
| "objective/train/theoretical_loss": 4.103996590471185, | |
| "objective/train/tokens_used": 348140000, | |
| "theoretical_loss": 4.103996590471185, | |
| "tokens_seen": 327680000 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.00044595233591829015, | |
| "loss": 2.7498, | |
| "theoretical_loss": 4.103996590471185, | |
| "tokens_seen": 327680000 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0004457159069415548, | |
| "loss": 2.754, | |
| "theoretical_loss": 4.102102909122308, | |
| "tokens_seen": 328990720 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.00044547947796481934, | |
| "loss": 2.7594, | |
| "theoretical_loss": 4.1002188602538645, | |
| "tokens_seen": 330301440 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.000445243048988084, | |
| "loss": 2.7563, | |
| "theoretical_loss": 4.098344356935348, | |
| "tokens_seen": 331612160 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0004450066200113486, | |
| "loss": 2.7293, | |
| "theoretical_loss": 4.096479313360415, | |
| "tokens_seen": 332922880 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0004447701910346132, | |
| "loss": 2.7519, | |
| "theoretical_loss": 4.094623644827995, | |
| "tokens_seen": 334233600 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.00044453376205787784, | |
| "loss": 2.7473, | |
| "theoretical_loss": 4.092777267723779, | |
| "tokens_seen": 335544320 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.00044429733308114244, | |
| "loss": 2.737, | |
| "theoretical_loss": 4.0909400995021, | |
| "tokens_seen": 336855040 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.00044406090410440704, | |
| "loss": 2.753, | |
| "theoretical_loss": 4.08911205866818, | |
| "tokens_seen": 338165760 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.00044382447512767163, | |
| "loss": 2.7609, | |
| "theoretical_loss": 4.087293064760739, | |
| "tokens_seen": 339476480 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0004435880461509363, | |
| "loss": 2.7481, | |
| "theoretical_loss": 4.0854830383349565, | |
| "tokens_seen": 340787200 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0004433516171742009, | |
| "loss": 2.7475, | |
| "theoretical_loss": 4.083681900945785, | |
| "tokens_seen": 342097920 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0004431151881974655, | |
| "loss": 2.7237, | |
| "theoretical_loss": 4.081889575131589, | |
| "tokens_seen": 343408640 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.00044287875922073013, | |
| "loss": 2.6944, | |
| "theoretical_loss": 4.080105984398123, | |
| "tokens_seen": 344719360 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0004426423302439947, | |
| "loss": 2.7102, | |
| "theoretical_loss": 4.078331053202827, | |
| "tokens_seen": 346030080 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0004424059012672593, | |
| "loss": 2.6958, | |
| "theoretical_loss": 4.076564706939434, | |
| "tokens_seen": 347340800 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.000442169472290524, | |
| "loss": 2.7259, | |
| "theoretical_loss": 4.074806871922881, | |
| "tokens_seen": 348651520 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "learning_rate": 0.0004419330433137885, | |
| "loss": 2.6776, | |
| "theoretical_loss": 4.0730574753745294, | |
| "tokens_seen": 349962240 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0004416966143370532, | |
| "loss": 2.6618, | |
| "theoretical_loss": 4.071316445407663, | |
| "tokens_seen": 351272960 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00044146018536031777, | |
| "loss": 2.6653, | |
| "theoretical_loss": 4.06958371101328, | |
| "tokens_seen": 352583680 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00044122375638358237, | |
| "loss": 2.6613, | |
| "theoretical_loss": 4.067859202046162, | |
| "tokens_seen": 353894400 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.000440987327406847, | |
| "loss": 2.6834, | |
| "theoretical_loss": 4.066142849211214, | |
| "tokens_seen": 355205120 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0004407508984301116, | |
| "loss": 2.6939, | |
| "theoretical_loss": 4.064434584050067, | |
| "tokens_seen": 356515840 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0004405144694533762, | |
| "loss": 2.6643, | |
| "theoretical_loss": 4.062734338927946, | |
| "tokens_seen": 357826560 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0004402780404766408, | |
| "loss": 2.712, | |
| "theoretical_loss": 4.061042047020785, | |
| "tokens_seen": 359137280 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00044004161149990546, | |
| "loss": 2.6548, | |
| "theoretical_loss": 4.0593576423025945, | |
| "tokens_seen": 360448000 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00043980518252317, | |
| "loss": 2.6663, | |
| "theoretical_loss": 4.057681059533071, | |
| "tokens_seen": 361758720 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00043956875354643466, | |
| "loss": 2.6802, | |
| "theoretical_loss": 4.056012234245436, | |
| "tokens_seen": 363069440 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0004393323245696993, | |
| "loss": 2.6734, | |
| "theoretical_loss": 4.05435110273452, | |
| "tokens_seen": 364380160 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00043909589559296385, | |
| "loss": 2.6716, | |
| "theoretical_loss": 4.052697602045054, | |
| "tokens_seen": 365690880 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0004388594666162285, | |
| "loss": 2.6507, | |
| "theoretical_loss": 4.0510516699602, | |
| "tokens_seen": 367001600 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0004386230376394931, | |
| "loss": 2.6149, | |
| "theoretical_loss": 4.049413244990284, | |
| "tokens_seen": 368312320 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0004383866086627577, | |
| "loss": 2.6752, | |
| "theoretical_loss": 4.04778226636175, | |
| "tokens_seen": 369623040 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00043815017968602235, | |
| "loss": 2.684, | |
| "theoretical_loss": 4.046158674006312, | |
| "tokens_seen": 370933760 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00043791375070928695, | |
| "loss": 2.6793, | |
| "theoretical_loss": 4.044542408550312, | |
| "tokens_seen": 372244480 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00043767732173255155, | |
| "loss": 2.7291, | |
| "theoretical_loss": 4.042933411304277, | |
| "tokens_seen": 373555200 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.00043744089275581614, | |
| "loss": 2.7191, | |
| "theoretical_loss": 4.041331624252669, | |
| "tokens_seen": 374865920 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0004372044637790808, | |
| "loss": 2.7378, | |
| "theoretical_loss": 4.039736990043815, | |
| "tokens_seen": 376176640 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 0.0004369680348023454, | |
| "loss": 2.6811, | |
| "theoretical_loss": 4.038149451980039, | |
| "tokens_seen": 377487360 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.00043673160582561, | |
| "loss": 2.676, | |
| "theoretical_loss": 4.0365689540079535, | |
| "tokens_seen": 378798080 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.00043649517684887464, | |
| "loss": 2.6898, | |
| "theoretical_loss": 4.034995440708947, | |
| "tokens_seen": 380108800 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0004362587478721392, | |
| "loss": 2.7186, | |
| "theoretical_loss": 4.033428857289831, | |
| "tokens_seen": 381419520 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.00043602231889540384, | |
| "loss": 2.7021, | |
| "theoretical_loss": 4.031869149573668, | |
| "tokens_seen": 382730240 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0004357858899186685, | |
| "loss": 2.6748, | |
| "theoretical_loss": 4.030316263990749, | |
| "tokens_seen": 384040960 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.00043554946094193303, | |
| "loss": 2.6475, | |
| "theoretical_loss": 4.028770147569752, | |
| "tokens_seen": 385351680 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0004353130319651977, | |
| "loss": 2.674, | |
| "theoretical_loss": 4.0272307479290435, | |
| "tokens_seen": 386662400 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0004350766029884623, | |
| "loss": 2.6497, | |
| "theoretical_loss": 4.025698013268144, | |
| "tokens_seen": 387973120 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0004348401740117269, | |
| "loss": 2.6448, | |
| "theoretical_loss": 4.024171892359345, | |
| "tokens_seen": 389283840 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.00043460374503499153, | |
| "loss": 2.6957, | |
| "theoretical_loss": 4.0226523345394645, | |
| "tokens_seen": 390594560 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.00043436731605825613, | |
| "loss": 2.6991, | |
| "theoretical_loss": 4.021139289701768, | |
| "tokens_seen": 391905280 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "objective/train/docs_used": 207277, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.730978488922119, | |
| "objective/train/theoretical_loss": 4.019632708288012, | |
| "objective/train/tokens_used": 413676000, | |
| "theoretical_loss": 4.019632708288012, | |
| "tokens_seen": 393216000 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0004341308870815207, | |
| "loss": 2.703, | |
| "theoretical_loss": 4.019632708288012, | |
| "tokens_seen": 393216000 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0004338944581047853, | |
| "loss": 2.6495, | |
| "theoretical_loss": 4.018132541280638, | |
| "tokens_seen": 394526720 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.00043365802912805, | |
| "loss": 2.6888, | |
| "theoretical_loss": 4.016638740195099, | |
| "tokens_seen": 395837440 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0004334216001513145, | |
| "loss": 2.6689, | |
| "theoretical_loss": 4.015151257072327, | |
| "tokens_seen": 397148160 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.00043318517117457917, | |
| "loss": 2.6964, | |
| "theoretical_loss": 4.013670044471318, | |
| "tokens_seen": 398458880 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0004329487421978438, | |
| "loss": 2.7314, | |
| "theoretical_loss": 4.012195055461859, | |
| "tokens_seen": 399769600 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.00043271231322110837, | |
| "loss": 2.7078, | |
| "theoretical_loss": 4.010726243617374, | |
| "tokens_seen": 401080320 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.000432475884244373, | |
| "loss": 2.7017, | |
| "theoretical_loss": 4.009263563007895, | |
| "tokens_seen": 402391040 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0004322394552676376, | |
| "loss": 2.7293, | |
| "theoretical_loss": 4.007806968193149, | |
| "tokens_seen": 403701760 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "learning_rate": 0.0004320030262909022, | |
| "loss": 2.6724, | |
| "theoretical_loss": 4.006356414215775, | |
| "tokens_seen": 405012480 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.00043176659731416686, | |
| "loss": 2.6728, | |
| "theoretical_loss": 4.004911856594639, | |
| "tokens_seen": 406323200 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004315301683374314, | |
| "loss": 2.6839, | |
| "theoretical_loss": 4.00347325131828, | |
| "tokens_seen": 407633920 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.00043129373936069606, | |
| "loss": 2.6871, | |
| "theoretical_loss": 4.002040554838458, | |
| "tokens_seen": 408944640 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.00043105731038396066, | |
| "loss": 2.6934, | |
| "theoretical_loss": 4.000613724063812, | |
| "tokens_seen": 410255360 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.00043082088140722525, | |
| "loss": 2.6553, | |
| "theoretical_loss": 3.9991927163536243, | |
| "tokens_seen": 411566080 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004305844524304899, | |
| "loss": 2.6389, | |
| "theoretical_loss": 3.997777489511691, | |
| "tokens_seen": 412876800 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004303480234537545, | |
| "loss": 2.6955, | |
| "theoretical_loss": 3.9963680017802945, | |
| "tokens_seen": 414187520 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004301115944770191, | |
| "loss": 2.6909, | |
| "theoretical_loss": 3.9949642118342794, | |
| "tokens_seen": 415498240 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004298751655002837, | |
| "loss": 2.6615, | |
| "theoretical_loss": 3.993566078775218, | |
| "tokens_seen": 416808960 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.00042963873652354835, | |
| "loss": 2.6478, | |
| "theoretical_loss": 3.9921735621256835, | |
| "tokens_seen": 418119680 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.00042940230754681295, | |
| "loss": 2.6654, | |
| "theoretical_loss": 3.990786621823611, | |
| "tokens_seen": 419430400 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.00042916587857007754, | |
| "loss": 2.6674, | |
| "theoretical_loss": 3.9894052182167536, | |
| "tokens_seen": 420741120 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004289294495933422, | |
| "loss": 2.6418, | |
| "theoretical_loss": 3.988029312057227, | |
| "tokens_seen": 422051840 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.00042869302061660674, | |
| "loss": 2.6856, | |
| "theoretical_loss": 3.986658864496149, | |
| "tokens_seen": 423362560 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004284565916398714, | |
| "loss": 2.5969, | |
| "theoretical_loss": 3.985293837078359, | |
| "tokens_seen": 424673280 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.000428220162663136, | |
| "loss": 2.6481, | |
| "theoretical_loss": 3.9839341917372293, | |
| "tokens_seen": 425984000 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004279837336864006, | |
| "loss": 2.622, | |
| "theoretical_loss": 3.9825798907895598, | |
| "tokens_seen": 427294720 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.00042774730470966524, | |
| "loss": 2.6051, | |
| "theoretical_loss": 3.9812308969305508, | |
| "tokens_seen": 428605440 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.00042751087573292984, | |
| "loss": 2.6218, | |
| "theoretical_loss": 3.9798871732288617, | |
| "tokens_seen": 429916160 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.00042727444675619443, | |
| "loss": 2.6252, | |
| "theoretical_loss": 3.9785486831217467, | |
| "tokens_seen": 431226880 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.00042703801777945903, | |
| "loss": 2.6805, | |
| "theoretical_loss": 3.9772153904102705, | |
| "tokens_seen": 432537600 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 0.0004268015888027237, | |
| "loss": 2.6644, | |
| "theoretical_loss": 3.9758872592545957, | |
| "tokens_seen": 433848320 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0004265651598259883, | |
| "loss": 2.6639, | |
| "theoretical_loss": 3.974564254169353, | |
| "tokens_seen": 435159040 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0004263287308492529, | |
| "loss": 2.6058, | |
| "theoretical_loss": 3.9732463400190756, | |
| "tokens_seen": 436469760 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042609230187251753, | |
| "loss": 2.6543, | |
| "theoretical_loss": 3.9719334820137124, | |
| "tokens_seen": 437780480 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042585587289578207, | |
| "loss": 2.6281, | |
| "theoretical_loss": 3.9706256457042133, | |
| "tokens_seen": 439091200 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0004256194439190467, | |
| "loss": 2.6488, | |
| "theoretical_loss": 3.969322796978175, | |
| "tokens_seen": 440401920 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0004253830149423114, | |
| "loss": 2.6534, | |
| "theoretical_loss": 3.968024902055565, | |
| "tokens_seen": 441712640 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0004251465859655759, | |
| "loss": 2.6554, | |
| "theoretical_loss": 3.966731927484508, | |
| "tokens_seen": 443023360 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042491015698884057, | |
| "loss": 2.6865, | |
| "theoretical_loss": 3.9654438401371372, | |
| "tokens_seen": 444334080 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042467372801210517, | |
| "loss": 2.674, | |
| "theoretical_loss": 3.9641606072055113, | |
| "tokens_seen": 445644800 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042443729903536976, | |
| "loss": 2.6204, | |
| "theoretical_loss": 3.9628821961975955, | |
| "tokens_seen": 446955520 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0004242008700586344, | |
| "loss": 2.6579, | |
| "theoretical_loss": 3.9616085749333028, | |
| "tokens_seen": 448266240 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.000423964441081899, | |
| "loss": 2.6452, | |
| "theoretical_loss": 3.9603397115406005, | |
| "tokens_seen": 449576960 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0004237280121051636, | |
| "loss": 2.665, | |
| "theoretical_loss": 3.9590755744516684, | |
| "tokens_seen": 450887680 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0004234915831284282, | |
| "loss": 2.7374, | |
| "theoretical_loss": 3.9578161323991243, | |
| "tokens_seen": 452198400 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042325515415169286, | |
| "loss": 2.7473, | |
| "theoretical_loss": 3.9565613544123046, | |
| "tokens_seen": 453509120 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0004230187251749574, | |
| "loss": 2.7422, | |
| "theoretical_loss": 3.955311209813596, | |
| "tokens_seen": 454819840 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042278229619822206, | |
| "loss": 2.6709, | |
| "theoretical_loss": 3.9540656682148327, | |
| "tokens_seen": 456130560 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0004225458672214867, | |
| "loss": 2.6763, | |
| "theoretical_loss": 3.952824699513741, | |
| "tokens_seen": 457441280 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "objective/train/docs_used": 239034, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.5824592113494873, | |
| "objective/train/theoretical_loss": 3.951588273890438, | |
| "objective/train/tokens_used": 479212000, | |
| "theoretical_loss": 3.951588273890438, | |
| "tokens_seen": 458752000 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.00042230943824475125, | |
| "loss": 2.7259, | |
| "theoretical_loss": 3.951588273890438, | |
| "tokens_seen": 458752000 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0004220730092680159, | |
| "loss": 2.6383, | |
| "theoretical_loss": 3.950356361803988, | |
| "tokens_seen": 460062720 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "learning_rate": 0.0004218365802912805, | |
| "loss": 2.6565, | |
| "theoretical_loss": 3.949128933989004, | |
| "tokens_seen": 461373440 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0004216001513145451, | |
| "loss": 2.7145, | |
| "theoretical_loss": 3.9479059614523058, | |
| "tokens_seen": 462684160 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.00042136372233780975, | |
| "loss": 2.716, | |
| "theoretical_loss": 3.9466874154696234, | |
| "tokens_seen": 463994880 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.00042112729336107435, | |
| "loss": 2.6788, | |
| "theoretical_loss": 3.945473267582355, | |
| "tokens_seen": 465305600 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.00042089086438433894, | |
| "loss": 2.7163, | |
| "theoretical_loss": 3.9442634895943662, | |
| "tokens_seen": 466616320 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.00042065443540760354, | |
| "loss": 2.6988, | |
| "theoretical_loss": 3.943058053568841, | |
| "tokens_seen": 467927040 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0004204180064308682, | |
| "loss": 2.701, | |
| "theoretical_loss": 3.94185693182518, | |
| "tokens_seen": 469237760 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0004201815774541328, | |
| "loss": 2.735, | |
| "theoretical_loss": 3.9406600969359413, | |
| "tokens_seen": 470548480 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0004199451484773974, | |
| "loss": 2.6981, | |
| "theoretical_loss": 3.9394675217238277, | |
| "tokens_seen": 471859200 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.00041970871950066204, | |
| "loss": 2.6811, | |
| "theoretical_loss": 3.9382791792587195, | |
| "tokens_seen": 473169920 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0004194722905239266, | |
| "loss": 2.6656, | |
| "theoretical_loss": 3.9370950428547475, | |
| "tokens_seen": 474480640 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.00041923586154719123, | |
| "loss": 2.6925, | |
| "theoretical_loss": 3.9359150860674106, | |
| "tokens_seen": 475791360 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0004189994325704559, | |
| "loss": 2.6785, | |
| "theoretical_loss": 3.9347392826907353, | |
| "tokens_seen": 477102080 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.00041876300359372043, | |
| "loss": 2.6739, | |
| "theoretical_loss": 3.933567606754476, | |
| "tokens_seen": 478412800 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0004185265746169851, | |
| "loss": 2.6884, | |
| "theoretical_loss": 3.9324000325213557, | |
| "tokens_seen": 479723520 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0004182901456402497, | |
| "loss": 2.636, | |
| "theoretical_loss": 3.931236534484346, | |
| "tokens_seen": 481034240 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0004180537166635143, | |
| "loss": 2.6669, | |
| "theoretical_loss": 3.930077087363987, | |
| "tokens_seen": 482344960 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.00041781728768677893, | |
| "loss": 2.6797, | |
| "theoretical_loss": 3.928921666105744, | |
| "tokens_seen": 483655680 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0004175808587100435, | |
| "loss": 2.6506, | |
| "theoretical_loss": 3.9277702458774058, | |
| "tokens_seen": 484966400 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0004173444297333081, | |
| "loss": 2.6612, | |
| "theoretical_loss": 3.926622802066514, | |
| "tokens_seen": 486277120 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.0004171080007565727, | |
| "loss": 2.6903, | |
| "theoretical_loss": 3.925479310277835, | |
| "tokens_seen": 487587840 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 0.00041687157177983737, | |
| "loss": 2.6531, | |
| "theoretical_loss": 3.924339746330864, | |
| "tokens_seen": 488898560 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0004166351428031019, | |
| "loss": 2.6889, | |
| "theoretical_loss": 3.923204086257365, | |
| "tokens_seen": 490209280 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.00041639871382636657, | |
| "loss": 2.6629, | |
| "theoretical_loss": 3.9220723062989444, | |
| "tokens_seen": 491520000 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0004161622848496312, | |
| "loss": 2.6323, | |
| "theoretical_loss": 3.920944382904665, | |
| "tokens_seen": 492830720 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.00041592585587289576, | |
| "loss": 2.674, | |
| "theoretical_loss": 3.919820292728682, | |
| "tokens_seen": 494141440 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0004156894268961604, | |
| "loss": 2.6522, | |
| "theoretical_loss": 3.918700012627923, | |
| "tokens_seen": 495452160 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.000415452997919425, | |
| "loss": 2.6556, | |
| "theoretical_loss": 3.9175835196597957, | |
| "tokens_seen": 496762880 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0004152165689426896, | |
| "loss": 2.6242, | |
| "theoretical_loss": 3.9164707910799264, | |
| "tokens_seen": 498073600 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.00041498013996595426, | |
| "loss": 2.6183, | |
| "theoretical_loss": 3.9153618043399296, | |
| "tokens_seen": 499384320 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.00041474371098921886, | |
| "loss": 2.6478, | |
| "theoretical_loss": 3.9142565370852136, | |
| "tokens_seen": 500695040 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.00041450728201248346, | |
| "loss": 2.6662, | |
| "theoretical_loss": 3.9131549671528116, | |
| "tokens_seen": 502005760 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.00041427085303574805, | |
| "loss": 2.6082, | |
| "theoretical_loss": 3.912057072569243, | |
| "tokens_seen": 503316480 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0004140344240590127, | |
| "loss": 2.6412, | |
| "theoretical_loss": 3.910962831548404, | |
| "tokens_seen": 504627200 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0004137979950822773, | |
| "loss": 2.6323, | |
| "theoretical_loss": 3.909872222489491, | |
| "tokens_seen": 505937920 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0004135615661055419, | |
| "loss": 2.6421, | |
| "theoretical_loss": 3.908785223974948, | |
| "tokens_seen": 507248640 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.00041332513712880655, | |
| "loss": 2.6641, | |
| "theoretical_loss": 3.9077018147684415, | |
| "tokens_seen": 508559360 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0004130887081520711, | |
| "loss": 2.5687, | |
| "theoretical_loss": 3.906621973812867, | |
| "tokens_seen": 509870080 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.00041285227917533575, | |
| "loss": 2.6077, | |
| "theoretical_loss": 3.9055456802283812, | |
| "tokens_seen": 511180800 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0004126158501986004, | |
| "loss": 2.6238, | |
| "theoretical_loss": 3.904472913310456, | |
| "tokens_seen": 512491520 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.00041237942122186494, | |
| "loss": 2.6237, | |
| "theoretical_loss": 3.9034036525279645, | |
| "tokens_seen": 513802240 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0004121429922451296, | |
| "loss": 2.6304, | |
| "theoretical_loss": 3.902337877521294, | |
| "tokens_seen": 515112960 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0004119065632683942, | |
| "loss": 2.5662, | |
| "theoretical_loss": 3.9012755681004756, | |
| "tokens_seen": 516423680 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "learning_rate": 0.0004116701342916588, | |
| "loss": 2.614, | |
| "theoretical_loss": 3.9002167042433475, | |
| "tokens_seen": 517734400 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.00041143370531492344, | |
| "loss": 2.5913, | |
| "theoretical_loss": 3.899161266093741, | |
| "tokens_seen": 519045120 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.00041119727633818804, | |
| "loss": 2.6343, | |
| "theoretical_loss": 3.8981092339596852, | |
| "tokens_seen": 520355840 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.00041096084736145263, | |
| "loss": 2.6397, | |
| "theoretical_loss": 3.8970605883116436, | |
| "tokens_seen": 521666560 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.00041072441838471723, | |
| "loss": 2.5789, | |
| "theoretical_loss": 3.896015309780767, | |
| "tokens_seen": 522977280 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "objective/train/docs_used": 270967, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.5732369422912598, | |
| "objective/train/theoretical_loss": 3.8949733791571752, | |
| "objective/train/tokens_used": 544748000, | |
| "theoretical_loss": 3.8949733791571752, | |
| "tokens_seen": 524288000 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0004104879894079819, | |
| "loss": 2.5953, | |
| "theoretical_loss": 3.8949733791571752, | |
| "tokens_seen": 524288000 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0004102515604312464, | |
| "loss": 2.605, | |
| "theoretical_loss": 3.8939347773882584, | |
| "tokens_seen": 525598720 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0004100151314545111, | |
| "loss": 2.5887, | |
| "theoretical_loss": 3.892899485576998, | |
| "tokens_seen": 526909440 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.00040977870247777573, | |
| "loss": 2.5966, | |
| "theoretical_loss": 3.8918674849803176, | |
| "tokens_seen": 528220160 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0004095422735010403, | |
| "loss": 2.5797, | |
| "theoretical_loss": 3.890838757007447, | |
| "tokens_seen": 529530880 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0004093058445243049, | |
| "loss": 2.6135, | |
| "theoretical_loss": 3.889813283218313, | |
| "tokens_seen": 530841600 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0004090694155475695, | |
| "loss": 2.5765, | |
| "theoretical_loss": 3.888791045321949, | |
| "tokens_seen": 532152320 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0004088329865708341, | |
| "loss": 2.5715, | |
| "theoretical_loss": 3.887772025174925, | |
| "tokens_seen": 533463040 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.00040859655759409877, | |
| "loss": 2.597, | |
| "theoretical_loss": 3.8867562047797986, | |
| "tokens_seen": 534773760 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.00040836012861736337, | |
| "loss": 2.5989, | |
| "theoretical_loss": 3.885743566283587, | |
| "tokens_seen": 536084480 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.00040812369964062797, | |
| "loss": 2.5891, | |
| "theoretical_loss": 3.8847340919762576, | |
| "tokens_seen": 537395200 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.00040788727066389256, | |
| "loss": 2.6195, | |
| "theoretical_loss": 3.883727764289236, | |
| "tokens_seen": 538705920 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.00040765084168715716, | |
| "loss": 2.635, | |
| "theoretical_loss": 3.8827245657939375, | |
| "tokens_seen": 540016640 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0004074144127104218, | |
| "loss": 2.578, | |
| "theoretical_loss": 3.8817244792003143, | |
| "tokens_seen": 541327360 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0004071779837336864, | |
| "loss": 2.5878, | |
| "theoretical_loss": 3.8807274873554216, | |
| "tokens_seen": 542638080 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.000406941554756951, | |
| "loss": 2.6538, | |
| "theoretical_loss": 3.8797335732420035, | |
| "tokens_seen": 543948800 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 0.0004067051257802156, | |
| "loss": 2.6775, | |
| "theoretical_loss": 3.878742719977094, | |
| "tokens_seen": 545259520 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.00040646869680348026, | |
| "loss": 2.6662, | |
| "theoretical_loss": 3.877754910810639, | |
| "tokens_seen": 546570240 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.00040623226782674485, | |
| "loss": 2.6387, | |
| "theoretical_loss": 3.876770129124133, | |
| "tokens_seen": 547880960 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.00040599583885000945, | |
| "loss": 2.6011, | |
| "theoretical_loss": 3.8757883584292774, | |
| "tokens_seen": 549191680 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004057594098732741, | |
| "loss": 2.6231, | |
| "theoretical_loss": 3.8748095823666473, | |
| "tokens_seen": 550502400 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.00040552298089653865, | |
| "loss": 2.5743, | |
| "theoretical_loss": 3.8738337847043836, | |
| "tokens_seen": 551813120 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004052865519198033, | |
| "loss": 2.544, | |
| "theoretical_loss": 3.8728609493368986, | |
| "tokens_seen": 553123840 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004050501229430679, | |
| "loss": 2.5784, | |
| "theoretical_loss": 3.8718910602835943, | |
| "tokens_seen": 554434560 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004048136939663325, | |
| "loss": 2.6569, | |
| "theoretical_loss": 3.870924101687603, | |
| "tokens_seen": 555745280 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.00040457726498959715, | |
| "loss": 2.6119, | |
| "theoretical_loss": 3.869960057814537, | |
| "tokens_seen": 557056000 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.00040434083601286174, | |
| "loss": 2.6406, | |
| "theoretical_loss": 3.868998913051261, | |
| "tokens_seen": 558366720 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.00040410440703612634, | |
| "loss": 2.5709, | |
| "theoretical_loss": 3.868040651904671, | |
| "tokens_seen": 559677440 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.00040386797805939094, | |
| "loss": 2.5896, | |
| "theoretical_loss": 3.867085259000496, | |
| "tokens_seen": 560988160 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004036315490826556, | |
| "loss": 2.5614, | |
| "theoretical_loss": 3.8661327190821124, | |
| "tokens_seen": 562298880 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004033951201059202, | |
| "loss": 2.63, | |
| "theoretical_loss": 3.8651830170093673, | |
| "tokens_seen": 563609600 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004031586911291848, | |
| "loss": 2.543, | |
| "theoretical_loss": 3.864236137757427, | |
| "tokens_seen": 564920320 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.00040292226215244944, | |
| "loss": 2.5417, | |
| "theoretical_loss": 3.863292066415628, | |
| "tokens_seen": 566231040 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.000402685833175714, | |
| "loss": 2.5676, | |
| "theoretical_loss": 3.862350788186353, | |
| "tokens_seen": 567541760 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.00040244940419897863, | |
| "loss": 2.5814, | |
| "theoretical_loss": 3.8614122883839093, | |
| "tokens_seen": 568852480 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004022129752222433, | |
| "loss": 2.5993, | |
| "theoretical_loss": 3.8604765524334312, | |
| "tokens_seen": 570163200 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004019765462455078, | |
| "loss": 2.5748, | |
| "theoretical_loss": 3.8595435658697888, | |
| "tokens_seen": 571473920 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "learning_rate": 0.0004017401172687725, | |
| "loss": 2.5808, | |
| "theoretical_loss": 3.858613314336514, | |
| "tokens_seen": 572784640 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0004015036882920371, | |
| "loss": 2.6407, | |
| "theoretical_loss": 3.8576857835847367, | |
| "tokens_seen": 574095360 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.00040126725931530167, | |
| "loss": 2.5983, | |
| "theoretical_loss": 3.8567609594721333, | |
| "tokens_seen": 575406080 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0004010308303385663, | |
| "loss": 2.5923, | |
| "theoretical_loss": 3.855838827961894, | |
| "tokens_seen": 576716800 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0004007944013618309, | |
| "loss": 2.5798, | |
| "theoretical_loss": 3.8549193751216935, | |
| "tokens_seen": 578027520 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0004005579723850955, | |
| "loss": 2.5567, | |
| "theoretical_loss": 3.854002587122681, | |
| "tokens_seen": 579338240 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0004003215434083601, | |
| "loss": 2.552, | |
| "theoretical_loss": 3.853088450238478, | |
| "tokens_seen": 580648960 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.00040008511443162477, | |
| "loss": 2.5662, | |
| "theoretical_loss": 3.8521769508441905, | |
| "tokens_seen": 581959680 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0003998486854548893, | |
| "loss": 2.5732, | |
| "theoretical_loss": 3.8512680754154323, | |
| "tokens_seen": 583270400 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.00039961225647815396, | |
| "loss": 2.5827, | |
| "theoretical_loss": 3.850361810527361, | |
| "tokens_seen": 584581120 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0003993758275014186, | |
| "loss": 2.5194, | |
| "theoretical_loss": 3.8494581428537202, | |
| "tokens_seen": 585891840 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.00039913939852468316, | |
| "loss": 2.5185, | |
| "theoretical_loss": 3.8485570591659006, | |
| "tokens_seen": 587202560 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0003989029695479478, | |
| "loss": 2.5326, | |
| "theoretical_loss": 3.847658546332007, | |
| "tokens_seen": 588513280 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "objective/train/docs_used": 303867, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.3803720474243164, | |
| "objective/train/theoretical_loss": 3.8467625913159385, | |
| "objective/train/tokens_used": 610284000, | |
| "theoretical_loss": 3.8467625913159385, | |
| "tokens_seen": 589824000 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0003986665405712124, | |
| "loss": 2.5762, | |
| "theoretical_loss": 3.8467625913159385, | |
| "tokens_seen": 589824000 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.000398430111594477, | |
| "loss": 2.5936, | |
| "theoretical_loss": 3.845869181176477, | |
| "tokens_seen": 591134720 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.00039819368261774166, | |
| "loss": 2.6104, | |
| "theoretical_loss": 3.844978303066389, | |
| "tokens_seen": 592445440 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.00039795725364100625, | |
| "loss": 2.6239, | |
| "theoretical_loss": 3.8440899442315395, | |
| "tokens_seen": 593756160 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.00039772082466427085, | |
| "loss": 2.5756, | |
| "theoretical_loss": 3.843204092010007, | |
| "tokens_seen": 595066880 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.00039748439568753545, | |
| "loss": 2.6374, | |
| "theoretical_loss": 3.8423207338312233, | |
| "tokens_seen": 596377600 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0003972479667108001, | |
| "loss": 2.6386, | |
| "theoretical_loss": 3.841439857215109, | |
| "tokens_seen": 597688320 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0003970115377340647, | |
| "loss": 2.5639, | |
| "theoretical_loss": 3.84056144977123, | |
| "tokens_seen": 598999040 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.0003967751087573293, | |
| "loss": 2.6217, | |
| "theoretical_loss": 3.839685499197956, | |
| "tokens_seen": 600309760 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 0.00039653867978059395, | |
| "loss": 2.6133, | |
| "theoretical_loss": 3.8388119932816327, | |
| "tokens_seen": 601620480 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0003963022508038585, | |
| "loss": 2.639, | |
| "theoretical_loss": 3.8379409198957637, | |
| "tokens_seen": 602931200 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.00039606582182712314, | |
| "loss": 2.5604, | |
| "theoretical_loss": 3.837072267000199, | |
| "tokens_seen": 604241920 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0003958293928503878, | |
| "loss": 2.5714, | |
| "theoretical_loss": 3.8362060226403347, | |
| "tokens_seen": 605552640 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.00039559296387365234, | |
| "loss": 2.6213, | |
| "theoretical_loss": 3.835342174946324, | |
| "tokens_seen": 606863360 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.000395356534896917, | |
| "loss": 2.6245, | |
| "theoretical_loss": 3.834480712132291, | |
| "tokens_seen": 608174080 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0003951201059201816, | |
| "loss": 2.6462, | |
| "theoretical_loss": 3.833621622495561, | |
| "tokens_seen": 609484800 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0003948836769434462, | |
| "loss": 2.5599, | |
| "theoretical_loss": 3.8327648944158907, | |
| "tokens_seen": 610795520 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.00039464724796671084, | |
| "loss": 2.62, | |
| "theoretical_loss": 3.8319105163547187, | |
| "tokens_seen": 612106240 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.00039441081898997543, | |
| "loss": 2.5858, | |
| "theoretical_loss": 3.8310584768544125, | |
| "tokens_seen": 613416960 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.00039417439001324003, | |
| "loss": 2.5531, | |
| "theoretical_loss": 3.8302087645375327, | |
| "tokens_seen": 614727680 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.00039393796103650463, | |
| "loss": 2.5948, | |
| "theoretical_loss": 3.829361368106099, | |
| "tokens_seen": 616038400 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0003937015320597693, | |
| "loss": 2.5967, | |
| "theoretical_loss": 3.8285162763408707, | |
| "tokens_seen": 617349120 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0003934651030830338, | |
| "loss": 2.6517, | |
| "theoretical_loss": 3.827673478100631, | |
| "tokens_seen": 618659840 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0003932286741062985, | |
| "loss": 2.5964, | |
| "theoretical_loss": 3.8268329623214803, | |
| "tokens_seen": 619970560 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0003929922451295631, | |
| "loss": 2.6025, | |
| "theoretical_loss": 3.8259947180161373, | |
| "tokens_seen": 621281280 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.00039275581615282767, | |
| "loss": 2.5797, | |
| "theoretical_loss": 3.8251587342732485, | |
| "tokens_seen": 622592000 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0003925193871760923, | |
| "loss": 2.5919, | |
| "theoretical_loss": 3.824325000256705, | |
| "tokens_seen": 623902720 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0003922829581993569, | |
| "loss": 2.6003, | |
| "theoretical_loss": 3.8234935052049663, | |
| "tokens_seen": 625213440 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.0003920465292226215, | |
| "loss": 2.5637, | |
| "theoretical_loss": 3.8226642384303933, | |
| "tokens_seen": 626524160 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.00039181010024588617, | |
| "loss": 2.6537, | |
| "theoretical_loss": 3.821837189318586, | |
| "tokens_seen": 627834880 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "learning_rate": 0.00039157367126915077, | |
| "loss": 2.6504, | |
| "theoretical_loss": 3.82101234732773, | |
| "tokens_seen": 629145600 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.00039133724229241536, | |
| "loss": 2.6062, | |
| "theoretical_loss": 3.820189701987953, | |
| "tokens_seen": 630456320 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.00039110081331567996, | |
| "loss": 2.6749, | |
| "theoretical_loss": 3.819369242900682, | |
| "tokens_seen": 631767040 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0003908643843389446, | |
| "loss": 2.6352, | |
| "theoretical_loss": 3.8185509597380105, | |
| "tokens_seen": 633077760 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0003906279553622092, | |
| "loss": 2.6567, | |
| "theoretical_loss": 3.8177348422420785, | |
| "tokens_seen": 634388480 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0003903915263854738, | |
| "loss": 2.6693, | |
| "theoretical_loss": 3.8169208802244476, | |
| "tokens_seen": 635699200 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.00039015509740873846, | |
| "loss": 2.6128, | |
| "theoretical_loss": 3.8161090635654937, | |
| "tokens_seen": 637009920 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.000389918668432003, | |
| "loss": 2.6761, | |
| "theoretical_loss": 3.815299382213799, | |
| "tokens_seen": 638320640 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.00038968223945526765, | |
| "loss": 2.6415, | |
| "theoretical_loss": 3.8144918261855536, | |
| "tokens_seen": 639631360 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0003894458104785323, | |
| "loss": 2.6437, | |
| "theoretical_loss": 3.813686385563966, | |
| "tokens_seen": 640942080 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.00038920938150179685, | |
| "loss": 2.6236, | |
| "theoretical_loss": 3.812883050498675, | |
| "tokens_seen": 642252800 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0003889729525250615, | |
| "loss": 2.5831, | |
| "theoretical_loss": 3.812081811205168, | |
| "tokens_seen": 643563520 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0003887365235483261, | |
| "loss": 2.5935, | |
| "theoretical_loss": 3.811282657964214, | |
| "tokens_seen": 644874240 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0003885000945715907, | |
| "loss": 2.6516, | |
| "theoretical_loss": 3.8104855811212905, | |
| "tokens_seen": 646184960 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.00038826366559485535, | |
| "loss": 2.5783, | |
| "theoretical_loss": 3.8096905710860254, | |
| "tokens_seen": 647495680 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.00038802723661811994, | |
| "loss": 2.6697, | |
| "theoretical_loss": 3.808897618331641, | |
| "tokens_seen": 648806400 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.00038779080764138454, | |
| "loss": 2.6191, | |
| "theoretical_loss": 3.8081067133944053, | |
| "tokens_seen": 650117120 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.00038755437866464914, | |
| "loss": 2.6005, | |
| "theoretical_loss": 3.8073178468730884, | |
| "tokens_seen": 651427840 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0003873179496879138, | |
| "loss": 2.5924, | |
| "theoretical_loss": 3.806531009428425, | |
| "tokens_seen": 652738560 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.00038708152071117833, | |
| "loss": 2.5748, | |
| "theoretical_loss": 3.805746191782582, | |
| "tokens_seen": 654049280 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.6156110408699026, | |
| "debugging/distinct-1-grams": 0.7851479308645805, | |
| "debugging/distinct-2-grams": 0.9568939183895592, | |
| "debugging/entropy-1-grams": 6.282807404374434, | |
| "debugging/entropy-2-grams": 7.2107758810946265, | |
| "debugging/length": 544.4090909090909, | |
| "debugging/num_segments": 22, | |
| "epoch": 0.23, | |
| "objective/train/docs_used": 335078, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.924189567565918, | |
| "objective/train/theoretical_loss": 3.8049633847186337, | |
| "objective/train/tokens_used": 675820000, | |
| "theoretical_loss": 3.8049633847186337, | |
| "tokens_seen": 655360000 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.000386845091734443, | |
| "loss": 2.6253, | |
| "theoretical_loss": 3.8049633847186337, | |
| "tokens_seen": 655360000 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.0003866204842065444, | |
| "loss": 2.6193, | |
| "theoretical_loss": 3.80418257908004, | |
| "tokens_seen": 656670720 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 0.000386384055229809, | |
| "loss": 2.5837, | |
| "theoretical_loss": 3.803403765770131, | |
| "tokens_seen": 657981440 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003861476262530736, | |
| "loss": 2.5726, | |
| "theoretical_loss": 3.802626935751598, | |
| "tokens_seen": 659292160 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.00038591119727633824, | |
| "loss": 2.5595, | |
| "theoretical_loss": 3.801852080045988, | |
| "tokens_seen": 660602880 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003856747682996028, | |
| "loss": 2.5589, | |
| "theoretical_loss": 3.801079189733206, | |
| "tokens_seen": 661913600 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.00038543833932286743, | |
| "loss": 2.5299, | |
| "theoretical_loss": 3.8003082559510197, | |
| "tokens_seen": 663224320 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003852019103461321, | |
| "loss": 2.5522, | |
| "theoretical_loss": 3.799539269894571, | |
| "tokens_seen": 664535040 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003849654813693966, | |
| "loss": 2.5565, | |
| "theoretical_loss": 3.798772222815893, | |
| "tokens_seen": 665845760 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003847290523926613, | |
| "loss": 2.5889, | |
| "theoretical_loss": 3.79800710602343, | |
| "tokens_seen": 667156480 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003844926234159259, | |
| "loss": 2.5368, | |
| "theoretical_loss": 3.7972439108815657, | |
| "tokens_seen": 668467200 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.00038425619443919047, | |
| "loss": 2.5703, | |
| "theoretical_loss": 3.7964826288101508, | |
| "tokens_seen": 669777920 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.00038401976546245507, | |
| "loss": 2.5297, | |
| "theoretical_loss": 3.795723251284044, | |
| "tokens_seen": 671088640 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003837833364857197, | |
| "loss": 2.5273, | |
| "theoretical_loss": 3.7949657698326487, | |
| "tokens_seen": 672399360 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003835469075089843, | |
| "loss": 2.5287, | |
| "theoretical_loss": 3.79421017603946, | |
| "tokens_seen": 673710080 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003833104785322489, | |
| "loss": 2.5946, | |
| "theoretical_loss": 3.7934564615416146, | |
| "tokens_seen": 675020800 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003830740495555135, | |
| "loss": 2.5696, | |
| "theoretical_loss": 3.7927046180294455, | |
| "tokens_seen": 676331520 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003828376205787781, | |
| "loss": 2.5691, | |
| "theoretical_loss": 3.7919546372460418, | |
| "tokens_seen": 677642240 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.00038260119160204276, | |
| "loss": 2.5818, | |
| "theoretical_loss": 3.7912065109868127, | |
| "tokens_seen": 678952960 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.00038236476262530736, | |
| "loss": 2.5929, | |
| "theoretical_loss": 3.790460231099053, | |
| "tokens_seen": 680263680 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.00038212833364857196, | |
| "loss": 2.611, | |
| "theoretical_loss": 3.789715789481521, | |
| "tokens_seen": 681574400 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003818919046718366, | |
| "loss": 2.5536, | |
| "theoretical_loss": 3.788973178084011, | |
| "tokens_seen": 682885120 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.00038165547569510115, | |
| "loss": 2.5714, | |
| "theoretical_loss": 3.788232388906935, | |
| "tokens_seen": 684195840 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "learning_rate": 0.0003814190467183658, | |
| "loss": 2.532, | |
| "theoretical_loss": 3.7874934140009096, | |
| "tokens_seen": 685506560 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.00038118261774163046, | |
| "loss": 2.5504, | |
| "theoretical_loss": 3.786756245466346, | |
| "tokens_seen": 686817280 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.000380946188764895, | |
| "loss": 2.5251, | |
| "theoretical_loss": 3.7860208754530422, | |
| "tokens_seen": 688128000 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.00038070975978815965, | |
| "loss": 2.5666, | |
| "theoretical_loss": 3.785287296159782, | |
| "tokens_seen": 689438720 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.00038047333081142425, | |
| "loss": 2.5693, | |
| "theoretical_loss": 3.784555499833936, | |
| "tokens_seen": 690749440 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.00038023690183468885, | |
| "loss": 2.5413, | |
| "theoretical_loss": 3.7838254787710683, | |
| "tokens_seen": 692060160 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0003800004728579535, | |
| "loss": 2.5132, | |
| "theoretical_loss": 3.7830972253145463, | |
| "tokens_seen": 693370880 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0003797640438812181, | |
| "loss": 2.4995, | |
| "theoretical_loss": 3.7823707318551536, | |
| "tokens_seen": 694681600 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0003795276149044827, | |
| "loss": 2.5142, | |
| "theoretical_loss": 3.7816459908307074, | |
| "tokens_seen": 695992320 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0003792911859277473, | |
| "loss": 2.5873, | |
| "theoretical_loss": 3.7809229947256826, | |
| "tokens_seen": 697303040 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.00037905475695101194, | |
| "loss": 2.5823, | |
| "theoretical_loss": 3.7802017360708327, | |
| "tokens_seen": 698613760 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0003788183279742765, | |
| "loss": 2.6181, | |
| "theoretical_loss": 3.779482207442821, | |
| "tokens_seen": 699924480 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.00037858189899754114, | |
| "loss": 2.5772, | |
| "theoretical_loss": 3.778764401463854, | |
| "tokens_seen": 701235200 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0003783454700208058, | |
| "loss": 2.6061, | |
| "theoretical_loss": 3.778048310801316, | |
| "tokens_seen": 702545920 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.00037810904104407033, | |
| "loss": 2.5717, | |
| "theoretical_loss": 3.7773339281674074, | |
| "tokens_seen": 703856640 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.000377872612067335, | |
| "loss": 2.5823, | |
| "theoretical_loss": 3.7766212463187916, | |
| "tokens_seen": 705167360 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0003776361830905996, | |
| "loss": 2.6044, | |
| "theoretical_loss": 3.7759102580562383, | |
| "tokens_seen": 706478080 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0003773997541138642, | |
| "loss": 2.6037, | |
| "theoretical_loss": 3.7752009562242757, | |
| "tokens_seen": 707788800 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.00037716332513712883, | |
| "loss": 2.6089, | |
| "theoretical_loss": 3.7744933337108444, | |
| "tokens_seen": 709099520 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.00037692689616039343, | |
| "loss": 2.6036, | |
| "theoretical_loss": 3.773787383446951, | |
| "tokens_seen": 710410240 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.000376690467183658, | |
| "loss": 2.649, | |
| "theoretical_loss": 3.773083098406333, | |
| "tokens_seen": 711720960 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 0.0003764540382069226, | |
| "loss": 2.5967, | |
| "theoretical_loss": 3.7723804716051195, | |
| "tokens_seen": 713031680 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0003762176092301873, | |
| "loss": 2.5888, | |
| "theoretical_loss": 3.7716794961014983, | |
| "tokens_seen": 714342400 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.00037598118025345187, | |
| "loss": 2.5893, | |
| "theoretical_loss": 3.770980164995387, | |
| "tokens_seen": 715653120 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.00037574475127671647, | |
| "loss": 2.5737, | |
| "theoretical_loss": 3.7702824714281062, | |
| "tokens_seen": 716963840 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0003755083222999811, | |
| "loss": 2.6064, | |
| "theoretical_loss": 3.7695864085820547, | |
| "tokens_seen": 718274560 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.00037527189332324566, | |
| "loss": 2.5623, | |
| "theoretical_loss": 3.7688919696803898, | |
| "tokens_seen": 719585280 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "objective/train/docs_used": 367737, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.6998071670532227, | |
| "objective/train/theoretical_loss": 3.76819914798671, | |
| "objective/train/tokens_used": 741356000, | |
| "theoretical_loss": 3.76819914798671, | |
| "tokens_seen": 720896000 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0003750354643465103, | |
| "loss": 2.5792, | |
| "theoretical_loss": 3.76819914798671, | |
| "tokens_seen": 720896000 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.00037479903536977497, | |
| "loss": 2.5341, | |
| "theoretical_loss": 3.7675079368047415, | |
| "tokens_seen": 722206720 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0003745626063930395, | |
| "loss": 2.5446, | |
| "theoretical_loss": 3.7668183294780246, | |
| "tokens_seen": 723517440 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.00037432617741630416, | |
| "loss": 2.5398, | |
| "theoretical_loss": 3.7661303193896076, | |
| "tokens_seen": 724828160 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.00037408974843956876, | |
| "loss": 2.4649, | |
| "theoretical_loss": 3.76544389996174, | |
| "tokens_seen": 726138880 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.00037385331946283336, | |
| "loss": 2.549, | |
| "theoretical_loss": 3.7647590646555713, | |
| "tokens_seen": 727449600 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.000373616890486098, | |
| "loss": 2.5627, | |
| "theoretical_loss": 3.7640758069708493, | |
| "tokens_seen": 728760320 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0003733804615093626, | |
| "loss": 2.5607, | |
| "theoretical_loss": 3.763394120445626, | |
| "tokens_seen": 730071040 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0003731440325326272, | |
| "loss": 2.5452, | |
| "theoretical_loss": 3.7627139986559595, | |
| "tokens_seen": 731381760 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0003729076035558918, | |
| "loss": 2.5804, | |
| "theoretical_loss": 3.7620354352156293, | |
| "tokens_seen": 732692480 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.00037267117457915645, | |
| "loss": 2.5176, | |
| "theoretical_loss": 3.7613584237758415, | |
| "tokens_seen": 734003200 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.000372434745602421, | |
| "loss": 2.5547, | |
| "theoretical_loss": 3.7606829580249457, | |
| "tokens_seen": 735313920 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.00037219831662568565, | |
| "loss": 2.5537, | |
| "theoretical_loss": 3.760009031688152, | |
| "tokens_seen": 736624640 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0003719618876489503, | |
| "loss": 2.5248, | |
| "theoretical_loss": 3.7593366385272513, | |
| "tokens_seen": 737935360 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.00037173728012105165, | |
| "loss": 2.5556, | |
| "theoretical_loss": 3.758665772340336, | |
| "tokens_seen": 739246080 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.00037150085114431625, | |
| "loss": 2.5898, | |
| "theoretical_loss": 3.7579964269615247, | |
| "tokens_seen": 740556800 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "learning_rate": 0.0003712644221675809, | |
| "loss": 2.5679, | |
| "theoretical_loss": 3.7573285962606917, | |
| "tokens_seen": 741867520 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.00037102799319084544, | |
| "loss": 2.5752, | |
| "theoretical_loss": 3.7566622741431956, | |
| "tokens_seen": 743178240 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0003707915642141101, | |
| "loss": 2.6096, | |
| "theoretical_loss": 3.75599745454961, | |
| "tokens_seen": 744488960 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.00037055513523737474, | |
| "loss": 2.6124, | |
| "theoretical_loss": 3.7553341314554625, | |
| "tokens_seen": 745799680 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0003703187062606393, | |
| "loss": 2.5859, | |
| "theoretical_loss": 3.754672298870968, | |
| "tokens_seen": 747110400 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.00037008227728390394, | |
| "loss": 2.6026, | |
| "theoretical_loss": 3.754011950840771, | |
| "tokens_seen": 748421120 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.00036984584830716854, | |
| "loss": 2.6295, | |
| "theoretical_loss": 3.753353081443688, | |
| "tokens_seen": 749731840 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.00036960941933043313, | |
| "loss": 2.6338, | |
| "theoretical_loss": 3.752695684792449, | |
| "tokens_seen": 751042560 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.00036937299035369773, | |
| "loss": 2.6569, | |
| "theoretical_loss": 3.752039755033449, | |
| "tokens_seen": 752353280 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0003691365613769624, | |
| "loss": 2.6009, | |
| "theoretical_loss": 3.751385286346493, | |
| "tokens_seen": 753664000 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.000368900132400227, | |
| "loss": 2.5792, | |
| "theoretical_loss": 3.7507322729445525, | |
| "tokens_seen": 754974720 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0003686637034234916, | |
| "loss": 2.5959, | |
| "theoretical_loss": 3.7500807090735115, | |
| "tokens_seen": 756285440 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.00036842727444675623, | |
| "loss": 2.64, | |
| "theoretical_loss": 3.749430589011933, | |
| "tokens_seen": 757596160 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.00036819084547002077, | |
| "loss": 2.6178, | |
| "theoretical_loss": 3.748781907070807, | |
| "tokens_seen": 758906880 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0003679544164932854, | |
| "loss": 2.6001, | |
| "theoretical_loss": 3.7481346575933197, | |
| "tokens_seen": 760217600 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0003677179875165501, | |
| "loss": 2.554, | |
| "theoretical_loss": 3.747488834954609, | |
| "tokens_seen": 761528320 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0003674815585398146, | |
| "loss": 2.5981, | |
| "theoretical_loss": 3.7468444335615363, | |
| "tokens_seen": 762839040 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.00036724512956307927, | |
| "loss": 2.583, | |
| "theoretical_loss": 3.746201447852446, | |
| "tokens_seen": 764149760 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.00036700870058634387, | |
| "loss": 2.5804, | |
| "theoretical_loss": 3.745559872296941, | |
| "tokens_seen": 765460480 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.00036677227160960847, | |
| "loss": 2.5431, | |
| "theoretical_loss": 3.7449197013956503, | |
| "tokens_seen": 766771200 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0003665358426328731, | |
| "loss": 2.5404, | |
| "theoretical_loss": 3.744280929680005, | |
| "tokens_seen": 768081920 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 0.0003662994136561377, | |
| "loss": 2.5224, | |
| "theoretical_loss": 3.743643551712009, | |
| "tokens_seen": 769392640 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0003660629846794023, | |
| "loss": 2.5551, | |
| "theoretical_loss": 3.7430075620840215, | |
| "tokens_seen": 770703360 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0003658265557026669, | |
| "loss": 2.5494, | |
| "theoretical_loss": 3.7423729554185337, | |
| "tokens_seen": 772014080 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.00036559012672593156, | |
| "loss": 2.5094, | |
| "theoretical_loss": 3.7417397263679493, | |
| "tokens_seen": 773324800 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.00036535369774919616, | |
| "loss": 2.5302, | |
| "theoretical_loss": 3.7411078696143694, | |
| "tokens_seen": 774635520 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.00036511726877246076, | |
| "loss": 2.5247, | |
| "theoretical_loss": 3.740477379869378, | |
| "tokens_seen": 775946240 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0003648808397957254, | |
| "loss": 2.4994, | |
| "theoretical_loss": 3.739848251873827, | |
| "tokens_seen": 777256960 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.00036464441081898995, | |
| "loss": 2.5491, | |
| "theoretical_loss": 3.739220480397626, | |
| "tokens_seen": 778567680 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0003644079818422546, | |
| "loss": 2.5228, | |
| "theoretical_loss": 3.7385940602395347, | |
| "tokens_seen": 779878400 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.00036417155286551926, | |
| "loss": 2.5346, | |
| "theoretical_loss": 3.737968986226954, | |
| "tokens_seen": 781189120 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0003639351238887838, | |
| "loss": 2.5475, | |
| "theoretical_loss": 3.7373452532157208, | |
| "tokens_seen": 782499840 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.00036369869491204845, | |
| "loss": 2.5443, | |
| "theoretical_loss": 3.736722856089904, | |
| "tokens_seen": 783810560 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.00036346226593531305, | |
| "loss": 2.6022, | |
| "theoretical_loss": 3.736101789761605, | |
| "tokens_seen": 785121280 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "objective/train/docs_used": 399996, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.852480173110962, | |
| "objective/train/theoretical_loss": 3.735482049170753, | |
| "objective/train/tokens_used": 806892000, | |
| "theoretical_loss": 3.735482049170753, | |
| "tokens_seen": 786432000 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.00036322583695857765, | |
| "loss": 2.6068, | |
| "theoretical_loss": 3.735482049170753, | |
| "tokens_seen": 786432000 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.00036298940798184224, | |
| "loss": 2.5888, | |
| "theoretical_loss": 3.734863629284911, | |
| "tokens_seen": 787742720 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0003627529790051069, | |
| "loss": 2.5753, | |
| "theoretical_loss": 3.7342465250990777, | |
| "tokens_seen": 789053440 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0003625165500283715, | |
| "loss": 2.5319, | |
| "theoretical_loss": 3.7336307316354933, | |
| "tokens_seen": 790364160 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0003622801210516361, | |
| "loss": 2.5401, | |
| "theoretical_loss": 3.7330162439434442, | |
| "tokens_seen": 791674880 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.00036204369207490074, | |
| "loss": 2.538, | |
| "theoretical_loss": 3.7324030570990736, | |
| "tokens_seen": 792985600 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0003618072630981653, | |
| "loss": 2.5322, | |
| "theoretical_loss": 3.7317911662051912, | |
| "tokens_seen": 794296320 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.00036157083412142994, | |
| "loss": 2.5192, | |
| "theoretical_loss": 3.731180566391086, | |
| "tokens_seen": 795607040 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "learning_rate": 0.0003613344051446946, | |
| "loss": 2.5101, | |
| "theoretical_loss": 3.7305712528123367, | |
| "tokens_seen": 796917760 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.00036109797616795913, | |
| "loss": 2.5084, | |
| "theoretical_loss": 3.729963220650631, | |
| "tokens_seen": 798228480 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0003608615471912238, | |
| "loss": 2.5016, | |
| "theoretical_loss": 3.729356465113578, | |
| "tokens_seen": 799539200 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0003606251182144884, | |
| "loss": 2.5087, | |
| "theoretical_loss": 3.7287509814345317, | |
| "tokens_seen": 800849920 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.000360388689237753, | |
| "loss": 2.5073, | |
| "theoretical_loss": 3.7281467648724043, | |
| "tokens_seen": 802160640 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.00036015226026101763, | |
| "loss": 2.4849, | |
| "theoretical_loss": 3.727543810711495, | |
| "tokens_seen": 803471360 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0003599158312842822, | |
| "loss": 2.4428, | |
| "theoretical_loss": 3.7269421142613055, | |
| "tokens_seen": 804782080 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0003596794023075468, | |
| "loss": 2.5142, | |
| "theoretical_loss": 3.7263416708563715, | |
| "tokens_seen": 806092800 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0003594429733308114, | |
| "loss": 2.5409, | |
| "theoretical_loss": 3.725742475856083, | |
| "tokens_seen": 807403520 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0003592065443540761, | |
| "loss": 2.5203, | |
| "theoretical_loss": 3.7251445246445147, | |
| "tokens_seen": 808714240 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.00035897011537734067, | |
| "loss": 2.5417, | |
| "theoretical_loss": 3.724547812630255, | |
| "tokens_seen": 810024960 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.00035873368640060527, | |
| "loss": 2.5169, | |
| "theoretical_loss": 3.7239523352462354, | |
| "tokens_seen": 811335680 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0003584972574238699, | |
| "loss": 2.4925, | |
| "theoretical_loss": 3.7233580879495616, | |
| "tokens_seen": 812646400 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.00035826082844713446, | |
| "loss": 2.5787, | |
| "theoretical_loss": 3.72276506622135, | |
| "tokens_seen": 813957120 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0003580243994703991, | |
| "loss": 2.5697, | |
| "theoretical_loss": 3.722173265566559, | |
| "tokens_seen": 815267840 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.00035778797049366366, | |
| "loss": 2.5479, | |
| "theoretical_loss": 3.7215826815138264, | |
| "tokens_seen": 816578560 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0003575515415169283, | |
| "loss": 2.5023, | |
| "theoretical_loss": 3.720993309615307, | |
| "tokens_seen": 817889280 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.00035731511254019296, | |
| "loss": 2.5445, | |
| "theoretical_loss": 3.720405145446512, | |
| "tokens_seen": 819200000 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.0003570786835634575, | |
| "loss": 2.5444, | |
| "theoretical_loss": 3.719818184606148, | |
| "tokens_seen": 820510720 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.00035684225458672216, | |
| "loss": 2.5484, | |
| "theoretical_loss": 3.719232422715959, | |
| "tokens_seen": 821821440 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.00035660582560998675, | |
| "loss": 2.4957, | |
| "theoretical_loss": 3.7186478554205693, | |
| "tokens_seen": 823132160 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.00035636939663325135, | |
| "loss": 2.5456, | |
| "theoretical_loss": 3.718064478387329, | |
| "tokens_seen": 824442880 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 0.000356132967656516, | |
| "loss": 2.5418, | |
| "theoretical_loss": 3.717482287306157, | |
| "tokens_seen": 825753600 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0003558965386797806, | |
| "loss": 2.5623, | |
| "theoretical_loss": 3.7169012778893897, | |
| "tokens_seen": 827064320 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0003556601097030452, | |
| "loss": 2.5488, | |
| "theoretical_loss": 3.71632144587163, | |
| "tokens_seen": 828375040 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0003554236807263098, | |
| "loss": 2.5591, | |
| "theoretical_loss": 3.7157427870095936, | |
| "tokens_seen": 829685760 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.00035518725174957445, | |
| "loss": 2.5166, | |
| "theoretical_loss": 3.7151652970819615, | |
| "tokens_seen": 830996480 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.00035495082277283904, | |
| "loss": 2.5355, | |
| "theoretical_loss": 3.714588971889232, | |
| "tokens_seen": 832307200 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.00035471439379610364, | |
| "loss": 2.5274, | |
| "theoretical_loss": 3.7140138072535738, | |
| "tokens_seen": 833617920 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0003544779648193683, | |
| "loss": 2.4853, | |
| "theoretical_loss": 3.7134397990186794, | |
| "tokens_seen": 834928640 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.00035424153584263284, | |
| "loss": 2.5191, | |
| "theoretical_loss": 3.7128669430496215, | |
| "tokens_seen": 836239360 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0003540051068658975, | |
| "loss": 2.5055, | |
| "theoretical_loss": 3.71229523523271, | |
| "tokens_seen": 837550080 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.00035376867788916214, | |
| "loss": 2.5017, | |
| "theoretical_loss": 3.7117246714753485, | |
| "tokens_seen": 838860800 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0003535322489124267, | |
| "loss": 2.5453, | |
| "theoretical_loss": 3.7111552477058947, | |
| "tokens_seen": 840171520 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.00035329581993569134, | |
| "loss": 2.562, | |
| "theoretical_loss": 3.7105869598735195, | |
| "tokens_seen": 841482240 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.00035305939095895593, | |
| "loss": 2.4942, | |
| "theoretical_loss": 3.71001980394807, | |
| "tokens_seen": 842792960 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.00035282296198222053, | |
| "loss": 2.5007, | |
| "theoretical_loss": 3.709453775919931, | |
| "tokens_seen": 844103680 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0003525865330054852, | |
| "loss": 2.4856, | |
| "theoretical_loss": 3.7088888717998865, | |
| "tokens_seen": 845414400 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0003523501040287498, | |
| "loss": 2.4917, | |
| "theoretical_loss": 3.70832508761899, | |
| "tokens_seen": 846725120 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0003521136750520144, | |
| "loss": 2.4869, | |
| "theoretical_loss": 3.707762419428425, | |
| "tokens_seen": 848035840 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.000351877246075279, | |
| "loss": 2.4647, | |
| "theoretical_loss": 3.7072008632993745, | |
| "tokens_seen": 849346560 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0003516408170985436, | |
| "loss": 2.4658, | |
| "theoretical_loss": 3.7066404153228873, | |
| "tokens_seen": 850657280 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "objective/train/docs_used": 433520, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.656919479370117, | |
| "objective/train/theoretical_loss": 3.7060810716097503, | |
| "objective/train/tokens_used": 872428000, | |
| "theoretical_loss": 3.7060810716097503, | |
| "tokens_seen": 851968000 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.00035140438812180817, | |
| "loss": 2.4919, | |
| "theoretical_loss": 3.7060810716097503, | |
| "tokens_seen": 851968000 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "learning_rate": 0.0003511679591450728, | |
| "loss": 2.5428, | |
| "theoretical_loss": 3.7055228282903556, | |
| "tokens_seen": 853278720 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00035093153016833747, | |
| "loss": 2.5117, | |
| "theoretical_loss": 3.704965681514573, | |
| "tokens_seen": 854589440 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.000350695101191602, | |
| "loss": 2.5613, | |
| "theoretical_loss": 3.7044096274516214, | |
| "tokens_seen": 855900160 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00035045867221486667, | |
| "loss": 2.5325, | |
| "theoretical_loss": 3.7038546622899435, | |
| "tokens_seen": 857210880 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00035022224323813127, | |
| "loss": 2.4928, | |
| "theoretical_loss": 3.70330078223708, | |
| "tokens_seen": 858521600 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00034998581426139586, | |
| "loss": 2.5232, | |
| "theoretical_loss": 3.7027479835195427, | |
| "tokens_seen": 859832320 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.0003497493852846605, | |
| "loss": 2.4996, | |
| "theoretical_loss": 3.702196262382694, | |
| "tokens_seen": 861143040 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.0003495129563079251, | |
| "loss": 2.4919, | |
| "theoretical_loss": 3.701645615090622, | |
| "tokens_seen": 862453760 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.0003492765273311897, | |
| "loss": 2.471, | |
| "theoretical_loss": 3.7010960379260185, | |
| "tokens_seen": 863764480 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.0003490400983544543, | |
| "loss": 2.5224, | |
| "theoretical_loss": 3.7005475271900616, | |
| "tokens_seen": 865075200 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00034880366937771896, | |
| "loss": 2.5535, | |
| "theoretical_loss": 3.7000000792022916, | |
| "tokens_seen": 866385920 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00034856724040098356, | |
| "loss": 2.6071, | |
| "theoretical_loss": 3.699453690300495, | |
| "tokens_seen": 867696640 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00034833081142424815, | |
| "loss": 2.5791, | |
| "theoretical_loss": 3.698908356840586, | |
| "tokens_seen": 869007360 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.0003480943824475128, | |
| "loss": 2.5762, | |
| "theoretical_loss": 3.6983640751964897, | |
| "tokens_seen": 870318080 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00034785795347077735, | |
| "loss": 2.5463, | |
| "theoretical_loss": 3.697820841760025, | |
| "tokens_seen": 871628800 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.000347621524494042, | |
| "loss": 2.5826, | |
| "theoretical_loss": 3.697278652940791, | |
| "tokens_seen": 872939520 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00034738509551730665, | |
| "loss": 2.5589, | |
| "theoretical_loss": 3.6967375051660527, | |
| "tokens_seen": 874250240 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.0003471486665405712, | |
| "loss": 2.5686, | |
| "theoretical_loss": 3.6961973948806266, | |
| "tokens_seen": 875560960 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00034691223756383585, | |
| "loss": 2.5419, | |
| "theoretical_loss": 3.6956583185467693, | |
| "tokens_seen": 876871680 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00034667580858710044, | |
| "loss": 2.5656, | |
| "theoretical_loss": 3.6951202726440666, | |
| "tokens_seen": 878182400 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00034643937961036504, | |
| "loss": 2.5629, | |
| "theoretical_loss": 3.6945832536693204, | |
| "tokens_seen": 879493120 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 0.00034620295063362964, | |
| "loss": 2.5165, | |
| "theoretical_loss": 3.6940472581364423, | |
| "tokens_seen": 880803840 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0003459665216568943, | |
| "loss": 2.53, | |
| "theoretical_loss": 3.6935122825763433, | |
| "tokens_seen": 882114560 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0003457300926801589, | |
| "loss": 2.5413, | |
| "theoretical_loss": 3.692978323536824, | |
| "tokens_seen": 883425280 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0003454936637034235, | |
| "loss": 2.5611, | |
| "theoretical_loss": 3.6924453775824704, | |
| "tokens_seen": 884736000 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.00034525723472668814, | |
| "loss": 2.5324, | |
| "theoretical_loss": 3.6919134412945462, | |
| "tokens_seen": 886046720 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0003450208057499527, | |
| "loss": 2.5451, | |
| "theoretical_loss": 3.691382511270887, | |
| "tokens_seen": 887357440 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.00034478437677321733, | |
| "loss": 2.5303, | |
| "theoretical_loss": 3.6908525841257953, | |
| "tokens_seen": 888668160 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.000344547947796482, | |
| "loss": 2.5513, | |
| "theoretical_loss": 3.6903236564899395, | |
| "tokens_seen": 889978880 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.00034431151881974653, | |
| "loss": 2.5623, | |
| "theoretical_loss": 3.6897957250102467, | |
| "tokens_seen": 891289600 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0003440750898430112, | |
| "loss": 2.5275, | |
| "theoretical_loss": 3.6892687863498024, | |
| "tokens_seen": 892600320 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0003438386608662758, | |
| "loss": 2.5715, | |
| "theoretical_loss": 3.6887428371877506, | |
| "tokens_seen": 893911040 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0003436022318895404, | |
| "loss": 2.5227, | |
| "theoretical_loss": 3.688217874219191, | |
| "tokens_seen": 895221760 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.000343365802912805, | |
| "loss": 2.5428, | |
| "theoretical_loss": 3.6876938941550783, | |
| "tokens_seen": 896532480 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0003431293739360696, | |
| "loss": 2.5328, | |
| "theoretical_loss": 3.6871708937221275, | |
| "tokens_seen": 897843200 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0003428929449593342, | |
| "loss": 2.5292, | |
| "theoretical_loss": 3.686648869662709, | |
| "tokens_seen": 899153920 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0003426565159825988, | |
| "loss": 2.4665, | |
| "theoretical_loss": 3.6861278187347573, | |
| "tokens_seen": 900464640 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.00034242008700586347, | |
| "loss": 2.5186, | |
| "theoretical_loss": 3.6856077377116696, | |
| "tokens_seen": 901775360 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.00034218365802912807, | |
| "loss": 2.5248, | |
| "theoretical_loss": 3.685088623382213, | |
| "tokens_seen": 903086080 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.00034194722905239266, | |
| "loss": 2.5546, | |
| "theoretical_loss": 3.6845704725504254, | |
| "tokens_seen": 904396800 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0003417108000756573, | |
| "loss": 2.5392, | |
| "theoretical_loss": 3.684053282035526, | |
| "tokens_seen": 905707520 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.00034147437109892186, | |
| "loss": 2.5244, | |
| "theoretical_loss": 3.6835370486718153, | |
| "tokens_seen": 907018240 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.0003412379421221865, | |
| "loss": 2.5324, | |
| "theoretical_loss": 3.6830217693085876, | |
| "tokens_seen": 908328960 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "learning_rate": 0.00034100151314545116, | |
| "loss": 2.5625, | |
| "theoretical_loss": 3.682507440810034, | |
| "tokens_seen": 909639680 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0003407650841687157, | |
| "loss": 2.5318, | |
| "theoretical_loss": 3.6819940600551533, | |
| "tokens_seen": 910950400 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00034052865519198036, | |
| "loss": 2.5141, | |
| "theoretical_loss": 3.6814816239376595, | |
| "tokens_seen": 912261120 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00034029222621524496, | |
| "loss": 2.5027, | |
| "theoretical_loss": 3.6809701293658934, | |
| "tokens_seen": 913571840 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00034005579723850955, | |
| "loss": 2.5628, | |
| "theoretical_loss": 3.6804595732627297, | |
| "tokens_seen": 914882560 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00033981936826177415, | |
| "loss": 2.5609, | |
| "theoretical_loss": 3.6799499525654915, | |
| "tokens_seen": 916193280 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "objective/train/docs_used": 465173, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.4846749305725098, | |
| "objective/train/theoretical_loss": 3.6794412642258583, | |
| "objective/train/tokens_used": 937964000, | |
| "theoretical_loss": 3.6794412642258583, | |
| "tokens_seen": 917504000 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0003395829392850388, | |
| "loss": 2.533, | |
| "theoretical_loss": 3.6794412642258583, | |
| "tokens_seen": 917504000 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0003393465103083034, | |
| "loss": 2.5336, | |
| "theoretical_loss": 3.6789335052097814, | |
| "tokens_seen": 918814720 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.000339110081331568, | |
| "loss": 2.5199, | |
| "theoretical_loss": 3.6784266724973946, | |
| "tokens_seen": 920125440 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00033887365235483265, | |
| "loss": 2.5118, | |
| "theoretical_loss": 3.6779207630829296, | |
| "tokens_seen": 921436160 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0003386372233780972, | |
| "loss": 2.4911, | |
| "theoretical_loss": 3.677415773974629, | |
| "tokens_seen": 922746880 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00033840079440136184, | |
| "loss": 2.5151, | |
| "theoretical_loss": 3.6769117021946625, | |
| "tokens_seen": 924057600 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0003381643654246265, | |
| "loss": 2.5285, | |
| "theoretical_loss": 3.676408544779039, | |
| "tokens_seen": 925368320 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00033792793644789104, | |
| "loss": 2.4638, | |
| "theoretical_loss": 3.6759062987775293, | |
| "tokens_seen": 926679040 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0003376915074711557, | |
| "loss": 2.489, | |
| "theoretical_loss": 3.6754049612535753, | |
| "tokens_seen": 927989760 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0003374550784944203, | |
| "loss": 2.4999, | |
| "theoretical_loss": 3.6749045292842126, | |
| "tokens_seen": 929300480 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0003372186495176849, | |
| "loss": 2.4951, | |
| "theoretical_loss": 3.674404999959987, | |
| "tokens_seen": 930611200 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0003369940419897863, | |
| "loss": 2.4598, | |
| "theoretical_loss": 3.6739063703848718, | |
| "tokens_seen": 931921920 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0003367576130130509, | |
| "loss": 2.4604, | |
| "theoretical_loss": 3.6734086376761907, | |
| "tokens_seen": 933232640 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.0003365211840363155, | |
| "loss": 2.4699, | |
| "theoretical_loss": 3.6729117989645332, | |
| "tokens_seen": 934543360 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00033628475505958013, | |
| "loss": 2.4874, | |
| "theoretical_loss": 3.6724158513936773, | |
| "tokens_seen": 935854080 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 0.00033604832608284473, | |
| "loss": 2.4748, | |
| "theoretical_loss": 3.6719207921205115, | |
| "tokens_seen": 937164800 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00033581189710610933, | |
| "loss": 2.5217, | |
| "theoretical_loss": 3.6714266183149533, | |
| "tokens_seen": 938475520 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0003355754681293739, | |
| "loss": 2.4858, | |
| "theoretical_loss": 3.670933327159875, | |
| "tokens_seen": 939786240 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0003353390391526386, | |
| "loss": 2.4642, | |
| "theoretical_loss": 3.670440915851024, | |
| "tokens_seen": 941096960 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0003351026101759032, | |
| "loss": 2.5167, | |
| "theoretical_loss": 3.6699493815969464, | |
| "tokens_seen": 942407680 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0003348661811991678, | |
| "loss": 2.5054, | |
| "theoretical_loss": 3.6694587216189114, | |
| "tokens_seen": 943718400 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0003346297522224324, | |
| "loss": 2.5119, | |
| "theoretical_loss": 3.6689689331508366, | |
| "tokens_seen": 945029120 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00033439332324569697, | |
| "loss": 2.4945, | |
| "theoretical_loss": 3.6684800134392104, | |
| "tokens_seen": 946339840 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0003341568942689616, | |
| "loss": 2.4933, | |
| "theoretical_loss": 3.6679919597430213, | |
| "tokens_seen": 947650560 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00033392046529222627, | |
| "loss": 2.488, | |
| "theoretical_loss": 3.6675047693336795, | |
| "tokens_seen": 948961280 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0003336840363154908, | |
| "loss": 2.4514, | |
| "theoretical_loss": 3.6670184394949477, | |
| "tokens_seen": 950272000 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00033344760733875547, | |
| "loss": 2.4698, | |
| "theoretical_loss": 3.6665329675228646, | |
| "tokens_seen": 951582720 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00033321117836202, | |
| "loss": 2.4827, | |
| "theoretical_loss": 3.666048350725677, | |
| "tokens_seen": 952893440 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00033297474938528466, | |
| "loss": 2.5038, | |
| "theoretical_loss": 3.6655645864237627, | |
| "tokens_seen": 954204160 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0003327383204085493, | |
| "loss": 2.4634, | |
| "theoretical_loss": 3.6650816719495625, | |
| "tokens_seen": 955514880 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00033250189143181386, | |
| "loss": 2.471, | |
| "theoretical_loss": 3.6645996046475107, | |
| "tokens_seen": 956825600 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0003322654624550785, | |
| "loss": 2.5101, | |
| "theoretical_loss": 3.664118381873961, | |
| "tokens_seen": 958136320 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0003320290334783431, | |
| "loss": 2.4648, | |
| "theoretical_loss": 3.66363800099712, | |
| "tokens_seen": 959447040 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0003317926045016077, | |
| "loss": 2.4884, | |
| "theoretical_loss": 3.6631584593969757, | |
| "tokens_seen": 960757760 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.0003315561755248723, | |
| "loss": 2.5334, | |
| "theoretical_loss": 3.6626797544652305, | |
| "tokens_seen": 962068480 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00033131974654813695, | |
| "loss": 2.5232, | |
| "theoretical_loss": 3.6622018836052317, | |
| "tokens_seen": 963379200 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "learning_rate": 0.00033108331757140155, | |
| "loss": 2.5273, | |
| "theoretical_loss": 3.6617248442319053, | |
| "tokens_seen": 964689920 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00033084688859466615, | |
| "loss": 2.5278, | |
| "theoretical_loss": 3.661248633771687, | |
| "tokens_seen": 966000640 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003306104596179308, | |
| "loss": 2.4733, | |
| "theoretical_loss": 3.6607732496624568, | |
| "tokens_seen": 967311360 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00033037403064119534, | |
| "loss": 2.5027, | |
| "theoretical_loss": 3.660298689353471, | |
| "tokens_seen": 968622080 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00033013760166446, | |
| "loss": 2.5527, | |
| "theoretical_loss": 3.659824950305297, | |
| "tokens_seen": 969932800 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00032990117268772465, | |
| "loss": 2.4435, | |
| "theoretical_loss": 3.6593520299897513, | |
| "tokens_seen": 971243520 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003296647437109892, | |
| "loss": 2.4751, | |
| "theoretical_loss": 3.6588799258898277, | |
| "tokens_seen": 972554240 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00032942831473425384, | |
| "loss": 2.4767, | |
| "theoretical_loss": 3.65840863549964, | |
| "tokens_seen": 973864960 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00032919188575751844, | |
| "loss": 2.5322, | |
| "theoretical_loss": 3.6579381563243527, | |
| "tokens_seen": 975175680 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00032895545678078304, | |
| "loss": 2.5267, | |
| "theoretical_loss": 3.6574684858801207, | |
| "tokens_seen": 976486400 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003287190278040477, | |
| "loss": 2.5045, | |
| "theoretical_loss": 3.656999621694025, | |
| "tokens_seen": 977797120 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003284825988273123, | |
| "loss": 2.5205, | |
| "theoretical_loss": 3.65653156130401, | |
| "tokens_seen": 979107840 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003282461698505769, | |
| "loss": 2.4807, | |
| "theoretical_loss": 3.6560643022588217, | |
| "tokens_seen": 980418560 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003280097408738415, | |
| "loss": 2.4452, | |
| "theoretical_loss": 3.655597842117945, | |
| "tokens_seen": 981729280 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.5533532515929408, | |
| "debugging/distinct-1-grams": 0.8167262089339179, | |
| "debugging/distinct-2-grams": 0.9729861472476561, | |
| "debugging/entropy-1-grams": 5.855099736313454, | |
| "debugging/entropy-2-grams": 6.517275157822866, | |
| "debugging/length": 523.5, | |
| "debugging/num_segments": 12, | |
| "epoch": 0.35, | |
| "objective/train/docs_used": 498539, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.215684175491333, | |
| "objective/train/theoretical_loss": 3.6551321784515434, | |
| "objective/train/tokens_used": 1003500000, | |
| "theoretical_loss": 3.6551321784515434, | |
| "tokens_seen": 983040000 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00032777331189710613, | |
| "loss": 2.5158, | |
| "theoretical_loss": 3.6551321784515434, | |
| "tokens_seen": 983040000 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00032753688292037073, | |
| "loss": 2.5053, | |
| "theoretical_loss": 3.6546673088403985, | |
| "tokens_seen": 984350720 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003273004539436353, | |
| "loss": 2.4746, | |
| "theoretical_loss": 3.654203230875848, | |
| "tokens_seen": 985661440 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003270640249669, | |
| "loss": 2.4698, | |
| "theoretical_loss": 3.653739942159728, | |
| "tokens_seen": 986972160 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003268275959901645, | |
| "loss": 2.5194, | |
| "theoretical_loss": 3.65327744030431, | |
| "tokens_seen": 988282880 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003265911670134292, | |
| "loss": 2.5108, | |
| "theoretical_loss": 3.6528157229322447, | |
| "tokens_seen": 989593600 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003263547380366938, | |
| "loss": 2.5272, | |
| "theoretical_loss": 3.6523547876765017, | |
| "tokens_seen": 990904320 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.0003261301305087951, | |
| "loss": 2.4555, | |
| "theoretical_loss": 3.651894632180312, | |
| "tokens_seen": 992215040 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "learning_rate": 0.00032589370153205977, | |
| "loss": 2.4508, | |
| "theoretical_loss": 3.65143525409711, | |
| "tokens_seen": 993525760 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0003256572725553244, | |
| "loss": 2.4642, | |
| "theoretical_loss": 3.650976651090474, | |
| "tokens_seen": 994836480 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.00032542084357858897, | |
| "loss": 2.4098, | |
| "theoretical_loss": 3.650518820834072, | |
| "tokens_seen": 996147200 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0003251844146018536, | |
| "loss": 2.4604, | |
| "theoretical_loss": 3.650061761011603, | |
| "tokens_seen": 997457920 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0003249479856251182, | |
| "loss": 2.4685, | |
| "theoretical_loss": 3.649605469316742, | |
| "tokens_seen": 998768640 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0003247115566483828, | |
| "loss": 2.4712, | |
| "theoretical_loss": 3.6491499434530814, | |
| "tokens_seen": 1000079360 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.00032447512767164746, | |
| "loss": 2.4303, | |
| "theoretical_loss": 3.648695181134078, | |
| "tokens_seen": 1001390080 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.00032423869869491206, | |
| "loss": 2.5146, | |
| "theoretical_loss": 3.648241180082998, | |
| "tokens_seen": 1002700800 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.00032400226971817666, | |
| "loss": 2.4957, | |
| "theoretical_loss": 3.64778793803286, | |
| "tokens_seen": 1004011520 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.00032376584074144126, | |
| "loss": 2.5036, | |
| "theoretical_loss": 3.6473354527263813, | |
| "tokens_seen": 1005322240 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0003235294117647059, | |
| "loss": 2.4748, | |
| "theoretical_loss": 3.6468837219159256, | |
| "tokens_seen": 1006632960 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0003232929827879705, | |
| "loss": 2.5204, | |
| "theoretical_loss": 3.6464327433634467, | |
| "tokens_seen": 1007943680 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0003230565538112351, | |
| "loss": 2.5071, | |
| "theoretical_loss": 3.645982514840437, | |
| "tokens_seen": 1009254400 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.00032282012483449975, | |
| "loss": 2.4906, | |
| "theoretical_loss": 3.6455330341278733, | |
| "tokens_seen": 1010565120 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0003225836958577643, | |
| "loss": 2.439, | |
| "theoretical_loss": 3.645084299016166, | |
| "tokens_seen": 1011875840 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.00032234726688102895, | |
| "loss": 2.4903, | |
| "theoretical_loss": 3.644636307305104, | |
| "tokens_seen": 1013186560 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.00032211083790429355, | |
| "loss": 2.5184, | |
| "theoretical_loss": 3.644189056803805, | |
| "tokens_seen": 1014497280 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.00032187440892755814, | |
| "loss": 2.4927, | |
| "theoretical_loss": 3.6437425453306638, | |
| "tokens_seen": 1015808000 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0003216379799508228, | |
| "loss": 2.4858, | |
| "theoretical_loss": 3.643296770713301, | |
| "tokens_seen": 1017118720 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0003214015509740874, | |
| "loss": 2.5156, | |
| "theoretical_loss": 3.642851730788511, | |
| "tokens_seen": 1018429440 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.000321165121997352, | |
| "loss": 2.4615, | |
| "theoretical_loss": 3.642407423402213, | |
| "tokens_seen": 1019740160 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 0.0003209286930206166, | |
| "loss": 2.4869, | |
| "theoretical_loss": 3.641963846409399, | |
| "tokens_seen": 1021050880 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00032069226404388124, | |
| "loss": 2.514, | |
| "theoretical_loss": 3.641520997674087, | |
| "tokens_seen": 1022361600 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00032045583506714584, | |
| "loss": 2.5132, | |
| "theoretical_loss": 3.641078875069269, | |
| "tokens_seen": 1023672320 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00032021940609041044, | |
| "loss": 2.494, | |
| "theoretical_loss": 3.640637476476862, | |
| "tokens_seen": 1024983040 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0003199829771136751, | |
| "loss": 2.5359, | |
| "theoretical_loss": 3.64019679978766, | |
| "tokens_seen": 1026293760 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00031974654813693963, | |
| "loss": 2.4907, | |
| "theoretical_loss": 3.639756842901286, | |
| "tokens_seen": 1027604480 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0003195101191602043, | |
| "loss": 2.5045, | |
| "theoretical_loss": 3.639317603726142, | |
| "tokens_seen": 1028915200 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00031927369018346893, | |
| "loss": 2.5303, | |
| "theoretical_loss": 3.638879080179364, | |
| "tokens_seen": 1030225920 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0003190372612067335, | |
| "loss": 2.4938, | |
| "theoretical_loss": 3.63844127018677, | |
| "tokens_seen": 1031536640 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00031880083222999813, | |
| "loss": 2.4617, | |
| "theoretical_loss": 3.6380041716828186, | |
| "tokens_seen": 1032847360 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0003185644032532627, | |
| "loss": 2.4787, | |
| "theoretical_loss": 3.637567782610558, | |
| "tokens_seen": 1034158080 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0003183279742765273, | |
| "loss": 2.503, | |
| "theoretical_loss": 3.637132100921579, | |
| "tokens_seen": 1035468800 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.000318091545299792, | |
| "loss": 2.4477, | |
| "theoretical_loss": 3.6366971245759734, | |
| "tokens_seen": 1036779520 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00031785511632305657, | |
| "loss": 2.4778, | |
| "theoretical_loss": 3.6362628515422815, | |
| "tokens_seen": 1038090240 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00031761868734632117, | |
| "loss": 2.4858, | |
| "theoretical_loss": 3.635829279797453, | |
| "tokens_seen": 1039400960 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00031738225836958577, | |
| "loss": 2.4664, | |
| "theoretical_loss": 3.6353964073267964, | |
| "tokens_seen": 1040711680 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0003171458293928504, | |
| "loss": 2.5042, | |
| "theoretical_loss": 3.6349642321239375, | |
| "tokens_seen": 1042022400 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.000316909400416115, | |
| "loss": 2.5046, | |
| "theoretical_loss": 3.6345327521907724, | |
| "tokens_seen": 1043333120 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0003166729714393796, | |
| "loss": 2.5006, | |
| "theoretical_loss": 3.6341019655374245, | |
| "tokens_seen": 1044643840 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00031643654246264427, | |
| "loss": 2.4485, | |
| "theoretical_loss": 3.6336718701821997, | |
| "tokens_seen": 1045954560 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.0003162001134859088, | |
| "loss": 2.4775, | |
| "theoretical_loss": 3.633242464151543, | |
| "tokens_seen": 1047265280 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "objective/train/docs_used": 530406, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.4532814025878906, | |
| "objective/train/theoretical_loss": 3.632813745479994, | |
| "objective/train/tokens_used": 1069036000, | |
| "theoretical_loss": 3.632813745479994, | |
| "tokens_seen": 1048576000 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00031596368450917346, | |
| "loss": 2.4618, | |
| "theoretical_loss": 3.632813745479994, | |
| "tokens_seen": 1048576000 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "learning_rate": 0.00031572725553243806, | |
| "loss": 2.4851, | |
| "theoretical_loss": 3.6323857122101444, | |
| "tokens_seen": 1049886720 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.00031549082655570266, | |
| "loss": 2.4556, | |
| "theoretical_loss": 3.631958362392595, | |
| "tokens_seen": 1051197440 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0003152543975789673, | |
| "loss": 2.4598, | |
| "theoretical_loss": 3.631531694085914, | |
| "tokens_seen": 1052508160 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0003150179686022319, | |
| "loss": 2.4197, | |
| "theoretical_loss": 3.6311057053565907, | |
| "tokens_seen": 1053818880 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0003147815396254965, | |
| "loss": 2.43, | |
| "theoretical_loss": 3.6306803942789987, | |
| "tokens_seen": 1055129600 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0003145451106487611, | |
| "loss": 2.4323, | |
| "theoretical_loss": 3.63025575893535, | |
| "tokens_seen": 1056440320 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.00031430868167202575, | |
| "loss": 2.4206, | |
| "theoretical_loss": 3.6298317974156555, | |
| "tokens_seen": 1057751040 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.00031407225269529035, | |
| "loss": 2.4124, | |
| "theoretical_loss": 3.629408507817682, | |
| "tokens_seen": 1059061760 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.00031383582371855495, | |
| "loss": 2.4507, | |
| "theoretical_loss": 3.6289858882469135, | |
| "tokens_seen": 1060372480 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0003135993947418196, | |
| "loss": 2.4023, | |
| "theoretical_loss": 3.6285639368165077, | |
| "tokens_seen": 1061683200 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.00031336296576508414, | |
| "loss": 2.4559, | |
| "theoretical_loss": 3.6281426516472584, | |
| "tokens_seen": 1062993920 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0003131265367883488, | |
| "loss": 2.4004, | |
| "theoretical_loss": 3.627722030867552, | |
| "tokens_seen": 1064304640 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.00031289010781161345, | |
| "loss": 2.4127, | |
| "theoretical_loss": 3.62730207261333, | |
| "tokens_seen": 1065615360 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.000312653678834878, | |
| "loss": 2.4589, | |
| "theoretical_loss": 3.626882775028048, | |
| "tokens_seen": 1066926080 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.00031241724985814264, | |
| "loss": 2.4704, | |
| "theoretical_loss": 3.6264641362626375, | |
| "tokens_seen": 1068236800 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.00031218082088140724, | |
| "loss": 2.4595, | |
| "theoretical_loss": 3.626046154475465, | |
| "tokens_seen": 1069547520 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.00031194439190467183, | |
| "loss": 2.4255, | |
| "theoretical_loss": 3.625628827832294, | |
| "tokens_seen": 1070858240 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0003117079629279365, | |
| "loss": 2.4386, | |
| "theoretical_loss": 3.625212154506246, | |
| "tokens_seen": 1072168960 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0003114715339512011, | |
| "loss": 2.4217, | |
| "theoretical_loss": 3.624796132677763, | |
| "tokens_seen": 1073479680 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0003112351049744657, | |
| "loss": 2.5306, | |
| "theoretical_loss": 3.624380760534567, | |
| "tokens_seen": 1074790400 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.0003109986759977303, | |
| "loss": 2.4447, | |
| "theoretical_loss": 3.6239660362716246, | |
| "tokens_seen": 1076101120 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 0.00031076224702099493, | |
| "loss": 2.4776, | |
| "theoretical_loss": 3.623551958091108, | |
| "tokens_seen": 1077411840 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003105258180442595, | |
| "loss": 2.4553, | |
| "theoretical_loss": 3.623138524202358, | |
| "tokens_seen": 1078722560 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003102893890675241, | |
| "loss": 2.4449, | |
| "theoretical_loss": 3.622725732821844, | |
| "tokens_seen": 1080033280 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003100529600907888, | |
| "loss": 2.4412, | |
| "theoretical_loss": 3.622313582173133, | |
| "tokens_seen": 1081344000 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003098165311140533, | |
| "loss": 2.4047, | |
| "theoretical_loss": 3.621902070486846, | |
| "tokens_seen": 1082654720 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00030958010213731797, | |
| "loss": 2.4902, | |
| "theoretical_loss": 3.6214911960006275, | |
| "tokens_seen": 1083965440 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00030934367316058257, | |
| "loss": 2.4576, | |
| "theoretical_loss": 3.621080956959104, | |
| "tokens_seen": 1085276160 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00030910724418384717, | |
| "loss": 2.4742, | |
| "theoretical_loss": 3.6206713516138507, | |
| "tokens_seen": 1086586880 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003088708152071118, | |
| "loss": 2.4061, | |
| "theoretical_loss": 3.620262378223357, | |
| "tokens_seen": 1087897600 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00030863438623037636, | |
| "loss": 2.4952, | |
| "theoretical_loss": 3.619854035052987, | |
| "tokens_seen": 1089208320 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.000308397957253641, | |
| "loss": 2.4592, | |
| "theoretical_loss": 3.619446320374947, | |
| "tokens_seen": 1090519040 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003081615282769056, | |
| "loss": 2.5259, | |
| "theoretical_loss": 3.61903923246825, | |
| "tokens_seen": 1091829760 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003079250993001702, | |
| "loss": 2.5005, | |
| "theoretical_loss": 3.6186327696186797, | |
| "tokens_seen": 1093140480 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00030768867032343486, | |
| "loss": 2.5096, | |
| "theoretical_loss": 3.618226930118757, | |
| "tokens_seen": 1094451200 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00030745224134669946, | |
| "loss": 2.4751, | |
| "theoretical_loss": 3.6178217122677045, | |
| "tokens_seen": 1095761920 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00030721581236996406, | |
| "loss": 2.5079, | |
| "theoretical_loss": 3.6174171143714124, | |
| "tokens_seen": 1097072640 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00030697938339322865, | |
| "loss": 2.5388, | |
| "theoretical_loss": 3.617013134742405, | |
| "tokens_seen": 1098383360 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003067429544164933, | |
| "loss": 2.5917, | |
| "theoretical_loss": 3.616609771699806, | |
| "tokens_seen": 1099694080 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003065065254397579, | |
| "loss": 2.5493, | |
| "theoretical_loss": 3.616207023569306, | |
| "tokens_seen": 1101004800 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003062700964630225, | |
| "loss": 2.5727, | |
| "theoretical_loss": 3.6158048886831287, | |
| "tokens_seen": 1102315520 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.00030603366748628715, | |
| "loss": 2.5582, | |
| "theoretical_loss": 3.6154033653799953, | |
| "tokens_seen": 1103626240 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "learning_rate": 0.0003057972385095517, | |
| "loss": 2.5653, | |
| "theoretical_loss": 3.615002452005096, | |
| "tokens_seen": 1104936960 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00030556080953281635, | |
| "loss": 2.5055, | |
| "theoretical_loss": 3.614602146910054, | |
| "tokens_seen": 1106247680 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.000305324380556081, | |
| "loss": 2.5686, | |
| "theoretical_loss": 3.614202448452893, | |
| "tokens_seen": 1107558400 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00030508795157934554, | |
| "loss": 2.5654, | |
| "theoretical_loss": 3.613803354998007, | |
| "tokens_seen": 1108869120 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0003048515226026102, | |
| "loss": 2.5513, | |
| "theoretical_loss": 3.6134048649161237, | |
| "tokens_seen": 1110179840 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0003046150936258748, | |
| "loss": 2.5548, | |
| "theoretical_loss": 3.6130069765842787, | |
| "tokens_seen": 1111490560 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0003043786646491394, | |
| "loss": 2.5599, | |
| "theoretical_loss": 3.6126096883857777, | |
| "tokens_seen": 1112801280 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "objective/train/docs_used": 560745, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.716017246246338, | |
| "objective/train/theoretical_loss": 3.61221299871017, | |
| "objective/train/tokens_used": 1134572000, | |
| "theoretical_loss": 3.61221299871017, | |
| "tokens_seen": 1114112000 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.000304142235672404, | |
| "loss": 2.5419, | |
| "theoretical_loss": 3.61221299871017, | |
| "tokens_seen": 1114112000 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00030390580669566864, | |
| "loss": 2.543, | |
| "theoretical_loss": 3.611816905953212, | |
| "tokens_seen": 1115422720 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00030366937771893323, | |
| "loss": 2.54, | |
| "theoretical_loss": 3.6114214085168417, | |
| "tokens_seen": 1116733440 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00030343294874219783, | |
| "loss": 2.5572, | |
| "theoretical_loss": 3.611026504809142, | |
| "tokens_seen": 1118044160 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0003031965197654625, | |
| "loss": 2.559, | |
| "theoretical_loss": 3.6106321932443137, | |
| "tokens_seen": 1119354880 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.000302960090788727, | |
| "loss": 2.5313, | |
| "theoretical_loss": 3.6102384722426435, | |
| "tokens_seen": 1120665600 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0003027236618119917, | |
| "loss": 2.5494, | |
| "theoretical_loss": 3.609845340230474, | |
| "tokens_seen": 1121976320 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00030248723283525633, | |
| "loss": 2.5043, | |
| "theoretical_loss": 3.6094527956401734, | |
| "tokens_seen": 1123287040 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0003022508038585209, | |
| "loss": 2.5192, | |
| "theoretical_loss": 3.6090608369101043, | |
| "tokens_seen": 1124597760 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0003020143748817855, | |
| "loss": 2.545, | |
| "theoretical_loss": 3.6086694624845954, | |
| "tokens_seen": 1125908480 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0003017779459050501, | |
| "loss": 2.5406, | |
| "theoretical_loss": 3.6082786708139114, | |
| "tokens_seen": 1127219200 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.0003015415169283147, | |
| "loss": 2.5063, | |
| "theoretical_loss": 3.6078884603542223, | |
| "tokens_seen": 1128529920 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00030130508795157937, | |
| "loss": 2.5123, | |
| "theoretical_loss": 3.607498829567576, | |
| "tokens_seen": 1129840640 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00030106865897484397, | |
| "loss": 2.4781, | |
| "theoretical_loss": 3.6071097769218676, | |
| "tokens_seen": 1131151360 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00030083222999810857, | |
| "loss": 2.4669, | |
| "theoretical_loss": 3.606721300890811, | |
| "tokens_seen": 1132462080 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 0.00030059580102137316, | |
| "loss": 2.4725, | |
| "theoretical_loss": 3.6063333999539102, | |
| "tokens_seen": 1133772800 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0003003593720446378, | |
| "loss": 2.4947, | |
| "theoretical_loss": 3.6059460725964305, | |
| "tokens_seen": 1135083520 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0003001229430679024, | |
| "loss": 2.5066, | |
| "theoretical_loss": 3.6055593173093694, | |
| "tokens_seen": 1136394240 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.000299886514091167, | |
| "loss": 2.5172, | |
| "theoretical_loss": 3.60517313258943, | |
| "tokens_seen": 1137704960 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00029965008511443166, | |
| "loss": 2.5151, | |
| "theoretical_loss": 3.604787516938992, | |
| "tokens_seen": 1139015680 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0002994136561376962, | |
| "loss": 2.5597, | |
| "theoretical_loss": 3.6044024688660823, | |
| "tokens_seen": 1140326400 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00029917722716096086, | |
| "loss": 2.5022, | |
| "theoretical_loss": 3.604017986884349, | |
| "tokens_seen": 1141637120 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00029894079818422545, | |
| "loss": 2.4671, | |
| "theoretical_loss": 3.603634069513035, | |
| "tokens_seen": 1142947840 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00029870436920749005, | |
| "loss": 2.5175, | |
| "theoretical_loss": 3.603250715276947, | |
| "tokens_seen": 1144258560 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0002984679402307547, | |
| "loss": 2.4882, | |
| "theoretical_loss": 3.6028679227064306, | |
| "tokens_seen": 1145569280 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0002982315112540193, | |
| "loss": 2.4754, | |
| "theoretical_loss": 3.6024856903373434, | |
| "tokens_seen": 1146880000 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0002979950822772839, | |
| "loss": 2.4985, | |
| "theoretical_loss": 3.602104016711027, | |
| "tokens_seen": 1148190720 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0002977586533005485, | |
| "loss": 2.4944, | |
| "theoretical_loss": 3.6017229003742806, | |
| "tokens_seen": 1149501440 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00029752222432381315, | |
| "loss": 2.5396, | |
| "theoretical_loss": 3.6013423398793347, | |
| "tokens_seen": 1150812160 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00029728579534707775, | |
| "loss": 2.5255, | |
| "theoretical_loss": 3.600962333783824, | |
| "tokens_seen": 1152122880 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00029704936637034234, | |
| "loss": 2.5681, | |
| "theoretical_loss": 3.6005828806507623, | |
| "tokens_seen": 1153433600 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.000296812937393607, | |
| "loss": 2.526, | |
| "theoretical_loss": 3.6002039790485147, | |
| "tokens_seen": 1154744320 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00029657650841687154, | |
| "loss": 2.5534, | |
| "theoretical_loss": 3.5998256275507723, | |
| "tokens_seen": 1156055040 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0002963400794401362, | |
| "loss": 2.4785, | |
| "theoretical_loss": 3.5994478247365276, | |
| "tokens_seen": 1157365760 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00029610365046340084, | |
| "loss": 2.5513, | |
| "theoretical_loss": 3.5990705691900464, | |
| "tokens_seen": 1158676480 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.0002958672214866654, | |
| "loss": 2.4692, | |
| "theoretical_loss": 3.5986938595008455, | |
| "tokens_seen": 1159987200 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "learning_rate": 0.00029563079250993004, | |
| "loss": 2.5105, | |
| "theoretical_loss": 3.5983176942636637, | |
| "tokens_seen": 1161297920 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00029539436353319463, | |
| "loss": 2.5134, | |
| "theoretical_loss": 3.597942072078439, | |
| "tokens_seen": 1162608640 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00029515793455645923, | |
| "loss": 2.4806, | |
| "theoretical_loss": 3.5975669915502833, | |
| "tokens_seen": 1163919360 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0002949215055797239, | |
| "loss": 2.5281, | |
| "theoretical_loss": 3.597192451289457, | |
| "tokens_seen": 1165230080 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0002946850766029885, | |
| "loss": 2.5286, | |
| "theoretical_loss": 3.5968184499113436, | |
| "tokens_seen": 1166540800 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0002944486476262531, | |
| "loss": 2.5412, | |
| "theoretical_loss": 3.5964449860364263, | |
| "tokens_seen": 1167851520 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0002942122186495177, | |
| "loss": 2.4897, | |
| "theoretical_loss": 3.5960720582902628, | |
| "tokens_seen": 1169162240 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00029397578967278233, | |
| "loss": 2.5326, | |
| "theoretical_loss": 3.595699665303462, | |
| "tokens_seen": 1170472960 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0002937393606960469, | |
| "loss": 2.4895, | |
| "theoretical_loss": 3.595327805711657, | |
| "tokens_seen": 1171783680 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0002935029317193115, | |
| "loss": 2.4776, | |
| "theoretical_loss": 3.5949564781554857, | |
| "tokens_seen": 1173094400 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0002932665027425762, | |
| "loss": 2.4995, | |
| "theoretical_loss": 3.5945856812805626, | |
| "tokens_seen": 1174405120 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0002930300737658407, | |
| "loss": 2.5078, | |
| "theoretical_loss": 3.594215413737457, | |
| "tokens_seen": 1175715840 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00029279364478910537, | |
| "loss": 2.4701, | |
| "theoretical_loss": 3.5938456741816682, | |
| "tokens_seen": 1177026560 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00029255721581236997, | |
| "loss": 2.4904, | |
| "theoretical_loss": 3.593476461273605, | |
| "tokens_seen": 1178337280 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "objective/train/docs_used": 592926, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.4532642364501953, | |
| "objective/train/theoretical_loss": 3.5931077736785584, | |
| "objective/train/tokens_used": 1200108000, | |
| "theoretical_loss": 3.5931077736785584, | |
| "tokens_seen": 1179648000 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00029232078683563456, | |
| "loss": 2.4595, | |
| "theoretical_loss": 3.5931077736785584, | |
| "tokens_seen": 1179648000 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0002920843578588992, | |
| "loss": 2.4661, | |
| "theoretical_loss": 3.5927396100666806, | |
| "tokens_seen": 1180958720 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0002918479288821638, | |
| "loss": 2.4604, | |
| "theoretical_loss": 3.592371969112962, | |
| "tokens_seen": 1182269440 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.0002916114999054284, | |
| "loss": 2.4523, | |
| "theoretical_loss": 3.5920048494972074, | |
| "tokens_seen": 1183580160 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.000291375070928693, | |
| "loss": 2.4164, | |
| "theoretical_loss": 3.5916382499040136, | |
| "tokens_seen": 1184890880 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00029113864195195766, | |
| "loss": 2.4312, | |
| "theoretical_loss": 3.591272169022747, | |
| "tokens_seen": 1186201600 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00029090221297522226, | |
| "loss": 2.4944, | |
| "theoretical_loss": 3.590906605547521, | |
| "tokens_seen": 1187512320 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 0.00029066578399848685, | |
| "loss": 2.5006, | |
| "theoretical_loss": 3.590541558177172, | |
| "tokens_seen": 1188823040 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0002904293550217515, | |
| "loss": 2.4771, | |
| "theoretical_loss": 3.5901770256152403, | |
| "tokens_seen": 1190133760 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.00029019292604501605, | |
| "loss": 2.4619, | |
| "theoretical_loss": 3.589813006569945, | |
| "tokens_seen": 1191444480 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0002899564970682807, | |
| "loss": 2.4777, | |
| "theoretical_loss": 3.589449499754164, | |
| "tokens_seen": 1192755200 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.00028972006809154535, | |
| "loss": 2.463, | |
| "theoretical_loss": 3.5890865038854103, | |
| "tokens_seen": 1194065920 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0002894836391148099, | |
| "loss": 2.5086, | |
| "theoretical_loss": 3.5887240176858115, | |
| "tokens_seen": 1195376640 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.00028924721013807455, | |
| "loss": 2.4832, | |
| "theoretical_loss": 3.588362039882088, | |
| "tokens_seen": 1196687360 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.00028901078116133915, | |
| "loss": 2.401, | |
| "theoretical_loss": 3.5880005692055317, | |
| "tokens_seen": 1197998080 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.00028877435218460374, | |
| "loss": 2.4258, | |
| "theoretical_loss": 3.5876396043919834, | |
| "tokens_seen": 1199308800 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0002885379232078684, | |
| "loss": 2.4724, | |
| "theoretical_loss": 3.587279144181813, | |
| "tokens_seen": 1200619520 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.000288301494231133, | |
| "loss": 2.4723, | |
| "theoretical_loss": 3.586919187319897, | |
| "tokens_seen": 1201930240 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0002880650652543976, | |
| "loss": 2.5126, | |
| "theoretical_loss": 3.5865597325555987, | |
| "tokens_seen": 1203240960 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0002878286362776622, | |
| "loss": 2.4961, | |
| "theoretical_loss": 3.5862007786427474, | |
| "tokens_seen": 1204551680 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.00028759220730092684, | |
| "loss": 2.4717, | |
| "theoretical_loss": 3.585842324339616, | |
| "tokens_seen": 1205862400 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0002873557783241914, | |
| "loss": 2.5088, | |
| "theoretical_loss": 3.5854843684089017, | |
| "tokens_seen": 1207173120 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.00028711934934745603, | |
| "loss": 2.4712, | |
| "theoretical_loss": 3.5851269096177054, | |
| "tokens_seen": 1208483840 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0002868829203707207, | |
| "loss": 2.4884, | |
| "theoretical_loss": 3.5847699467375103, | |
| "tokens_seen": 1209794560 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.00028664649139398523, | |
| "loss": 2.5599, | |
| "theoretical_loss": 3.584413478544163, | |
| "tokens_seen": 1211105280 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0002864100624172499, | |
| "loss": 2.4758, | |
| "theoretical_loss": 3.584057503817852, | |
| "tokens_seen": 1212416000 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0002861736334405145, | |
| "loss": 2.4395, | |
| "theoretical_loss": 3.5837020213430884, | |
| "tokens_seen": 1213726720 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0002859372044637791, | |
| "loss": 2.4717, | |
| "theoretical_loss": 3.583347029908686, | |
| "tokens_seen": 1215037440 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0002857007754870437, | |
| "loss": 2.4185, | |
| "theoretical_loss": 3.582992528307741, | |
| "tokens_seen": 1216348160 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "learning_rate": 0.0002854643465103083, | |
| "loss": 2.4256, | |
| "theoretical_loss": 3.5826385153376124, | |
| "tokens_seen": 1217658880 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002852279175335729, | |
| "loss": 2.4022, | |
| "theoretical_loss": 3.582284989799902, | |
| "tokens_seen": 1218969600 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002849914885568375, | |
| "loss": 2.4816, | |
| "theoretical_loss": 3.5819319505004366, | |
| "tokens_seen": 1220280320 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002847550595801021, | |
| "loss": 2.4522, | |
| "theoretical_loss": 3.581579396249245, | |
| "tokens_seen": 1221591040 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00028451863060336677, | |
| "loss": 2.4846, | |
| "theoretical_loss": 3.5812273258605436, | |
| "tokens_seen": 1222901760 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00028428220162663137, | |
| "loss": 2.4302, | |
| "theoretical_loss": 3.580875738152713, | |
| "tokens_seen": 1224212480 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00028404577264989596, | |
| "loss": 2.4204, | |
| "theoretical_loss": 3.5805246319482817, | |
| "tokens_seen": 1225523200 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00028380934367316056, | |
| "loss": 2.4361, | |
| "theoretical_loss": 3.5801740060739036, | |
| "tokens_seen": 1226833920 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002835729146964252, | |
| "loss": 2.4882, | |
| "theoretical_loss": 3.579823859360345, | |
| "tokens_seen": 1228144640 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002833364857196898, | |
| "loss": 2.476, | |
| "theoretical_loss": 3.5794741906424594, | |
| "tokens_seen": 1229455360 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002831000567429544, | |
| "loss": 2.5241, | |
| "theoretical_loss": 3.5791249987591733, | |
| "tokens_seen": 1230766080 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00028286362776621906, | |
| "loss": 2.4985, | |
| "theoretical_loss": 3.5787762825534664, | |
| "tokens_seen": 1232076800 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002826271987894836, | |
| "loss": 2.4425, | |
| "theoretical_loss": 3.578428040872351, | |
| "tokens_seen": 1233387520 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00028239076981274825, | |
| "loss": 2.4441, | |
| "theoretical_loss": 3.5780802725668575, | |
| "tokens_seen": 1234698240 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002821543408360129, | |
| "loss": 2.4903, | |
| "theoretical_loss": 3.5777329764920136, | |
| "tokens_seen": 1236008960 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00028191791185927745, | |
| "loss": 2.4451, | |
| "theoretical_loss": 3.5773861515068273, | |
| "tokens_seen": 1237319680 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002816814828825421, | |
| "loss": 2.4624, | |
| "theoretical_loss": 3.5770397964742675, | |
| "tokens_seen": 1238630400 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002814450539058067, | |
| "loss": 2.4855, | |
| "theoretical_loss": 3.5766939102612474, | |
| "tokens_seen": 1239941120 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002812086249290713, | |
| "loss": 2.476, | |
| "theoretical_loss": 3.5763484917386057, | |
| "tokens_seen": 1241251840 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.0002809721959523359, | |
| "loss": 2.4888, | |
| "theoretical_loss": 3.576003539781091, | |
| "tokens_seen": 1242562560 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00028073576697560054, | |
| "loss": 2.4709, | |
| "theoretical_loss": 3.5756590532673407, | |
| "tokens_seen": 1243873280 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "objective/train/docs_used": 624893, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.579110622406006, | |
| "objective/train/theoretical_loss": 3.5753150310798665, | |
| "objective/train/tokens_used": 1265644000, | |
| "theoretical_loss": 3.5753150310798665, | |
| "tokens_seen": 1245184000 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 0.00028049933799886514, | |
| "loss": 2.4214, | |
| "theoretical_loss": 3.5753150310798665, | |
| "tokens_seen": 1245184000 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00028026290902212974, | |
| "loss": 2.4559, | |
| "theoretical_loss": 3.5749714721050343, | |
| "tokens_seen": 1246494720 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.0002800264800453944, | |
| "loss": 2.4665, | |
| "theoretical_loss": 3.5746283752330505, | |
| "tokens_seen": 1247805440 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00027979005106865893, | |
| "loss": 2.4597, | |
| "theoretical_loss": 3.574285739357941, | |
| "tokens_seen": 1249116160 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.0002795536220919236, | |
| "loss": 2.4444, | |
| "theoretical_loss": 3.5739435633775365, | |
| "tokens_seen": 1250426880 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00027931719311518824, | |
| "loss": 2.4506, | |
| "theoretical_loss": 3.573601846193453, | |
| "tokens_seen": 1251737600 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.0002790807641384528, | |
| "loss": 2.499, | |
| "theoretical_loss": 3.573260586711079, | |
| "tokens_seen": 1253048320 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00027884433516171743, | |
| "loss": 2.4716, | |
| "theoretical_loss": 3.5729197838395543, | |
| "tokens_seen": 1254359040 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00027860790618498203, | |
| "loss": 2.4803, | |
| "theoretical_loss": 3.5725794364917554, | |
| "tokens_seen": 1255669760 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00027837147720824663, | |
| "loss": 2.4964, | |
| "theoretical_loss": 3.5722395435842778, | |
| "tokens_seen": 1256980480 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.0002781350482315113, | |
| "loss": 2.4236, | |
| "theoretical_loss": 3.5719001040374216, | |
| "tokens_seen": 1258291200 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.0002778986192547759, | |
| "loss": 2.4553, | |
| "theoretical_loss": 3.5715611167751717, | |
| "tokens_seen": 1259601920 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.0002776621902780405, | |
| "loss": 2.4184, | |
| "theoretical_loss": 3.571222580725184, | |
| "tokens_seen": 1260912640 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00027742576130130507, | |
| "loss": 2.457, | |
| "theoretical_loss": 3.5708844948187686, | |
| "tokens_seen": 1262223360 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.0002771893323245697, | |
| "loss": 2.4436, | |
| "theoretical_loss": 3.5705468579908715, | |
| "tokens_seen": 1263534080 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.0002769529033478343, | |
| "loss": 2.4645, | |
| "theoretical_loss": 3.5702096691800627, | |
| "tokens_seen": 1264844800 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.0002767164743710989, | |
| "loss": 2.483, | |
| "theoretical_loss": 3.5698729273285146, | |
| "tokens_seen": 1266155520 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00027648004539436357, | |
| "loss": 2.5248, | |
| "theoretical_loss": 3.5695366313819914, | |
| "tokens_seen": 1267466240 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.0002762436164176281, | |
| "loss": 2.5202, | |
| "theoretical_loss": 3.56920078028983, | |
| "tokens_seen": 1268776960 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00027600718744089277, | |
| "loss": 2.4975, | |
| "theoretical_loss": 3.5688653730049245, | |
| "tokens_seen": 1270087680 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00027577075846415736, | |
| "loss": 2.4559, | |
| "theoretical_loss": 3.5685304084837126, | |
| "tokens_seen": 1271398400 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "learning_rate": 0.00027553432948742196, | |
| "loss": 2.5062, | |
| "theoretical_loss": 3.5681958856861558, | |
| "tokens_seen": 1272709120 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0002752979005106866, | |
| "loss": 2.4937, | |
| "theoretical_loss": 3.5678618035757297, | |
| "tokens_seen": 1274019840 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0002750614715339512, | |
| "loss": 2.4728, | |
| "theoretical_loss": 3.5675281611194025, | |
| "tokens_seen": 1275330560 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0002748250425572158, | |
| "loss": 2.5095, | |
| "theoretical_loss": 3.5671949572876245, | |
| "tokens_seen": 1276641280 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0002745886135804804, | |
| "loss": 2.4447, | |
| "theoretical_loss": 3.5668621910543097, | |
| "tokens_seen": 1277952000 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00027435218460374506, | |
| "loss": 2.4882, | |
| "theoretical_loss": 3.566529861396822, | |
| "tokens_seen": 1279262720 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00027411575562700965, | |
| "loss": 2.5012, | |
| "theoretical_loss": 3.566197967295959, | |
| "tokens_seen": 1280573440 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00027387932665027425, | |
| "loss": 2.4544, | |
| "theoretical_loss": 3.56586650773594, | |
| "tokens_seen": 1281884160 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0002736428976735389, | |
| "loss": 2.4641, | |
| "theoretical_loss": 3.5655354817043863, | |
| "tokens_seen": 1283194880 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00027340646869680345, | |
| "loss": 2.4538, | |
| "theoretical_loss": 3.5652048881923095, | |
| "tokens_seen": 1284505600 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0002731700397200681, | |
| "loss": 2.4365, | |
| "theoretical_loss": 3.5648747261940974, | |
| "tokens_seen": 1285816320 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00027293361074333275, | |
| "loss": 2.4463, | |
| "theoretical_loss": 3.5645449947074956, | |
| "tokens_seen": 1287127040 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0002726971817665973, | |
| "loss": 2.4691, | |
| "theoretical_loss": 3.5642156927335975, | |
| "tokens_seen": 1288437760 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00027246075278986194, | |
| "loss": 2.4447, | |
| "theoretical_loss": 3.563886819276826, | |
| "tokens_seen": 1289748480 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00027222432381312654, | |
| "loss": 2.4527, | |
| "theoretical_loss": 3.5635583733449203, | |
| "tokens_seen": 1291059200 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00027198789483639114, | |
| "loss": 2.5079, | |
| "theoretical_loss": 3.5632303539489225, | |
| "tokens_seen": 1292369920 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0002717514658596558, | |
| "loss": 2.4833, | |
| "theoretical_loss": 3.562902760103161, | |
| "tokens_seen": 1293680640 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0002715150368829204, | |
| "loss": 2.4673, | |
| "theoretical_loss": 3.56257559082524, | |
| "tokens_seen": 1294991360 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.000271278607906185, | |
| "loss": 2.4464, | |
| "theoretical_loss": 3.562248845136021, | |
| "tokens_seen": 1296302080 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.0002710421789294496, | |
| "loss": 2.4174, | |
| "theoretical_loss": 3.5619225220596116, | |
| "tokens_seen": 1297612800 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00027080574995271423, | |
| "loss": 2.4684, | |
| "theoretical_loss": 3.56159662062335, | |
| "tokens_seen": 1298923520 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00027056932097597883, | |
| "loss": 2.4636, | |
| "theoretical_loss": 3.5612711398577916, | |
| "tokens_seen": 1300234240 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 0.00027033289199924343, | |
| "loss": 2.4581, | |
| "theoretical_loss": 3.5609460787966962, | |
| "tokens_seen": 1301544960 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0002700964630225081, | |
| "loss": 2.4231, | |
| "theoretical_loss": 3.560621436477012, | |
| "tokens_seen": 1302855680 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0002698600340457726, | |
| "loss": 2.5019, | |
| "theoretical_loss": 3.560297211938863, | |
| "tokens_seen": 1304166400 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0002696236050690373, | |
| "loss": 2.4015, | |
| "theoretical_loss": 3.5599734042255373, | |
| "tokens_seen": 1305477120 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0002693871760923019, | |
| "loss": 2.4166, | |
| "theoretical_loss": 3.559650012383469, | |
| "tokens_seen": 1306787840 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00026915074711556647, | |
| "loss": 2.4846, | |
| "theoretical_loss": 3.559327035462228, | |
| "tokens_seen": 1308098560 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0002689143181388311, | |
| "loss": 2.3862, | |
| "theoretical_loss": 3.5590044725145074, | |
| "tokens_seen": 1309409280 | |
| }, | |
| { | |
| "debugging/Self-BLEU-5": 0.5058204594636662, | |
| "debugging/distinct-1-grams": 0.8158854958318196, | |
| "debugging/distinct-2-grams": 0.977447166054761, | |
| "debugging/entropy-1-grams": 5.536854678153857, | |
| "debugging/entropy-2-grams": 6.187807570471228, | |
| "debugging/length": 519.25, | |
| "debugging/num_segments": 8, | |
| "epoch": 0.47, | |
| "objective/train/docs_used": 655766, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.4339327812194824, | |
| "objective/train/theoretical_loss": 3.5586823225961064, | |
| "objective/train/tokens_used": 1331180000, | |
| "theoretical_loss": 3.5586823225961064, | |
| "tokens_seen": 1310720000 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0002686778891620957, | |
| "loss": 2.4181, | |
| "theoretical_loss": 3.5586823225961064, | |
| "tokens_seen": 1310720000 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0002684414601853603, | |
| "loss": 2.4189, | |
| "theoretical_loss": 3.5583605847659214, | |
| "tokens_seen": 1312030720 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0002682050312086249, | |
| "loss": 2.4651, | |
| "theoretical_loss": 3.558039258085929, | |
| "tokens_seen": 1313341440 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00026796860223188957, | |
| "loss": 2.5319, | |
| "theoretical_loss": 3.5577183416211744, | |
| "tokens_seen": 1314652160 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00026773217325515416, | |
| "loss": 2.4463, | |
| "theoretical_loss": 3.5573978344397594, | |
| "tokens_seen": 1315962880 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00026749574427841876, | |
| "loss": 2.5003, | |
| "theoretical_loss": 3.5570777356128276, | |
| "tokens_seen": 1317273600 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0002672593153016834, | |
| "loss": 2.456, | |
| "theoretical_loss": 3.556758044214552, | |
| "tokens_seen": 1318584320 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00026702288632494796, | |
| "loss": 2.4585, | |
| "theoretical_loss": 3.5564387593221225, | |
| "tokens_seen": 1319895040 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0002667864573482126, | |
| "loss": 2.4403, | |
| "theoretical_loss": 3.556119880015733, | |
| "tokens_seen": 1321205760 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00026655002837147726, | |
| "loss": 2.4689, | |
| "theoretical_loss": 3.5558014053785687, | |
| "tokens_seen": 1322516480 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0002663135993947418, | |
| "loss": 2.4285, | |
| "theoretical_loss": 3.5554833344967927, | |
| "tokens_seen": 1323827200 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00026607717041800646, | |
| "loss": 2.4258, | |
| "theoretical_loss": 3.5551656664595344, | |
| "tokens_seen": 1325137920 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00026584074144127105, | |
| "loss": 2.4489, | |
| "theoretical_loss": 3.5548484003588756, | |
| "tokens_seen": 1326448640 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.00026560431246453565, | |
| "loss": 2.4222, | |
| "theoretical_loss": 3.554531535289841, | |
| "tokens_seen": 1327759360 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "learning_rate": 0.0002653678834878003, | |
| "loss": 2.4673, | |
| "theoretical_loss": 3.5542150703503808, | |
| "tokens_seen": 1329070080 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.0002651314545110649, | |
| "loss": 2.4297, | |
| "theoretical_loss": 3.553899004641363, | |
| "tokens_seen": 1330380800 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.0002648950255343295, | |
| "loss": 2.4342, | |
| "theoretical_loss": 3.55358333726656, | |
| "tokens_seen": 1331691520 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.0002646585965575941, | |
| "loss": 2.4331, | |
| "theoretical_loss": 3.553268067332634, | |
| "tokens_seen": 1333002240 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026442216758085875, | |
| "loss": 2.4072, | |
| "theoretical_loss": 3.5529531939491275, | |
| "tokens_seen": 1334312960 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.0002641857386041233, | |
| "loss": 2.4151, | |
| "theoretical_loss": 3.552638716228449, | |
| "tokens_seen": 1335623680 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026394930962738794, | |
| "loss": 2.484, | |
| "theoretical_loss": 3.552324633285865, | |
| "tokens_seen": 1336934400 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.0002637128806506526, | |
| "loss": 2.4541, | |
| "theoretical_loss": 3.5520109442394827, | |
| "tokens_seen": 1338245120 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026347645167391714, | |
| "loss": 2.4481, | |
| "theoretical_loss": 3.5516976482102414, | |
| "tokens_seen": 1339555840 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.0002632400226971818, | |
| "loss": 2.4962, | |
| "theoretical_loss": 3.5513847443218998, | |
| "tokens_seen": 1340866560 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.0002630035937204464, | |
| "loss": 2.4925, | |
| "theoretical_loss": 3.551072231701025, | |
| "tokens_seen": 1342177280 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.000262767164743711, | |
| "loss": 2.4094, | |
| "theoretical_loss": 3.550760109476979, | |
| "tokens_seen": 1343488000 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026253073576697563, | |
| "loss": 2.468, | |
| "theoretical_loss": 3.550448376781909, | |
| "tokens_seen": 1344798720 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026229430679024023, | |
| "loss": 2.447, | |
| "theoretical_loss": 3.550137032750734, | |
| "tokens_seen": 1346109440 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026205787781350483, | |
| "loss": 2.5067, | |
| "theoretical_loss": 3.5498260765211356, | |
| "tokens_seen": 1347420160 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.0002618214488367694, | |
| "loss": 2.4833, | |
| "theoretical_loss": 3.549515507233543, | |
| "tokens_seen": 1348730880 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.000261585019860034, | |
| "loss": 2.4671, | |
| "theoretical_loss": 3.5492053240311257, | |
| "tokens_seen": 1350041600 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.0002613485908832987, | |
| "loss": 2.4447, | |
| "theoretical_loss": 3.5488955260597783, | |
| "tokens_seen": 1351352320 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.0002611121619065633, | |
| "loss": 2.4603, | |
| "theoretical_loss": 3.5485861124681124, | |
| "tokens_seen": 1352663040 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026087573292982787, | |
| "loss": 2.4596, | |
| "theoretical_loss": 3.5482770824074428, | |
| "tokens_seen": 1353973760 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.00026063930395309247, | |
| "loss": 2.4567, | |
| "theoretical_loss": 3.5479684350317777, | |
| "tokens_seen": 1355284480 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.0002604028749763571, | |
| "loss": 2.4831, | |
| "theoretical_loss": 3.5476601694978083, | |
| "tokens_seen": 1356595200 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 0.0002601664459996217, | |
| "loss": 2.4635, | |
| "theoretical_loss": 3.5473522849648944, | |
| "tokens_seen": 1357905920 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.0002599300170228863, | |
| "loss": 2.4339, | |
| "theoretical_loss": 3.547044780595058, | |
| "tokens_seen": 1359216640 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00025969358804615097, | |
| "loss": 2.4639, | |
| "theoretical_loss": 3.5467376555529686, | |
| "tokens_seen": 1360527360 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.0002594571590694155, | |
| "loss": 2.4921, | |
| "theoretical_loss": 3.546430909005934, | |
| "tokens_seen": 1361838080 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00025922073009268016, | |
| "loss": 2.4489, | |
| "theoretical_loss": 3.5461245401238894, | |
| "tokens_seen": 1363148800 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.0002589843011159448, | |
| "loss": 2.4698, | |
| "theoretical_loss": 3.545818548079385, | |
| "tokens_seen": 1364459520 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00025874787213920936, | |
| "loss": 2.4541, | |
| "theoretical_loss": 3.545512932047579, | |
| "tokens_seen": 1365770240 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.000258511443162474, | |
| "loss": 2.5069, | |
| "theoretical_loss": 3.5452076912062216, | |
| "tokens_seen": 1367080960 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.0002582750141857386, | |
| "loss": 2.454, | |
| "theoretical_loss": 3.54490282473565, | |
| "tokens_seen": 1368391680 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.0002580385852090032, | |
| "loss": 2.4961, | |
| "theoretical_loss": 3.544598331818772, | |
| "tokens_seen": 1369702400 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.0002578021562322678, | |
| "loss": 2.4931, | |
| "theoretical_loss": 3.5442942116410614, | |
| "tokens_seen": 1371013120 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00025756572725553245, | |
| "loss": 2.485, | |
| "theoretical_loss": 3.5439904633905424, | |
| "tokens_seen": 1372323840 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00025732929827879705, | |
| "loss": 2.4956, | |
| "theoretical_loss": 3.5436870862577825, | |
| "tokens_seen": 1373634560 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00025709286930206165, | |
| "loss": 2.476, | |
| "theoretical_loss": 3.5433840794358806, | |
| "tokens_seen": 1374945280 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "objective/train/docs_used": 687492, | |
| "objective/train/instantaneous_batch_size": 32, | |
| "objective/train/instantaneous_microbatch_size": 32768, | |
| "objective/train/original_loss": 2.51528263092041, | |
| "objective/train/theoretical_loss": 3.5430814421204575, | |
| "objective/train/tokens_used": 1396716000, | |
| "theoretical_loss": 3.5430814421204575, | |
| "tokens_seen": 1376256000 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.0002568564403253263, | |
| "loss": 2.5171, | |
| "theoretical_loss": 3.5430814421204575, | |
| "tokens_seen": 1376256000 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00025662001134859084, | |
| "loss": 2.5266, | |
| "theoretical_loss": 3.5427791735096448, | |
| "tokens_seen": 1377566720 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.0002563835823718555, | |
| "loss": 2.51, | |
| "theoretical_loss": 3.542477272804075, | |
| "tokens_seen": 1378877440 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00025614715339512015, | |
| "loss": 2.5329, | |
| "theoretical_loss": 3.542175739206872, | |
| "tokens_seen": 1380188160 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.0002559107244183847, | |
| "loss": 2.4585, | |
| "theoretical_loss": 3.541874571923639, | |
| "tokens_seen": 1381498880 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00025567429544164934, | |
| "loss": 2.4753, | |
| "theoretical_loss": 3.5415737701624517, | |
| "tokens_seen": 1382809600 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00025543786646491394, | |
| "loss": 2.4891, | |
| "theoretical_loss": 3.5412733331338453, | |
| "tokens_seen": 1384120320 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "learning_rate": 0.00025520143748817854, | |
| "loss": 2.5257, | |
| "theoretical_loss": 3.5409732600508055, | |
| "tokens_seen": 1385431040 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.0002549650085114432, | |
| "loss": 2.4603, | |
| "theoretical_loss": 3.5406735501287594, | |
| "tokens_seen": 1386741760 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.0002547285795347078, | |
| "loss": 2.5005, | |
| "theoretical_loss": 3.5403742025855647, | |
| "tokens_seen": 1388052480 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.0002544921505579724, | |
| "loss": 2.4448, | |
| "theoretical_loss": 3.5400752166415, | |
| "tokens_seen": 1389363200 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.000254255721581237, | |
| "loss": 2.4745, | |
| "theoretical_loss": 3.5397765915192547, | |
| "tokens_seen": 1390673920 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.00025401929260450163, | |
| "loss": 2.4534, | |
| "theoretical_loss": 3.5394783264439207, | |
| "tokens_seen": 1391984640 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.00025378286362776623, | |
| "loss": 2.4716, | |
| "theoretical_loss": 3.5391804206429818, | |
| "tokens_seen": 1393295360 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.0002535464346510308, | |
| "loss": 2.4854, | |
| "theoretical_loss": 3.538882873346303, | |
| "tokens_seen": 1394606080 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.0002533100056742955, | |
| "loss": 2.4159, | |
| "theoretical_loss": 3.5385856837861223, | |
| "tokens_seen": 1395916800 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.00025307357669756, | |
| "loss": 2.4517, | |
| "theoretical_loss": 3.538288851197042, | |
| "tokens_seen": 1397227520 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.00025283714772082467, | |
| "loss": 2.4685, | |
| "theoretical_loss": 3.537992374816016, | |
| "tokens_seen": 1398538240 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 0.00025260071874408927, | |
| "loss": 2.4713, | |
| "theoretical_loss": 3.537696253882345, | |
| "tokens_seen": 1399848960 | |
| } | |
| ], | |
| "max_steps": 42724, | |
| "num_train_epochs": 9223372036854775807, | |
| "total_flos": 7.14460209610752e+17, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |