|
{ |
|
"best_metric": null, |
|
"best_model_checkpoint": null, |
|
"epoch": 0.5, |
|
"global_step": 21362, |
|
"is_hyper_param_search": false, |
|
"is_local_process_zero": true, |
|
"is_world_process_zero": true, |
|
"log_history": [ |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 1.1682242990654204e-06, |
|
"loss": 10.8658, |
|
"theoretical_loss": 20.812802717066116, |
|
"tokens_seen": 65536 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 2.336448598130841e-05, |
|
"loss": 9.7159, |
|
"theoretical_loss": 10.363548859282295, |
|
"tokens_seen": 1310720 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 4.672897196261682e-05, |
|
"loss": 8.8339, |
|
"theoretical_loss": 8.96027440542832, |
|
"tokens_seen": 2621440 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 7.009345794392523e-05, |
|
"loss": 7.9042, |
|
"theoretical_loss": 8.257160837770828, |
|
"tokens_seen": 3932160 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 9.345794392523364e-05, |
|
"loss": 7.0675, |
|
"theoretical_loss": 7.80455017047313, |
|
"tokens_seen": 5242880 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 0.00011682242990654206, |
|
"loss": 6.5675, |
|
"theoretical_loss": 7.477743633566995, |
|
"tokens_seen": 6553600 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 0.00014018691588785047, |
|
"loss": 6.3483, |
|
"theoretical_loss": 7.225472152031461, |
|
"tokens_seen": 7864320 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 0.00016355140186915886, |
|
"loss": 6.0955, |
|
"theoretical_loss": 7.022000360175642, |
|
"tokens_seen": 9175040 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 0.00018691588785046728, |
|
"loss": 5.9627, |
|
"theoretical_loss": 6.852706072125594, |
|
"tokens_seen": 10485760 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 0.0002102803738317757, |
|
"loss": 5.7575, |
|
"theoretical_loss": 6.7085424020813775, |
|
"tokens_seen": 11796480 |
|
}, |
|
{ |
|
"epoch": 0.0, |
|
"learning_rate": 0.00023364485981308412, |
|
"loss": 5.6734, |
|
"theoretical_loss": 6.583551143945378, |
|
"tokens_seen": 13107200 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0002570093457943925, |
|
"loss": 5.5759, |
|
"theoretical_loss": 6.4736158466608344, |
|
"tokens_seen": 14417920 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.00028037383177570094, |
|
"loss": 5.4962, |
|
"theoretical_loss": 6.375782617839931, |
|
"tokens_seen": 15728640 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.00030373831775700936, |
|
"loss": 5.387, |
|
"theoretical_loss": 6.287865514264695, |
|
"tokens_seen": 17039360 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0003271028037383177, |
|
"loss": 5.31, |
|
"theoretical_loss": 6.208205077800142, |
|
"tokens_seen": 18350080 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.00035046728971962614, |
|
"loss": 5.2546, |
|
"theoretical_loss": 6.13551414745929, |
|
"tokens_seen": 19660800 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.00037383177570093456, |
|
"loss": 5.196, |
|
"theoretical_loss": 6.068775822887412, |
|
"tokens_seen": 20971520 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.000397196261682243, |
|
"loss": 5.1134, |
|
"theoretical_loss": 6.007173857995872, |
|
"tokens_seen": 22282240 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0004205607476635514, |
|
"loss": 5.0982, |
|
"theoretical_loss": 5.950043919222669, |
|
"tokens_seen": 23592960 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0004439252336448598, |
|
"loss": 5.0706, |
|
"theoretical_loss": 5.89683866811525, |
|
"tokens_seen": 24903680 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.00046728971962616824, |
|
"loss": 4.9896, |
|
"theoretical_loss": 5.847102241784889, |
|
"tokens_seen": 26214400 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0004906542056074767, |
|
"loss": 4.8916, |
|
"theoretical_loss": 5.80045126829951, |
|
"tokens_seen": 27525120 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0004998581426139588, |
|
"loss": 4.9238, |
|
"theoretical_loss": 5.756560518460697, |
|
"tokens_seen": 28835840 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0004996217136372233, |
|
"loss": 4.8673, |
|
"theoretical_loss": 5.715151906614886, |
|
"tokens_seen": 30146560 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.000499385284660488, |
|
"loss": 4.7589, |
|
"theoretical_loss": 5.675985950013518, |
|
"tokens_seen": 31457280 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0004991488556837526, |
|
"loss": 4.7546, |
|
"theoretical_loss": 5.638855059590318, |
|
"tokens_seen": 32768000 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0004989124267070172, |
|
"loss": 4.6908, |
|
"theoretical_loss": 5.603578213244772, |
|
"tokens_seen": 34078720 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0004986759977302818, |
|
"loss": 4.7011, |
|
"theoretical_loss": 5.569996685496657, |
|
"tokens_seen": 35389440 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0004984395687535465, |
|
"loss": 4.6645, |
|
"theoretical_loss": 5.53797059334095, |
|
"tokens_seen": 36700160 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.000498203139776811, |
|
"loss": 4.5751, |
|
"theoretical_loss": 5.507376079224074, |
|
"tokens_seen": 38010880 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0004979667108000757, |
|
"loss": 4.493, |
|
"theoretical_loss": 5.478102996075402, |
|
"tokens_seen": 39321600 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0004977302818233403, |
|
"loss": 4.5319, |
|
"theoretical_loss": 5.450052991438716, |
|
"tokens_seen": 40632320 |
|
}, |
|
{ |
|
"epoch": 0.01, |
|
"learning_rate": 0.0004974938528466049, |
|
"loss": 4.4256, |
|
"theoretical_loss": 5.4231379114505165, |
|
"tokens_seen": 41943040 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004972574238698695, |
|
"loss": 4.4082, |
|
"theoretical_loss": 5.397278463098006, |
|
"tokens_seen": 43253760 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004970209948931342, |
|
"loss": 4.4953, |
|
"theoretical_loss": 5.372403086519811, |
|
"tokens_seen": 44564480 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004967845659163987, |
|
"loss": 4.3646, |
|
"theoretical_loss": 5.348446999254799, |
|
"tokens_seen": 45875200 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004965481369396634, |
|
"loss": 4.4128, |
|
"theoretical_loss": 5.3253513821291545, |
|
"tokens_seen": 47185920 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.000496311707962928, |
|
"loss": 4.3023, |
|
"theoretical_loss": 5.303062682496617, |
|
"tokens_seen": 48496640 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004960752789861926, |
|
"loss": 4.3546, |
|
"theoretical_loss": 5.281532015245618, |
|
"tokens_seen": 49807360 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004958388500094572, |
|
"loss": 4.2839, |
|
"theoretical_loss": 5.260714645678606, |
|
"tokens_seen": 51118080 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004956024210327218, |
|
"loss": 4.2506, |
|
"theoretical_loss": 5.240569541288942, |
|
"tokens_seen": 52428800 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004953659920559864, |
|
"loss": 4.2113, |
|
"theoretical_loss": 5.221058981785759, |
|
"tokens_seen": 53739520 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.000495129563079251, |
|
"loss": 4.2045, |
|
"theoretical_loss": 5.202148218579597, |
|
"tokens_seen": 55050240 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004948931341025156, |
|
"loss": 4.1605, |
|
"theoretical_loss": 5.1838051764423945, |
|
"tokens_seen": 56360960 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004946567051257802, |
|
"loss": 4.1034, |
|
"theoretical_loss": 5.166000191271213, |
|
"tokens_seen": 57671680 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004944202761490448, |
|
"loss": 4.1023, |
|
"theoretical_loss": 5.148705778875405, |
|
"tokens_seen": 58982400 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004941838471723094, |
|
"loss": 4.015, |
|
"theoretical_loss": 5.131896430517482, |
|
"tokens_seen": 60293120 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.000493947418195574, |
|
"loss": 3.9781, |
|
"theoretical_loss": 5.1155484316045445, |
|
"tokens_seen": 61603840 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004937109892188386, |
|
"loss": 3.9207, |
|
"theoretical_loss": 5.099639700477905, |
|
"tokens_seen": 62914560 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004934745602421033, |
|
"loss": 3.8888, |
|
"theoretical_loss": 5.084149644705477, |
|
"tokens_seen": 64225280 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"objective/train/docs_used": 46774, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 3.873431921005249, |
|
"objective/train/theoretical_loss": 5.06905903266221, |
|
"objective/train/tokens_used": 85996000, |
|
"theoretical_loss": 5.06905903266221, |
|
"tokens_seen": 65536000 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004932381312653678, |
|
"loss": 3.7964, |
|
"theoretical_loss": 5.06905903266221, |
|
"tokens_seen": 65536000 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004930017022886325, |
|
"loss": 3.761, |
|
"theoretical_loss": 5.054349878502332, |
|
"tokens_seen": 66846720 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004927652733118971, |
|
"loss": 3.8266, |
|
"theoretical_loss": 5.04000533889449, |
|
"tokens_seen": 68157440 |
|
}, |
|
{ |
|
"epoch": 0.02, |
|
"learning_rate": 0.0004925288443351617, |
|
"loss": 3.7863, |
|
"theoretical_loss": 5.02600962011627, |
|
"tokens_seen": 69468160 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004922924153584263, |
|
"loss": 3.7525, |
|
"theoretical_loss": 5.012347894295022, |
|
"tokens_seen": 70778880 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.000492055986381691, |
|
"loss": 3.7151, |
|
"theoretical_loss": 4.999006223743629, |
|
"tokens_seen": 72089600 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004918195574049555, |
|
"loss": 3.7239, |
|
"theoretical_loss": 4.985971492477399, |
|
"tokens_seen": 73400320 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004915831284282202, |
|
"loss": 3.7674, |
|
"theoretical_loss": 4.973231344115766, |
|
"tokens_seen": 74711040 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004913466994514848, |
|
"loss": 3.6962, |
|
"theoretical_loss": 4.960774125473073, |
|
"tokens_seen": 76021760 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004911102704747494, |
|
"loss": 3.5964, |
|
"theoretical_loss": 4.948588835229052, |
|
"tokens_seen": 77332480 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.000490873841498014, |
|
"loss": 3.7099, |
|
"theoretical_loss": 4.9366650771441005, |
|
"tokens_seen": 78643200 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004906374125212787, |
|
"loss": 3.5834, |
|
"theoretical_loss": 4.924993017348572, |
|
"tokens_seen": 79953920 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004904009835445432, |
|
"loss": 3.6168, |
|
"theoretical_loss": 4.913563345291011, |
|
"tokens_seen": 81264640 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004901645545678079, |
|
"loss": 3.6325, |
|
"theoretical_loss": 4.90236723797842, |
|
"tokens_seen": 82575360 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004899281255910725, |
|
"loss": 3.6213, |
|
"theoretical_loss": 4.891396327183692, |
|
"tokens_seen": 83886080 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004896916966143371, |
|
"loss": 3.5461, |
|
"theoretical_loss": 4.880642669331891, |
|
"tokens_seen": 85196800 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004894552676376017, |
|
"loss": 3.6126, |
|
"theoretical_loss": 4.8700987178090696, |
|
"tokens_seen": 86507520 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004892188386608663, |
|
"loss": 3.5591, |
|
"theoretical_loss": 4.859757297465288, |
|
"tokens_seen": 87818240 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004889824096841309, |
|
"loss": 3.5483, |
|
"theoretical_loss": 4.84961158110812, |
|
"tokens_seen": 89128960 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004887459807073955, |
|
"loss": 3.5316, |
|
"theoretical_loss": 4.839655067804491, |
|
"tokens_seen": 90439680 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004885095517306601, |
|
"loss": 3.4951, |
|
"theoretical_loss": 4.829881562827778, |
|
"tokens_seen": 91750400 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.0004882731227539247, |
|
"loss": 3.4582, |
|
"theoretical_loss": 4.820285159103877, |
|
"tokens_seen": 93061120 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.00048803669377718936, |
|
"loss": 3.4803, |
|
"theoretical_loss": 4.810860220024846, |
|
"tokens_seen": 94371840 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.00048780026480045396, |
|
"loss": 3.4445, |
|
"theoretical_loss": 4.801601363511853, |
|
"tokens_seen": 95682560 |
|
}, |
|
{ |
|
"epoch": 0.03, |
|
"learning_rate": 0.00048756383582371855, |
|
"loss": 3.4563, |
|
"theoretical_loss": 4.792503447220913, |
|
"tokens_seen": 96993280 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0004873274068469832, |
|
"loss": 3.5527, |
|
"theoretical_loss": 4.783561554795222, |
|
"tokens_seen": 98304000 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0004870909778702478, |
|
"loss": 3.4383, |
|
"theoretical_loss": 4.774770983077213, |
|
"tokens_seen": 99614720 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0004868545488935124, |
|
"loss": 3.5052, |
|
"theoretical_loss": 4.766127230201681, |
|
"tokens_seen": 100925440 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.000486618119916777, |
|
"loss": 3.4738, |
|
"theoretical_loss": 4.757625984498683, |
|
"tokens_seen": 102236160 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.00048638169094004165, |
|
"loss": 3.5177, |
|
"theoretical_loss": 4.7492631141415815, |
|
"tokens_seen": 103546880 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0004861452619633062, |
|
"loss": 3.4476, |
|
"theoretical_loss": 4.741034657481437, |
|
"tokens_seen": 104857600 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.00048590883298657084, |
|
"loss": 3.4467, |
|
"theoretical_loss": 4.732936814014305, |
|
"tokens_seen": 106168320 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0004856724040098355, |
|
"loss": 3.4405, |
|
"theoretical_loss": 4.724965935932751, |
|
"tokens_seen": 107479040 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.00048543597503310004, |
|
"loss": 3.4147, |
|
"theoretical_loss": 4.717118520217186, |
|
"tokens_seen": 108789760 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0004851995460563647, |
|
"loss": 3.4864, |
|
"theoretical_loss": 4.709391201226483, |
|
"tokens_seen": 110100480 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0004849631170796293, |
|
"loss": 3.4632, |
|
"theoretical_loss": 4.70178074375083, |
|
"tokens_seen": 111411200 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0004847266881028939, |
|
"loss": 3.5115, |
|
"theoretical_loss": 4.694284036492922, |
|
"tokens_seen": 112721920 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.00048449025912615854, |
|
"loss": 3.4733, |
|
"theoretical_loss": 4.686898085946441, |
|
"tokens_seen": 114032640 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.00048425383014942313, |
|
"loss": 3.4572, |
|
"theoretical_loss": 4.6796200106433545, |
|
"tokens_seen": 115343360 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.00048401740117268773, |
|
"loss": 3.4342, |
|
"theoretical_loss": 4.6724470357438985, |
|
"tokens_seen": 116654080 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.00048378097219595233, |
|
"loss": 3.4469, |
|
"theoretical_loss": 4.665376487945246, |
|
"tokens_seen": 117964800 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.000483544543219217, |
|
"loss": 3.3911, |
|
"theoretical_loss": 4.658405790686763, |
|
"tokens_seen": 119275520 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0004833081142424816, |
|
"loss": 3.3665, |
|
"theoretical_loss": 4.6515324596315395, |
|
"tokens_seen": 120586240 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.0004830716852657462, |
|
"loss": 3.3685, |
|
"theoretical_loss": 4.6447540984054605, |
|
"tokens_seen": 121896960 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.00048283525628901083, |
|
"loss": 3.4102, |
|
"theoretical_loss": 4.638068394576544, |
|
"tokens_seen": 123207680 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.00048259882731227537, |
|
"loss": 3.4189, |
|
"theoretical_loss": 4.631473115858608, |
|
"tokens_seen": 124518400 |
|
}, |
|
{ |
|
"epoch": 0.04, |
|
"learning_rate": 0.00048236239833554, |
|
"loss": 3.4026, |
|
"theoretical_loss": 4.624966106524518, |
|
"tokens_seen": 125829120 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0004821259693588047, |
|
"loss": 3.3896, |
|
"theoretical_loss": 4.61854528401543, |
|
"tokens_seen": 127139840 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0004818895403820692, |
|
"loss": 3.4057, |
|
"theoretical_loss": 4.6122086357333725, |
|
"tokens_seen": 128450560 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.00048165311140533387, |
|
"loss": 3.3771, |
|
"theoretical_loss": 4.605954216005544, |
|
"tokens_seen": 129761280 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"objective/train/docs_used": 82490, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 3.441986322402954, |
|
"objective/train/theoretical_loss": 4.599780143209469, |
|
"objective/train/tokens_used": 151532000, |
|
"theoretical_loss": 4.599780143209469, |
|
"tokens_seen": 131072000 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.00048141668242859847, |
|
"loss": 3.3107, |
|
"theoretical_loss": 4.599780143209469, |
|
"tokens_seen": 131072000 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.00048118025345186306, |
|
"loss": 3.3261, |
|
"theoretical_loss": 4.593684597048995, |
|
"tokens_seen": 132382720 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0004809438244751277, |
|
"loss": 3.3271, |
|
"theoretical_loss": 4.587665815971789, |
|
"tokens_seen": 133693440 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0004807073954983923, |
|
"loss": 3.3729, |
|
"theoretical_loss": 4.581722094719703, |
|
"tokens_seen": 135004160 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0004804709665216569, |
|
"loss": 3.3567, |
|
"theoretical_loss": 4.575851782003932, |
|
"tokens_seen": 136314880 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0004802345375449215, |
|
"loss": 3.3513, |
|
"theoretical_loss": 4.570053278297481, |
|
"tokens_seen": 137625600 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.00047999810856818616, |
|
"loss": 3.2978, |
|
"theoretical_loss": 4.564325033737986, |
|
"tokens_seen": 138936320 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0004797616795914507, |
|
"loss": 3.3, |
|
"theoretical_loss": 4.558665546134362, |
|
"tokens_seen": 140247040 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.00047952525061471536, |
|
"loss": 3.362, |
|
"theoretical_loss": 4.5530733590712344, |
|
"tokens_seen": 141557760 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.00047928882163798, |
|
"loss": 3.313, |
|
"theoretical_loss": 4.547547060105504, |
|
"tokens_seen": 142868480 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.00047905239266124455, |
|
"loss": 3.3083, |
|
"theoretical_loss": 4.542085279049763, |
|
"tokens_seen": 144179200 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0004788159636845092, |
|
"loss": 3.2759, |
|
"theoretical_loss": 4.53668668633763, |
|
"tokens_seen": 145489920 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.00047857953470777375, |
|
"loss": 3.264, |
|
"theoretical_loss": 4.531349991466399, |
|
"tokens_seen": 146800640 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0004783431057310384, |
|
"loss": 3.286, |
|
"theoretical_loss": 4.526073941512697, |
|
"tokens_seen": 148111360 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.00047810667675430305, |
|
"loss": 3.3484, |
|
"theoretical_loss": 4.520857319717103, |
|
"tokens_seen": 149422080 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.0004778702477775676, |
|
"loss": 3.2817, |
|
"theoretical_loss": 4.515698944133961, |
|
"tokens_seen": 150732800 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.00047763381880083224, |
|
"loss": 3.2996, |
|
"theoretical_loss": 4.510597666342842, |
|
"tokens_seen": 152043520 |
|
}, |
|
{ |
|
"epoch": 0.05, |
|
"learning_rate": 0.00047739738982409684, |
|
"loss": 3.2335, |
|
"theoretical_loss": 4.5055523702183375, |
|
"tokens_seen": 153354240 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.00047716096084736144, |
|
"loss": 3.2571, |
|
"theoretical_loss": 4.500561970755069, |
|
"tokens_seen": 154664960 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0004769245318706261, |
|
"loss": 3.2895, |
|
"theoretical_loss": 4.495625412944987, |
|
"tokens_seen": 155975680 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0004766881028938907, |
|
"loss": 3.3261, |
|
"theoretical_loss": 4.490741670704231, |
|
"tokens_seen": 157286400 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0004764516739171553, |
|
"loss": 3.3318, |
|
"theoretical_loss": 4.485909745846939, |
|
"tokens_seen": 158597120 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0004762152449404199, |
|
"loss": 3.2679, |
|
"theoretical_loss": 4.481128667103624, |
|
"tokens_seen": 159907840 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.00047597881596368453, |
|
"loss": 3.2825, |
|
"theoretical_loss": 4.476397489181794, |
|
"tokens_seen": 161218560 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.00047574238698694913, |
|
"loss": 3.2408, |
|
"theoretical_loss": 4.471715291866703, |
|
"tokens_seen": 162529280 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.00047550595801021373, |
|
"loss": 3.2514, |
|
"theoretical_loss": 4.467081179160184, |
|
"tokens_seen": 163840000 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0004752695290334784, |
|
"loss": 3.3081, |
|
"theoretical_loss": 4.462494278455688, |
|
"tokens_seen": 165150720 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0004750331000567429, |
|
"loss": 3.267, |
|
"theoretical_loss": 4.457953739747708, |
|
"tokens_seen": 166461440 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0004747966710800076, |
|
"loss": 3.1616, |
|
"theoretical_loss": 4.453458734873907, |
|
"tokens_seen": 167772160 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0004745602421032722, |
|
"loss": 3.2412, |
|
"theoretical_loss": 4.449008456788359, |
|
"tokens_seen": 169082880 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.00047432381312653677, |
|
"loss": 3.3008, |
|
"theoretical_loss": 4.4446021188643945, |
|
"tokens_seen": 170393600 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0004740873841498014, |
|
"loss": 3.2234, |
|
"theoretical_loss": 4.440238954225603, |
|
"tokens_seen": 171704320 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.000473850955173066, |
|
"loss": 3.2253, |
|
"theoretical_loss": 4.435918215103694, |
|
"tokens_seen": 173015040 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0004736145261963306, |
|
"loss": 3.2172, |
|
"theoretical_loss": 4.431639172221903, |
|
"tokens_seen": 174325760 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0004733780972195952, |
|
"loss": 3.186, |
|
"theoretical_loss": 4.427401114202768, |
|
"tokens_seen": 175636480 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.00047314166824285987, |
|
"loss": 3.1797, |
|
"theoretical_loss": 4.4232033469991245, |
|
"tokens_seen": 176947200 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.00047290523926612446, |
|
"loss": 3.199, |
|
"theoretical_loss": 4.419045193347259, |
|
"tokens_seen": 178257920 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.00047266881028938906, |
|
"loss": 3.1988, |
|
"theoretical_loss": 4.414925992241184, |
|
"tokens_seen": 179568640 |
|
}, |
|
{ |
|
"epoch": 0.06, |
|
"learning_rate": 0.0004724323813126537, |
|
"loss": 3.1514, |
|
"theoretical_loss": 4.410845098427085, |
|
"tokens_seen": 180879360 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.00047219595233591826, |
|
"loss": 3.1928, |
|
"theoretical_loss": 4.406801881917029, |
|
"tokens_seen": 182190080 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0004719595233591829, |
|
"loss": 3.1594, |
|
"theoretical_loss": 4.402795727521044, |
|
"tokens_seen": 183500800 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.00047172309438244756, |
|
"loss": 3.1795, |
|
"theoretical_loss": 4.398826034396784, |
|
"tokens_seen": 184811520 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0004714866654057121, |
|
"loss": 3.1768, |
|
"theoretical_loss": 4.394892215615966, |
|
"tokens_seen": 186122240 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.00047125023642897675, |
|
"loss": 3.1642, |
|
"theoretical_loss": 4.390993697746873, |
|
"tokens_seen": 187432960 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.00047101380745224135, |
|
"loss": 3.2163, |
|
"theoretical_loss": 4.387129920452183, |
|
"tokens_seen": 188743680 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.00047077737847550595, |
|
"loss": 3.162, |
|
"theoretical_loss": 4.383300336101497, |
|
"tokens_seen": 190054400 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0004705409494987706, |
|
"loss": 3.1984, |
|
"theoretical_loss": 4.379504409397913, |
|
"tokens_seen": 191365120 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0004703045205220352, |
|
"loss": 3.1694, |
|
"theoretical_loss": 4.375741617018028, |
|
"tokens_seen": 192675840 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0004700680915452998, |
|
"loss": 3.2412, |
|
"theoretical_loss": 4.372011447264842, |
|
"tokens_seen": 193986560 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0004698316625685644, |
|
"loss": 3.2147, |
|
"theoretical_loss": 4.368313399732967, |
|
"tokens_seen": 195297280 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"objective/train/docs_used": 119577, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.7380270957946777, |
|
"objective/train/theoretical_loss": 4.364646984985669, |
|
"objective/train/tokens_used": 217068000, |
|
"theoretical_loss": 4.364646984985669, |
|
"tokens_seen": 196608000 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.00046959523359182905, |
|
"loss": 3.1619, |
|
"theoretical_loss": 4.364646984985669, |
|
"tokens_seen": 196608000 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0004693588046150936, |
|
"loss": 3.1615, |
|
"theoretical_loss": 4.3610117242432125, |
|
"tokens_seen": 197918720 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.00046912237563835824, |
|
"loss": 3.1752, |
|
"theoretical_loss": 4.357407149082071, |
|
"tokens_seen": 199229440 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0004688859466616229, |
|
"loss": 3.1492, |
|
"theoretical_loss": 4.353832801144535, |
|
"tokens_seen": 200540160 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.00046864951768488744, |
|
"loss": 3.1317, |
|
"theoretical_loss": 4.350288231858301, |
|
"tokens_seen": 201850880 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0004684130887081521, |
|
"loss": 3.1784, |
|
"theoretical_loss": 4.346773002165636, |
|
"tokens_seen": 203161600 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0004681766597314167, |
|
"loss": 3.1369, |
|
"theoretical_loss": 4.343286682261728, |
|
"tokens_seen": 204472320 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.0004679402307546813, |
|
"loss": 3.1272, |
|
"theoretical_loss": 4.339828851341859, |
|
"tokens_seen": 205783040 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.00046770380177794593, |
|
"loss": 3.196, |
|
"theoretical_loss": 4.336399097357047, |
|
"tokens_seen": 207093760 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.00046746737280121053, |
|
"loss": 3.1771, |
|
"theoretical_loss": 4.332997016777814, |
|
"tokens_seen": 208404480 |
|
}, |
|
{ |
|
"epoch": 0.07, |
|
"learning_rate": 0.00046723094382447513, |
|
"loss": 3.1667, |
|
"theoretical_loss": 4.329622214365776, |
|
"tokens_seen": 209715200 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0004669945148477397, |
|
"loss": 3.1952, |
|
"theoretical_loss": 4.326274302952741, |
|
"tokens_seen": 211025920 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0004667580858710044, |
|
"loss": 3.1607, |
|
"theoretical_loss": 4.322952903227008, |
|
"tokens_seen": 212336640 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.000466521656894269, |
|
"loss": 3.1631, |
|
"theoretical_loss": 4.319657643526625, |
|
"tokens_seen": 213647360 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.00046628522791753357, |
|
"loss": 3.1079, |
|
"theoretical_loss": 4.316388159639304, |
|
"tokens_seen": 214958080 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0004660487989407982, |
|
"loss": 3.1118, |
|
"theoretical_loss": 4.313144094608757, |
|
"tokens_seen": 216268800 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.00046581236996406277, |
|
"loss": 3.1383, |
|
"theoretical_loss": 4.3099250985472, |
|
"tokens_seen": 217579520 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0004655759409873274, |
|
"loss": 3.1002, |
|
"theoretical_loss": 4.306730828453813, |
|
"tokens_seen": 218890240 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.00046533951201059207, |
|
"loss": 3.125, |
|
"theoretical_loss": 4.303560948038896, |
|
"tokens_seen": 220200960 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0004651030830338566, |
|
"loss": 3.0721, |
|
"theoretical_loss": 4.300415127553551, |
|
"tokens_seen": 221511680 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.00046486665405712127, |
|
"loss": 3.13, |
|
"theoretical_loss": 4.297293043624652, |
|
"tokens_seen": 222822400 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.00046463022508038586, |
|
"loss": 3.1433, |
|
"theoretical_loss": 4.294194379094925, |
|
"tokens_seen": 224133120 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.00046439379610365046, |
|
"loss": 3.1008, |
|
"theoretical_loss": 4.291118822867954, |
|
"tokens_seen": 225443840 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0004641573671269151, |
|
"loss": 3.206, |
|
"theoretical_loss": 4.288066069757916, |
|
"tokens_seen": 226754560 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0004639209381501797, |
|
"loss": 3.1231, |
|
"theoretical_loss": 4.285035820343889, |
|
"tokens_seen": 228065280 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0004636845091734443, |
|
"loss": 3.1608, |
|
"theoretical_loss": 4.28202778082858, |
|
"tokens_seen": 229376000 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0004634480801967089, |
|
"loss": 3.1397, |
|
"theoretical_loss": 4.27904166290127, |
|
"tokens_seen": 230686720 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.00046321165121997356, |
|
"loss": 3.1775, |
|
"theoretical_loss": 4.276077183604883, |
|
"tokens_seen": 231997440 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0004629752222432381, |
|
"loss": 3.1744, |
|
"theoretical_loss": 4.2731340652069925, |
|
"tokens_seen": 233308160 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.00046273879326650275, |
|
"loss": 3.1164, |
|
"theoretical_loss": 4.270212035074646, |
|
"tokens_seen": 234618880 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.0004625023642897674, |
|
"loss": 3.2551, |
|
"theoretical_loss": 4.267310825552862, |
|
"tokens_seen": 235929600 |
|
}, |
|
{ |
|
"epoch": 0.08, |
|
"learning_rate": 0.00046226593531303195, |
|
"loss": 3.1724, |
|
"theoretical_loss": 4.264430173846689, |
|
"tokens_seen": 237240320 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004620295063362966, |
|
"loss": 3.1622, |
|
"theoretical_loss": 4.261569821906684, |
|
"tokens_seen": 238551040 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004617930773595612, |
|
"loss": 3.1072, |
|
"theoretical_loss": 4.258729516317703, |
|
"tokens_seen": 239861760 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004615566483828258, |
|
"loss": 3.1874, |
|
"theoretical_loss": 4.255909008190896, |
|
"tokens_seen": 241172480 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.00046132021940609044, |
|
"loss": 3.0979, |
|
"theoretical_loss": 4.253108053058778, |
|
"tokens_seen": 242483200 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.00046108379042935504, |
|
"loss": 3.0572, |
|
"theoretical_loss": 4.250326410773302, |
|
"tokens_seen": 243793920 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.00046084736145261964, |
|
"loss": 3.0634, |
|
"theoretical_loss": 4.247563845406793, |
|
"tokens_seen": 245104640 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.00046061093247588424, |
|
"loss": 3.0988, |
|
"theoretical_loss": 4.2448201251556945, |
|
"tokens_seen": 246415360 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004603745034991489, |
|
"loss": 3.0756, |
|
"theoretical_loss": 4.242095022246987, |
|
"tokens_seen": 247726080 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004601380745224135, |
|
"loss": 3.1142, |
|
"theoretical_loss": 4.239388312847225, |
|
"tokens_seen": 249036800 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004599016455456781, |
|
"loss": 3.081, |
|
"theoretical_loss": 4.236699776974095, |
|
"tokens_seen": 250347520 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.00045966521656894274, |
|
"loss": 3.1122, |
|
"theoretical_loss": 4.234029198410394, |
|
"tokens_seen": 251658240 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004594287875922073, |
|
"loss": 3.1194, |
|
"theoretical_loss": 4.23137636462039, |
|
"tokens_seen": 252968960 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.00045919235861547193, |
|
"loss": 3.0914, |
|
"theoretical_loss": 4.228741066668439, |
|
"tokens_seen": 254279680 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004589559296387366, |
|
"loss": 3.1158, |
|
"theoretical_loss": 4.226123099139831, |
|
"tokens_seen": 255590400 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004587195006620011, |
|
"loss": 3.1033, |
|
"theoretical_loss": 4.223522260063753, |
|
"tokens_seen": 256901120 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004584830716852658, |
|
"loss": 3.0847, |
|
"theoretical_loss": 4.220938350838335, |
|
"tokens_seen": 258211840 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004582466427085304, |
|
"loss": 3.1236, |
|
"theoretical_loss": 4.218371176157692, |
|
"tokens_seen": 259522560 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.00045801021373179497, |
|
"loss": 3.0795, |
|
"theoretical_loss": 4.215820543940906, |
|
"tokens_seen": 260833280 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"objective/train/docs_used": 155958, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 3.078427791595459, |
|
"objective/train/theoretical_loss": 4.213286265262889, |
|
"objective/train/tokens_used": 282604000, |
|
"theoretical_loss": 4.213286265262889, |
|
"tokens_seen": 262144000 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004577737847550596, |
|
"loss": 3.0425, |
|
"theoretical_loss": 4.213286265262889, |
|
"tokens_seen": 262144000 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004575373557783242, |
|
"loss": 3.0468, |
|
"theoretical_loss": 4.210768154287058, |
|
"tokens_seen": 263454720 |
|
}, |
|
{ |
|
"epoch": 0.09, |
|
"learning_rate": 0.0004573009268015888, |
|
"loss": 3.0588, |
|
"theoretical_loss": 4.208266028199782, |
|
"tokens_seen": 264765440 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0004570644978248534, |
|
"loss": 3.0243, |
|
"theoretical_loss": 4.205779707146535, |
|
"tokens_seen": 266076160 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.00045682806884811807, |
|
"loss": 2.9936, |
|
"theoretical_loss": 4.203309014169696, |
|
"tokens_seen": 267386880 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0004565916398713826, |
|
"loss": 3.0404, |
|
"theoretical_loss": 4.200853775147967, |
|
"tokens_seen": 268697600 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.00045635521089464726, |
|
"loss": 3.0029, |
|
"theoretical_loss": 4.198413818737336, |
|
"tokens_seen": 270008320 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0004561187819179119, |
|
"loss": 3.025, |
|
"theoretical_loss": 4.195988976313554, |
|
"tokens_seen": 271319040 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.00045588235294117646, |
|
"loss": 2.9815, |
|
"theoretical_loss": 4.193579081916074, |
|
"tokens_seen": 272629760 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0004556459239644411, |
|
"loss": 3.0173, |
|
"theoretical_loss": 4.191183972193405, |
|
"tokens_seen": 273940480 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0004554094949877057, |
|
"loss": 3.0058, |
|
"theoretical_loss": 4.1888034863498484, |
|
"tokens_seen": 275251200 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0004551730660109703, |
|
"loss": 2.9504, |
|
"theoretical_loss": 4.186437466093564, |
|
"tokens_seen": 276561920 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.00045493663703423496, |
|
"loss": 2.9439, |
|
"theoretical_loss": 4.1840857555859365, |
|
"tokens_seen": 277872640 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0004547002080574995, |
|
"loss": 2.9548, |
|
"theoretical_loss": 4.181748201392195, |
|
"tokens_seen": 279183360 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.00045446377908076415, |
|
"loss": 2.9823, |
|
"theoretical_loss": 4.179424652433255, |
|
"tokens_seen": 280494080 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.00045422735010402875, |
|
"loss": 3.022, |
|
"theoretical_loss": 4.177114959938745, |
|
"tokens_seen": 281804800 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.00045399092112729335, |
|
"loss": 3.0318, |
|
"theoretical_loss": 4.174818977401182, |
|
"tokens_seen": 283115520 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.000453754492150558, |
|
"loss": 3.0489, |
|
"theoretical_loss": 4.172536560531271, |
|
"tokens_seen": 284426240 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0004535180631738226, |
|
"loss": 3.0029, |
|
"theoretical_loss": 4.170267567214273, |
|
"tokens_seen": 285736960 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0004532816341970872, |
|
"loss": 3.0065, |
|
"theoretical_loss": 4.168011857467442, |
|
"tokens_seen": 287047680 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0004530452052203518, |
|
"loss": 3.0111, |
|
"theoretical_loss": 4.1657692933984825, |
|
"tokens_seen": 288358400 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.00045280877624361644, |
|
"loss": 2.9825, |
|
"theoretical_loss": 4.163539739164992, |
|
"tokens_seen": 289669120 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.00045257234726688104, |
|
"loss": 3.0399, |
|
"theoretical_loss": 4.161323060934883, |
|
"tokens_seen": 290979840 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.00045233591829014564, |
|
"loss": 3.0392, |
|
"theoretical_loss": 4.159119126847729, |
|
"tokens_seen": 292290560 |
|
}, |
|
{ |
|
"epoch": 0.1, |
|
"learning_rate": 0.0004520994893134103, |
|
"loss": 3.0364, |
|
"theoretical_loss": 4.1569278069770395, |
|
"tokens_seen": 293601280 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.00045186306033667483, |
|
"loss": 3.005, |
|
"theoretical_loss": 4.154748973293404, |
|
"tokens_seen": 294912000 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0004516266313599395, |
|
"loss": 3.0766, |
|
"theoretical_loss": 4.152582499628515, |
|
"tokens_seen": 296222720 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0004513902023832041, |
|
"loss": 3.0355, |
|
"theoretical_loss": 4.150428261640019, |
|
"tokens_seen": 297533440 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0004511537734064687, |
|
"loss": 3.044, |
|
"theoretical_loss": 4.148286136777182, |
|
"tokens_seen": 298844160 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.00045091734442973333, |
|
"loss": 3.0471, |
|
"theoretical_loss": 4.1461560042473575, |
|
"tokens_seen": 300154880 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.00045068091545299793, |
|
"loss": 2.9748, |
|
"theoretical_loss": 4.144037744983214, |
|
"tokens_seen": 301465600 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0004504444864762625, |
|
"loss": 3.0411, |
|
"theoretical_loss": 4.141931241610721, |
|
"tokens_seen": 302776320 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0004502080574995271, |
|
"loss": 3.0282, |
|
"theoretical_loss": 4.139836378417861, |
|
"tokens_seen": 304087040 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0004499716285227918, |
|
"loss": 2.9842, |
|
"theoretical_loss": 4.1377530413240535, |
|
"tokens_seen": 305397760 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.00044973519954605637, |
|
"loss": 3.0131, |
|
"theoretical_loss": 4.135681117850265, |
|
"tokens_seen": 306708480 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.00044949877056932097, |
|
"loss": 3.0458, |
|
"theoretical_loss": 4.1336204970898, |
|
"tokens_seen": 308019200 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0004492623415925856, |
|
"loss": 2.9775, |
|
"theoretical_loss": 4.131571069679739, |
|
"tokens_seen": 309329920 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.00044902591261585016, |
|
"loss": 2.9762, |
|
"theoretical_loss": 4.129532727773022, |
|
"tokens_seen": 310640640 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0004487894836391148, |
|
"loss": 2.948, |
|
"theoretical_loss": 4.127505365011135, |
|
"tokens_seen": 311951360 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.00044855305466237947, |
|
"loss": 2.9521, |
|
"theoretical_loss": 4.125488876497428, |
|
"tokens_seen": 313262080 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.000448316625685644, |
|
"loss": 2.9967, |
|
"theoretical_loss": 4.123483158770997, |
|
"tokens_seen": 314572800 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.00044808019670890866, |
|
"loss": 2.9995, |
|
"theoretical_loss": 4.121488109781147, |
|
"tokens_seen": 315883520 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.00044784376773217326, |
|
"loss": 2.9331, |
|
"theoretical_loss": 4.11950362886242, |
|
"tokens_seen": 317194240 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.00044760733875543786, |
|
"loss": 2.9987, |
|
"theoretical_loss": 4.1175296167101525, |
|
"tokens_seen": 318504960 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0004473709097787025, |
|
"loss": 2.9851, |
|
"theoretical_loss": 4.115565975356576, |
|
"tokens_seen": 319815680 |
|
}, |
|
{ |
|
"epoch": 0.11, |
|
"learning_rate": 0.0004471344808019671, |
|
"loss": 2.9513, |
|
"theoretical_loss": 4.113612608147429, |
|
"tokens_seen": 321126400 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0004468980518252317, |
|
"loss": 2.9103, |
|
"theoretical_loss": 4.111669419719063, |
|
"tokens_seen": 322437120 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0004466616228484963, |
|
"loss": 2.9268, |
|
"theoretical_loss": 4.109736315976054, |
|
"tokens_seen": 323747840 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.00044642519387176095, |
|
"loss": 2.9573, |
|
"theoretical_loss": 4.107813204069279, |
|
"tokens_seen": 325058560 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0004461887648950255, |
|
"loss": 2.9633, |
|
"theoretical_loss": 4.105899992374464, |
|
"tokens_seen": 326369280 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.5416083918384781, |
|
"debugging/distinct-1-grams": 0.7980827691008653, |
|
"debugging/distinct-2-grams": 0.9666932256014988, |
|
"debugging/entropy-1-grams": 6.182765900392966, |
|
"debugging/entropy-2-grams": 7.172235247032516, |
|
"debugging/length": 490.5, |
|
"debugging/num_segments": 20, |
|
"epoch": 0.12, |
|
"objective/train/docs_used": 191985, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 3.0822439193725586, |
|
"objective/train/theoretical_loss": 4.103996590471185, |
|
"objective/train/tokens_used": 348140000, |
|
"theoretical_loss": 4.103996590471185, |
|
"tokens_seen": 327680000 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.00044595233591829015, |
|
"loss": 2.9667, |
|
"theoretical_loss": 4.103996590471185, |
|
"tokens_seen": 327680000 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0004457159069415548, |
|
"loss": 2.9485, |
|
"theoretical_loss": 4.102102909122308, |
|
"tokens_seen": 328990720 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.00044547947796481934, |
|
"loss": 2.946, |
|
"theoretical_loss": 4.1002188602538645, |
|
"tokens_seen": 330301440 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.000445243048988084, |
|
"loss": 3.0404, |
|
"theoretical_loss": 4.098344356935348, |
|
"tokens_seen": 331612160 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0004450066200113486, |
|
"loss": 3.0199, |
|
"theoretical_loss": 4.096479313360415, |
|
"tokens_seen": 332922880 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0004447701910346132, |
|
"loss": 2.9987, |
|
"theoretical_loss": 4.094623644827995, |
|
"tokens_seen": 334233600 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.00044453376205787784, |
|
"loss": 2.9626, |
|
"theoretical_loss": 4.092777267723779, |
|
"tokens_seen": 335544320 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.00044429733308114244, |
|
"loss": 2.9576, |
|
"theoretical_loss": 4.0909400995021, |
|
"tokens_seen": 336855040 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.00044406090410440704, |
|
"loss": 2.9351, |
|
"theoretical_loss": 4.08911205866818, |
|
"tokens_seen": 338165760 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.00044382447512767163, |
|
"loss": 3.0207, |
|
"theoretical_loss": 4.087293064760739, |
|
"tokens_seen": 339476480 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0004435880461509363, |
|
"loss": 2.9825, |
|
"theoretical_loss": 4.0854830383349565, |
|
"tokens_seen": 340787200 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0004433516171742009, |
|
"loss": 2.9368, |
|
"theoretical_loss": 4.083681900945785, |
|
"tokens_seen": 342097920 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0004431151881974655, |
|
"loss": 2.9302, |
|
"theoretical_loss": 4.081889575131589, |
|
"tokens_seen": 343408640 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.00044287875922073013, |
|
"loss": 2.9436, |
|
"theoretical_loss": 4.080105984398123, |
|
"tokens_seen": 344719360 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0004426423302439947, |
|
"loss": 2.9709, |
|
"theoretical_loss": 4.078331053202827, |
|
"tokens_seen": 346030080 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0004424059012672593, |
|
"loss": 2.8916, |
|
"theoretical_loss": 4.076564706939434, |
|
"tokens_seen": 347340800 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.000442169472290524, |
|
"loss": 2.9711, |
|
"theoretical_loss": 4.074806871922881, |
|
"tokens_seen": 348651520 |
|
}, |
|
{ |
|
"epoch": 0.12, |
|
"learning_rate": 0.0004419330433137885, |
|
"loss": 2.9344, |
|
"theoretical_loss": 4.0730574753745294, |
|
"tokens_seen": 349962240 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.0004416966143370532, |
|
"loss": 2.946, |
|
"theoretical_loss": 4.071316445407663, |
|
"tokens_seen": 351272960 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.00044146018536031777, |
|
"loss": 2.9271, |
|
"theoretical_loss": 4.06958371101328, |
|
"tokens_seen": 352583680 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.00044122375638358237, |
|
"loss": 2.9213, |
|
"theoretical_loss": 4.067859202046162, |
|
"tokens_seen": 353894400 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.000440987327406847, |
|
"loss": 2.9237, |
|
"theoretical_loss": 4.066142849211214, |
|
"tokens_seen": 355205120 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.0004407508984301116, |
|
"loss": 2.9702, |
|
"theoretical_loss": 4.064434584050067, |
|
"tokens_seen": 356515840 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.0004405144694533762, |
|
"loss": 2.9979, |
|
"theoretical_loss": 4.062734338927946, |
|
"tokens_seen": 357826560 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.0004402780404766408, |
|
"loss": 2.9666, |
|
"theoretical_loss": 4.061042047020785, |
|
"tokens_seen": 359137280 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.00044004161149990546, |
|
"loss": 2.9475, |
|
"theoretical_loss": 4.0593576423025945, |
|
"tokens_seen": 360448000 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.00043980518252317, |
|
"loss": 2.9903, |
|
"theoretical_loss": 4.057681059533071, |
|
"tokens_seen": 361758720 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.00043956875354643466, |
|
"loss": 3.0275, |
|
"theoretical_loss": 4.056012234245436, |
|
"tokens_seen": 363069440 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.0004393323245696993, |
|
"loss": 2.9532, |
|
"theoretical_loss": 4.05435110273452, |
|
"tokens_seen": 364380160 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.00043909589559296385, |
|
"loss": 3.0061, |
|
"theoretical_loss": 4.052697602045054, |
|
"tokens_seen": 365690880 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.0004388594666162285, |
|
"loss": 2.9492, |
|
"theoretical_loss": 4.0510516699602, |
|
"tokens_seen": 367001600 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.0004386230376394931, |
|
"loss": 2.9299, |
|
"theoretical_loss": 4.049413244990284, |
|
"tokens_seen": 368312320 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.0004383866086627577, |
|
"loss": 2.917, |
|
"theoretical_loss": 4.04778226636175, |
|
"tokens_seen": 369623040 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.00043815017968602235, |
|
"loss": 2.9734, |
|
"theoretical_loss": 4.046158674006312, |
|
"tokens_seen": 370933760 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.00043791375070928695, |
|
"loss": 2.9703, |
|
"theoretical_loss": 4.044542408550312, |
|
"tokens_seen": 372244480 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.00043767732173255155, |
|
"loss": 2.9203, |
|
"theoretical_loss": 4.042933411304277, |
|
"tokens_seen": 373555200 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.00043744089275581614, |
|
"loss": 2.9796, |
|
"theoretical_loss": 4.041331624252669, |
|
"tokens_seen": 374865920 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.0004372044637790808, |
|
"loss": 3.0013, |
|
"theoretical_loss": 4.039736990043815, |
|
"tokens_seen": 376176640 |
|
}, |
|
{ |
|
"epoch": 0.13, |
|
"learning_rate": 0.0004369680348023454, |
|
"loss": 2.9045, |
|
"theoretical_loss": 4.038149451980039, |
|
"tokens_seen": 377487360 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.00043673160582561, |
|
"loss": 2.8935, |
|
"theoretical_loss": 4.0365689540079535, |
|
"tokens_seen": 378798080 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.00043649517684887464, |
|
"loss": 2.9313, |
|
"theoretical_loss": 4.034995440708947, |
|
"tokens_seen": 380108800 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0004362587478721392, |
|
"loss": 2.9622, |
|
"theoretical_loss": 4.033428857289831, |
|
"tokens_seen": 381419520 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.00043602231889540384, |
|
"loss": 2.9375, |
|
"theoretical_loss": 4.031869149573668, |
|
"tokens_seen": 382730240 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0004357858899186685, |
|
"loss": 2.8995, |
|
"theoretical_loss": 4.030316263990749, |
|
"tokens_seen": 384040960 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.00043554946094193303, |
|
"loss": 2.9652, |
|
"theoretical_loss": 4.028770147569752, |
|
"tokens_seen": 385351680 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0004353130319651977, |
|
"loss": 2.8495, |
|
"theoretical_loss": 4.0272307479290435, |
|
"tokens_seen": 386662400 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0004350766029884623, |
|
"loss": 2.9227, |
|
"theoretical_loss": 4.025698013268144, |
|
"tokens_seen": 387973120 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0004348401740117269, |
|
"loss": 2.9515, |
|
"theoretical_loss": 4.024171892359345, |
|
"tokens_seen": 389283840 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.00043460374503499153, |
|
"loss": 2.9514, |
|
"theoretical_loss": 4.0226523345394645, |
|
"tokens_seen": 390594560 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.00043436731605825613, |
|
"loss": 2.9178, |
|
"theoretical_loss": 4.021139289701768, |
|
"tokens_seen": 391905280 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"objective/train/docs_used": 228129, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.7468600273132324, |
|
"objective/train/theoretical_loss": 4.019632708288012, |
|
"objective/train/tokens_used": 413676000, |
|
"theoretical_loss": 4.019632708288012, |
|
"tokens_seen": 393216000 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0004341308870815207, |
|
"loss": 2.9228, |
|
"theoretical_loss": 4.019632708288012, |
|
"tokens_seen": 393216000 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0004338944581047853, |
|
"loss": 2.9153, |
|
"theoretical_loss": 4.018132541280638, |
|
"tokens_seen": 394526720 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.00043365802912805, |
|
"loss": 2.9176, |
|
"theoretical_loss": 4.016638740195099, |
|
"tokens_seen": 395837440 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0004334216001513145, |
|
"loss": 2.9225, |
|
"theoretical_loss": 4.015151257072327, |
|
"tokens_seen": 397148160 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.00043318517117457917, |
|
"loss": 2.9261, |
|
"theoretical_loss": 4.013670044471318, |
|
"tokens_seen": 398458880 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0004329487421978438, |
|
"loss": 2.9174, |
|
"theoretical_loss": 4.012195055461859, |
|
"tokens_seen": 399769600 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.00043271231322110837, |
|
"loss": 2.9585, |
|
"theoretical_loss": 4.010726243617374, |
|
"tokens_seen": 401080320 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.000432475884244373, |
|
"loss": 2.9271, |
|
"theoretical_loss": 4.009263563007895, |
|
"tokens_seen": 402391040 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0004322394552676376, |
|
"loss": 2.9148, |
|
"theoretical_loss": 4.007806968193149, |
|
"tokens_seen": 403701760 |
|
}, |
|
{ |
|
"epoch": 0.14, |
|
"learning_rate": 0.0004320030262909022, |
|
"loss": 2.9607, |
|
"theoretical_loss": 4.006356414215775, |
|
"tokens_seen": 405012480 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.00043176659731416686, |
|
"loss": 2.9537, |
|
"theoretical_loss": 4.004911856594639, |
|
"tokens_seen": 406323200 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0004315301683374314, |
|
"loss": 2.9175, |
|
"theoretical_loss": 4.00347325131828, |
|
"tokens_seen": 407633920 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.00043129373936069606, |
|
"loss": 2.9827, |
|
"theoretical_loss": 4.002040554838458, |
|
"tokens_seen": 408944640 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.00043105731038396066, |
|
"loss": 2.926, |
|
"theoretical_loss": 4.000613724063812, |
|
"tokens_seen": 410255360 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.00043082088140722525, |
|
"loss": 2.884, |
|
"theoretical_loss": 3.9991927163536243, |
|
"tokens_seen": 411566080 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0004305844524304899, |
|
"loss": 2.9218, |
|
"theoretical_loss": 3.997777489511691, |
|
"tokens_seen": 412876800 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0004303480234537545, |
|
"loss": 2.9585, |
|
"theoretical_loss": 3.9963680017802945, |
|
"tokens_seen": 414187520 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0004301115944770191, |
|
"loss": 2.9845, |
|
"theoretical_loss": 3.9949642118342794, |
|
"tokens_seen": 415498240 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0004298751655002837, |
|
"loss": 3.0169, |
|
"theoretical_loss": 3.993566078775218, |
|
"tokens_seen": 416808960 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.00042963873652354835, |
|
"loss": 2.9619, |
|
"theoretical_loss": 3.9921735621256835, |
|
"tokens_seen": 418119680 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.00042940230754681295, |
|
"loss": 2.9916, |
|
"theoretical_loss": 3.990786621823611, |
|
"tokens_seen": 419430400 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.00042916587857007754, |
|
"loss": 3.0605, |
|
"theoretical_loss": 3.9894052182167536, |
|
"tokens_seen": 420741120 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0004289294495933422, |
|
"loss": 2.9475, |
|
"theoretical_loss": 3.988029312057227, |
|
"tokens_seen": 422051840 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.00042869302061660674, |
|
"loss": 2.9815, |
|
"theoretical_loss": 3.986658864496149, |
|
"tokens_seen": 423362560 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0004284565916398714, |
|
"loss": 2.9608, |
|
"theoretical_loss": 3.985293837078359, |
|
"tokens_seen": 424673280 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.000428220162663136, |
|
"loss": 2.943, |
|
"theoretical_loss": 3.9839341917372293, |
|
"tokens_seen": 425984000 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0004279837336864006, |
|
"loss": 3.0196, |
|
"theoretical_loss": 3.9825798907895598, |
|
"tokens_seen": 427294720 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.00042774730470966524, |
|
"loss": 2.9766, |
|
"theoretical_loss": 3.9812308969305508, |
|
"tokens_seen": 428605440 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.00042751087573292984, |
|
"loss": 3.0151, |
|
"theoretical_loss": 3.9798871732288617, |
|
"tokens_seen": 429916160 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.00042727444675619443, |
|
"loss": 2.9232, |
|
"theoretical_loss": 3.9785486831217467, |
|
"tokens_seen": 431226880 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.00042703801777945903, |
|
"loss": 2.9393, |
|
"theoretical_loss": 3.9772153904102705, |
|
"tokens_seen": 432537600 |
|
}, |
|
{ |
|
"epoch": 0.15, |
|
"learning_rate": 0.0004268015888027237, |
|
"loss": 2.9605, |
|
"theoretical_loss": 3.9758872592545957, |
|
"tokens_seen": 433848320 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.0004265651598259883, |
|
"loss": 2.9803, |
|
"theoretical_loss": 3.974564254169353, |
|
"tokens_seen": 435159040 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.0004263287308492529, |
|
"loss": 2.978, |
|
"theoretical_loss": 3.9732463400190756, |
|
"tokens_seen": 436469760 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.00042609230187251753, |
|
"loss": 2.9034, |
|
"theoretical_loss": 3.9719334820137124, |
|
"tokens_seen": 437780480 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.00042585587289578207, |
|
"loss": 2.9457, |
|
"theoretical_loss": 3.9706256457042133, |
|
"tokens_seen": 439091200 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.0004256194439190467, |
|
"loss": 2.9394, |
|
"theoretical_loss": 3.969322796978175, |
|
"tokens_seen": 440401920 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.0004253830149423114, |
|
"loss": 2.9263, |
|
"theoretical_loss": 3.968024902055565, |
|
"tokens_seen": 441712640 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.0004251465859655759, |
|
"loss": 2.927, |
|
"theoretical_loss": 3.966731927484508, |
|
"tokens_seen": 443023360 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.00042491015698884057, |
|
"loss": 2.9259, |
|
"theoretical_loss": 3.9654438401371372, |
|
"tokens_seen": 444334080 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.00042467372801210517, |
|
"loss": 2.9706, |
|
"theoretical_loss": 3.9641606072055113, |
|
"tokens_seen": 445644800 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.00042443729903536976, |
|
"loss": 2.9153, |
|
"theoretical_loss": 3.9628821961975955, |
|
"tokens_seen": 446955520 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.0004242008700586344, |
|
"loss": 2.9132, |
|
"theoretical_loss": 3.9616085749333028, |
|
"tokens_seen": 448266240 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.000423964441081899, |
|
"loss": 2.9324, |
|
"theoretical_loss": 3.9603397115406005, |
|
"tokens_seen": 449576960 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.0004237280121051636, |
|
"loss": 2.9444, |
|
"theoretical_loss": 3.9590755744516684, |
|
"tokens_seen": 450887680 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.0004234915831284282, |
|
"loss": 2.9114, |
|
"theoretical_loss": 3.9578161323991243, |
|
"tokens_seen": 452198400 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.00042325515415169286, |
|
"loss": 2.9374, |
|
"theoretical_loss": 3.9565613544123046, |
|
"tokens_seen": 453509120 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.0004230187251749574, |
|
"loss": 2.8609, |
|
"theoretical_loss": 3.955311209813596, |
|
"tokens_seen": 454819840 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.00042278229619822206, |
|
"loss": 2.8826, |
|
"theoretical_loss": 3.9540656682148327, |
|
"tokens_seen": 456130560 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.0004225458672214867, |
|
"loss": 2.9145, |
|
"theoretical_loss": 3.952824699513741, |
|
"tokens_seen": 457441280 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"objective/train/docs_used": 262792, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.892580032348633, |
|
"objective/train/theoretical_loss": 3.951588273890438, |
|
"objective/train/tokens_used": 479212000, |
|
"theoretical_loss": 3.951588273890438, |
|
"tokens_seen": 458752000 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.00042230943824475125, |
|
"loss": 2.9221, |
|
"theoretical_loss": 3.951588273890438, |
|
"tokens_seen": 458752000 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.0004220730092680159, |
|
"loss": 2.9077, |
|
"theoretical_loss": 3.950356361803988, |
|
"tokens_seen": 460062720 |
|
}, |
|
{ |
|
"epoch": 0.16, |
|
"learning_rate": 0.0004218365802912805, |
|
"loss": 2.9355, |
|
"theoretical_loss": 3.949128933989004, |
|
"tokens_seen": 461373440 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0004216001513145451, |
|
"loss": 2.9047, |
|
"theoretical_loss": 3.9479059614523058, |
|
"tokens_seen": 462684160 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.00042136372233780975, |
|
"loss": 2.8407, |
|
"theoretical_loss": 3.9466874154696234, |
|
"tokens_seen": 463994880 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.00042112729336107435, |
|
"loss": 2.8673, |
|
"theoretical_loss": 3.945473267582355, |
|
"tokens_seen": 465305600 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.00042089086438433894, |
|
"loss": 2.835, |
|
"theoretical_loss": 3.9442634895943662, |
|
"tokens_seen": 466616320 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.00042065443540760354, |
|
"loss": 2.8983, |
|
"theoretical_loss": 3.943058053568841, |
|
"tokens_seen": 467927040 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0004204180064308682, |
|
"loss": 2.875, |
|
"theoretical_loss": 3.94185693182518, |
|
"tokens_seen": 469237760 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0004201815774541328, |
|
"loss": 2.8836, |
|
"theoretical_loss": 3.9406600969359413, |
|
"tokens_seen": 470548480 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0004199451484773974, |
|
"loss": 2.8861, |
|
"theoretical_loss": 3.9394675217238277, |
|
"tokens_seen": 471859200 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.00041970871950066204, |
|
"loss": 2.8541, |
|
"theoretical_loss": 3.9382791792587195, |
|
"tokens_seen": 473169920 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0004194722905239266, |
|
"loss": 2.9094, |
|
"theoretical_loss": 3.9370950428547475, |
|
"tokens_seen": 474480640 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.00041923586154719123, |
|
"loss": 2.8365, |
|
"theoretical_loss": 3.9359150860674106, |
|
"tokens_seen": 475791360 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0004189994325704559, |
|
"loss": 2.8452, |
|
"theoretical_loss": 3.9347392826907353, |
|
"tokens_seen": 477102080 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.00041876300359372043, |
|
"loss": 2.8346, |
|
"theoretical_loss": 3.933567606754476, |
|
"tokens_seen": 478412800 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0004185265746169851, |
|
"loss": 2.8767, |
|
"theoretical_loss": 3.9324000325213557, |
|
"tokens_seen": 479723520 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0004182901456402497, |
|
"loss": 2.8949, |
|
"theoretical_loss": 3.931236534484346, |
|
"tokens_seen": 481034240 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0004180537166635143, |
|
"loss": 2.8451, |
|
"theoretical_loss": 3.930077087363987, |
|
"tokens_seen": 482344960 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.00041781728768677893, |
|
"loss": 2.8033, |
|
"theoretical_loss": 3.928921666105744, |
|
"tokens_seen": 483655680 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0004175808587100435, |
|
"loss": 2.8546, |
|
"theoretical_loss": 3.9277702458774058, |
|
"tokens_seen": 484966400 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0004173444297333081, |
|
"loss": 2.8101, |
|
"theoretical_loss": 3.926622802066514, |
|
"tokens_seen": 486277120 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.0004171080007565727, |
|
"loss": 2.8595, |
|
"theoretical_loss": 3.925479310277835, |
|
"tokens_seen": 487587840 |
|
}, |
|
{ |
|
"epoch": 0.17, |
|
"learning_rate": 0.00041687157177983737, |
|
"loss": 2.928, |
|
"theoretical_loss": 3.924339746330864, |
|
"tokens_seen": 488898560 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0004166351428031019, |
|
"loss": 2.8558, |
|
"theoretical_loss": 3.923204086257365, |
|
"tokens_seen": 490209280 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.00041639871382636657, |
|
"loss": 2.8963, |
|
"theoretical_loss": 3.9220723062989444, |
|
"tokens_seen": 491520000 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0004161622848496312, |
|
"loss": 2.9296, |
|
"theoretical_loss": 3.920944382904665, |
|
"tokens_seen": 492830720 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.00041592585587289576, |
|
"loss": 2.9416, |
|
"theoretical_loss": 3.919820292728682, |
|
"tokens_seen": 494141440 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0004156894268961604, |
|
"loss": 2.8974, |
|
"theoretical_loss": 3.918700012627923, |
|
"tokens_seen": 495452160 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.000415452997919425, |
|
"loss": 2.911, |
|
"theoretical_loss": 3.9175835196597957, |
|
"tokens_seen": 496762880 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0004152165689426896, |
|
"loss": 2.8654, |
|
"theoretical_loss": 3.9164707910799264, |
|
"tokens_seen": 498073600 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.00041498013996595426, |
|
"loss": 2.8717, |
|
"theoretical_loss": 3.9153618043399296, |
|
"tokens_seen": 499384320 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.00041474371098921886, |
|
"loss": 2.8459, |
|
"theoretical_loss": 3.9142565370852136, |
|
"tokens_seen": 500695040 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.00041450728201248346, |
|
"loss": 2.8187, |
|
"theoretical_loss": 3.9131549671528116, |
|
"tokens_seen": 502005760 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.00041427085303574805, |
|
"loss": 2.8665, |
|
"theoretical_loss": 3.912057072569243, |
|
"tokens_seen": 503316480 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0004140344240590127, |
|
"loss": 2.8217, |
|
"theoretical_loss": 3.910962831548404, |
|
"tokens_seen": 504627200 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0004137979950822773, |
|
"loss": 2.8274, |
|
"theoretical_loss": 3.909872222489491, |
|
"tokens_seen": 505937920 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0004135615661055419, |
|
"loss": 2.845, |
|
"theoretical_loss": 3.908785223974948, |
|
"tokens_seen": 507248640 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.00041332513712880655, |
|
"loss": 2.8363, |
|
"theoretical_loss": 3.9077018147684415, |
|
"tokens_seen": 508559360 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0004130887081520711, |
|
"loss": 2.878, |
|
"theoretical_loss": 3.906621973812867, |
|
"tokens_seen": 509870080 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.00041285227917533575, |
|
"loss": 2.8849, |
|
"theoretical_loss": 3.9055456802283812, |
|
"tokens_seen": 511180800 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0004126158501986004, |
|
"loss": 2.8734, |
|
"theoretical_loss": 3.904472913310456, |
|
"tokens_seen": 512491520 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.00041237942122186494, |
|
"loss": 2.7937, |
|
"theoretical_loss": 3.9034036525279645, |
|
"tokens_seen": 513802240 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0004121429922451296, |
|
"loss": 2.8423, |
|
"theoretical_loss": 3.902337877521294, |
|
"tokens_seen": 515112960 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0004119065632683942, |
|
"loss": 2.7962, |
|
"theoretical_loss": 3.9012755681004756, |
|
"tokens_seen": 516423680 |
|
}, |
|
{ |
|
"epoch": 0.18, |
|
"learning_rate": 0.0004116701342916588, |
|
"loss": 2.821, |
|
"theoretical_loss": 3.9002167042433475, |
|
"tokens_seen": 517734400 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.00041143370531492344, |
|
"loss": 2.8321, |
|
"theoretical_loss": 3.899161266093741, |
|
"tokens_seen": 519045120 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.00041119727633818804, |
|
"loss": 2.8604, |
|
"theoretical_loss": 3.8981092339596852, |
|
"tokens_seen": 520355840 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.00041096084736145263, |
|
"loss": 2.8132, |
|
"theoretical_loss": 3.8970605883116436, |
|
"tokens_seen": 521666560 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.00041072441838471723, |
|
"loss": 2.8032, |
|
"theoretical_loss": 3.896015309780767, |
|
"tokens_seen": 522977280 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"objective/train/docs_used": 298912, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.98516845703125, |
|
"objective/train/theoretical_loss": 3.8949733791571752, |
|
"objective/train/tokens_used": 544748000, |
|
"theoretical_loss": 3.8949733791571752, |
|
"tokens_seen": 524288000 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0004104879894079819, |
|
"loss": 2.7451, |
|
"theoretical_loss": 3.8949733791571752, |
|
"tokens_seen": 524288000 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0004102515604312464, |
|
"loss": 2.8263, |
|
"theoretical_loss": 3.8939347773882584, |
|
"tokens_seen": 525598720 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0004100151314545111, |
|
"loss": 2.8216, |
|
"theoretical_loss": 3.892899485576998, |
|
"tokens_seen": 526909440 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.00040977870247777573, |
|
"loss": 2.8367, |
|
"theoretical_loss": 3.8918674849803176, |
|
"tokens_seen": 528220160 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0004095422735010403, |
|
"loss": 2.826, |
|
"theoretical_loss": 3.890838757007447, |
|
"tokens_seen": 529530880 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0004093058445243049, |
|
"loss": 2.7991, |
|
"theoretical_loss": 3.889813283218313, |
|
"tokens_seen": 530841600 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0004090694155475695, |
|
"loss": 2.8456, |
|
"theoretical_loss": 3.888791045321949, |
|
"tokens_seen": 532152320 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0004088329865708341, |
|
"loss": 2.8602, |
|
"theoretical_loss": 3.887772025174925, |
|
"tokens_seen": 533463040 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.00040859655759409877, |
|
"loss": 2.8292, |
|
"theoretical_loss": 3.8867562047797986, |
|
"tokens_seen": 534773760 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.00040836012861736337, |
|
"loss": 2.8996, |
|
"theoretical_loss": 3.885743566283587, |
|
"tokens_seen": 536084480 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.00040812369964062797, |
|
"loss": 2.8527, |
|
"theoretical_loss": 3.8847340919762576, |
|
"tokens_seen": 537395200 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.00040788727066389256, |
|
"loss": 2.8743, |
|
"theoretical_loss": 3.883727764289236, |
|
"tokens_seen": 538705920 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.00040765084168715716, |
|
"loss": 2.8987, |
|
"theoretical_loss": 3.8827245657939375, |
|
"tokens_seen": 540016640 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0004074144127104218, |
|
"loss": 2.8296, |
|
"theoretical_loss": 3.8817244792003143, |
|
"tokens_seen": 541327360 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0004071779837336864, |
|
"loss": 2.8508, |
|
"theoretical_loss": 3.8807274873554216, |
|
"tokens_seen": 542638080 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.000406941554756951, |
|
"loss": 2.8473, |
|
"theoretical_loss": 3.8797335732420035, |
|
"tokens_seen": 543948800 |
|
}, |
|
{ |
|
"epoch": 0.19, |
|
"learning_rate": 0.0004067051257802156, |
|
"loss": 2.82, |
|
"theoretical_loss": 3.878742719977094, |
|
"tokens_seen": 545259520 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.00040646869680348026, |
|
"loss": 2.839, |
|
"theoretical_loss": 3.877754910810639, |
|
"tokens_seen": 546570240 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.00040623226782674485, |
|
"loss": 2.8439, |
|
"theoretical_loss": 3.876770129124133, |
|
"tokens_seen": 547880960 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.00040599583885000945, |
|
"loss": 2.8545, |
|
"theoretical_loss": 3.8757883584292774, |
|
"tokens_seen": 549191680 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0004057594098732741, |
|
"loss": 2.8427, |
|
"theoretical_loss": 3.8748095823666473, |
|
"tokens_seen": 550502400 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.00040552298089653865, |
|
"loss": 2.8971, |
|
"theoretical_loss": 3.8738337847043836, |
|
"tokens_seen": 551813120 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0004052865519198033, |
|
"loss": 2.8655, |
|
"theoretical_loss": 3.8728609493368986, |
|
"tokens_seen": 553123840 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0004050501229430679, |
|
"loss": 2.8534, |
|
"theoretical_loss": 3.8718910602835943, |
|
"tokens_seen": 554434560 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0004048136939663325, |
|
"loss": 2.8304, |
|
"theoretical_loss": 3.870924101687603, |
|
"tokens_seen": 555745280 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.00040457726498959715, |
|
"loss": 2.8699, |
|
"theoretical_loss": 3.869960057814537, |
|
"tokens_seen": 557056000 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.00040434083601286174, |
|
"loss": 2.8513, |
|
"theoretical_loss": 3.868998913051261, |
|
"tokens_seen": 558366720 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.00040410440703612634, |
|
"loss": 2.882, |
|
"theoretical_loss": 3.868040651904671, |
|
"tokens_seen": 559677440 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.00040386797805939094, |
|
"loss": 2.8468, |
|
"theoretical_loss": 3.867085259000496, |
|
"tokens_seen": 560988160 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0004036315490826556, |
|
"loss": 2.8575, |
|
"theoretical_loss": 3.8661327190821124, |
|
"tokens_seen": 562298880 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0004033951201059202, |
|
"loss": 2.8521, |
|
"theoretical_loss": 3.8651830170093673, |
|
"tokens_seen": 563609600 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0004031586911291848, |
|
"loss": 2.8827, |
|
"theoretical_loss": 3.864236137757427, |
|
"tokens_seen": 564920320 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.00040292226215244944, |
|
"loss": 2.8591, |
|
"theoretical_loss": 3.863292066415628, |
|
"tokens_seen": 566231040 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.000402685833175714, |
|
"loss": 2.9049, |
|
"theoretical_loss": 3.862350788186353, |
|
"tokens_seen": 567541760 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.00040244940419897863, |
|
"loss": 2.8928, |
|
"theoretical_loss": 3.8614122883839093, |
|
"tokens_seen": 568852480 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0004022129752222433, |
|
"loss": 2.9413, |
|
"theoretical_loss": 3.8604765524334312, |
|
"tokens_seen": 570163200 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0004019765462455078, |
|
"loss": 2.9571, |
|
"theoretical_loss": 3.8595435658697888, |
|
"tokens_seen": 571473920 |
|
}, |
|
{ |
|
"epoch": 0.2, |
|
"learning_rate": 0.0004017401172687725, |
|
"loss": 2.9104, |
|
"theoretical_loss": 3.858613314336514, |
|
"tokens_seen": 572784640 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0004015036882920371, |
|
"loss": 2.9053, |
|
"theoretical_loss": 3.8576857835847367, |
|
"tokens_seen": 574095360 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.00040126725931530167, |
|
"loss": 2.9107, |
|
"theoretical_loss": 3.8567609594721333, |
|
"tokens_seen": 575406080 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0004010308303385663, |
|
"loss": 2.8914, |
|
"theoretical_loss": 3.855838827961894, |
|
"tokens_seen": 576716800 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0004007944013618309, |
|
"loss": 2.8867, |
|
"theoretical_loss": 3.8549193751216935, |
|
"tokens_seen": 578027520 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0004005579723850955, |
|
"loss": 2.955, |
|
"theoretical_loss": 3.854002587122681, |
|
"tokens_seen": 579338240 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0004003215434083601, |
|
"loss": 2.8915, |
|
"theoretical_loss": 3.853088450238478, |
|
"tokens_seen": 580648960 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.00040008511443162477, |
|
"loss": 2.836, |
|
"theoretical_loss": 3.8521769508441905, |
|
"tokens_seen": 581959680 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0003998486854548893, |
|
"loss": 2.8512, |
|
"theoretical_loss": 3.8512680754154323, |
|
"tokens_seen": 583270400 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.00039961225647815396, |
|
"loss": 2.8983, |
|
"theoretical_loss": 3.850361810527361, |
|
"tokens_seen": 584581120 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0003993758275014186, |
|
"loss": 2.8751, |
|
"theoretical_loss": 3.8494581428537202, |
|
"tokens_seen": 585891840 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.00039913939852468316, |
|
"loss": 2.8825, |
|
"theoretical_loss": 3.8485570591659006, |
|
"tokens_seen": 587202560 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0003989029695479478, |
|
"loss": 2.8639, |
|
"theoretical_loss": 3.847658546332007, |
|
"tokens_seen": 588513280 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"objective/train/docs_used": 333293, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 3.010171413421631, |
|
"objective/train/theoretical_loss": 3.8467625913159385, |
|
"objective/train/tokens_used": 610284000, |
|
"theoretical_loss": 3.8467625913159385, |
|
"tokens_seen": 589824000 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0003986665405712124, |
|
"loss": 2.8745, |
|
"theoretical_loss": 3.8467625913159385, |
|
"tokens_seen": 589824000 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.000398430111594477, |
|
"loss": 2.9131, |
|
"theoretical_loss": 3.845869181176477, |
|
"tokens_seen": 591134720 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.00039819368261774166, |
|
"loss": 2.8483, |
|
"theoretical_loss": 3.844978303066389, |
|
"tokens_seen": 592445440 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.00039795725364100625, |
|
"loss": 2.8512, |
|
"theoretical_loss": 3.8440899442315395, |
|
"tokens_seen": 593756160 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.00039772082466427085, |
|
"loss": 2.8047, |
|
"theoretical_loss": 3.843204092010007, |
|
"tokens_seen": 595066880 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.00039748439568753545, |
|
"loss": 2.7938, |
|
"theoretical_loss": 3.8423207338312233, |
|
"tokens_seen": 596377600 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0003972479667108001, |
|
"loss": 2.8365, |
|
"theoretical_loss": 3.841439857215109, |
|
"tokens_seen": 597688320 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0003970115377340647, |
|
"loss": 2.7964, |
|
"theoretical_loss": 3.84056144977123, |
|
"tokens_seen": 598999040 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.0003967751087573293, |
|
"loss": 2.8111, |
|
"theoretical_loss": 3.839685499197956, |
|
"tokens_seen": 600309760 |
|
}, |
|
{ |
|
"epoch": 0.21, |
|
"learning_rate": 0.00039653867978059395, |
|
"loss": 2.7922, |
|
"theoretical_loss": 3.8388119932816327, |
|
"tokens_seen": 601620480 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0003963022508038585, |
|
"loss": 2.8055, |
|
"theoretical_loss": 3.8379409198957637, |
|
"tokens_seen": 602931200 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.00039606582182712314, |
|
"loss": 2.813, |
|
"theoretical_loss": 3.837072267000199, |
|
"tokens_seen": 604241920 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0003958293928503878, |
|
"loss": 2.8134, |
|
"theoretical_loss": 3.8362060226403347, |
|
"tokens_seen": 605552640 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.00039559296387365234, |
|
"loss": 2.8747, |
|
"theoretical_loss": 3.835342174946324, |
|
"tokens_seen": 606863360 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.000395356534896917, |
|
"loss": 2.8115, |
|
"theoretical_loss": 3.834480712132291, |
|
"tokens_seen": 608174080 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0003951201059201816, |
|
"loss": 2.8253, |
|
"theoretical_loss": 3.833621622495561, |
|
"tokens_seen": 609484800 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0003948836769434462, |
|
"loss": 2.8841, |
|
"theoretical_loss": 3.8327648944158907, |
|
"tokens_seen": 610795520 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.00039464724796671084, |
|
"loss": 2.8109, |
|
"theoretical_loss": 3.8319105163547187, |
|
"tokens_seen": 612106240 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.00039441081898997543, |
|
"loss": 2.8429, |
|
"theoretical_loss": 3.8310584768544125, |
|
"tokens_seen": 613416960 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.00039417439001324003, |
|
"loss": 2.8154, |
|
"theoretical_loss": 3.8302087645375327, |
|
"tokens_seen": 614727680 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.00039393796103650463, |
|
"loss": 2.8656, |
|
"theoretical_loss": 3.829361368106099, |
|
"tokens_seen": 616038400 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0003937015320597693, |
|
"loss": 2.7981, |
|
"theoretical_loss": 3.8285162763408707, |
|
"tokens_seen": 617349120 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0003934651030830338, |
|
"loss": 2.7823, |
|
"theoretical_loss": 3.827673478100631, |
|
"tokens_seen": 618659840 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0003932286741062985, |
|
"loss": 2.7954, |
|
"theoretical_loss": 3.8268329623214803, |
|
"tokens_seen": 619970560 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0003929922451295631, |
|
"loss": 2.8648, |
|
"theoretical_loss": 3.8259947180161373, |
|
"tokens_seen": 621281280 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.00039275581615282767, |
|
"loss": 2.8326, |
|
"theoretical_loss": 3.8251587342732485, |
|
"tokens_seen": 622592000 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0003925193871760923, |
|
"loss": 2.8281, |
|
"theoretical_loss": 3.824325000256705, |
|
"tokens_seen": 623902720 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0003922829581993569, |
|
"loss": 2.7531, |
|
"theoretical_loss": 3.8234935052049663, |
|
"tokens_seen": 625213440 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.0003920465292226215, |
|
"loss": 2.7447, |
|
"theoretical_loss": 3.8226642384303933, |
|
"tokens_seen": 626524160 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.00039181010024588617, |
|
"loss": 2.8173, |
|
"theoretical_loss": 3.821837189318586, |
|
"tokens_seen": 627834880 |
|
}, |
|
{ |
|
"epoch": 0.22, |
|
"learning_rate": 0.00039157367126915077, |
|
"loss": 2.7845, |
|
"theoretical_loss": 3.82101234732773, |
|
"tokens_seen": 629145600 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.00039133724229241536, |
|
"loss": 2.8092, |
|
"theoretical_loss": 3.820189701987953, |
|
"tokens_seen": 630456320 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.00039110081331567996, |
|
"loss": 2.8167, |
|
"theoretical_loss": 3.819369242900682, |
|
"tokens_seen": 631767040 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0003908643843389446, |
|
"loss": 2.8124, |
|
"theoretical_loss": 3.8185509597380105, |
|
"tokens_seen": 633077760 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0003906279553622092, |
|
"loss": 2.8588, |
|
"theoretical_loss": 3.8177348422420785, |
|
"tokens_seen": 634388480 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0003903915263854738, |
|
"loss": 2.8766, |
|
"theoretical_loss": 3.8169208802244476, |
|
"tokens_seen": 635699200 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.00039015509740873846, |
|
"loss": 2.864, |
|
"theoretical_loss": 3.8161090635654937, |
|
"tokens_seen": 637009920 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.000389918668432003, |
|
"loss": 2.8529, |
|
"theoretical_loss": 3.815299382213799, |
|
"tokens_seen": 638320640 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.00038968223945526765, |
|
"loss": 2.8858, |
|
"theoretical_loss": 3.8144918261855536, |
|
"tokens_seen": 639631360 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0003894458104785323, |
|
"loss": 2.8387, |
|
"theoretical_loss": 3.813686385563966, |
|
"tokens_seen": 640942080 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.00038920938150179685, |
|
"loss": 2.8272, |
|
"theoretical_loss": 3.812883050498675, |
|
"tokens_seen": 642252800 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0003889729525250615, |
|
"loss": 2.9091, |
|
"theoretical_loss": 3.812081811205168, |
|
"tokens_seen": 643563520 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0003887365235483261, |
|
"loss": 2.8777, |
|
"theoretical_loss": 3.811282657964214, |
|
"tokens_seen": 644874240 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0003885000945715907, |
|
"loss": 2.8015, |
|
"theoretical_loss": 3.8104855811212905, |
|
"tokens_seen": 646184960 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.00038826366559485535, |
|
"loss": 2.8613, |
|
"theoretical_loss": 3.8096905710860254, |
|
"tokens_seen": 647495680 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.00038802723661811994, |
|
"loss": 2.8904, |
|
"theoretical_loss": 3.808897618331641, |
|
"tokens_seen": 648806400 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.00038779080764138454, |
|
"loss": 2.8777, |
|
"theoretical_loss": 3.8081067133944053, |
|
"tokens_seen": 650117120 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.00038755437866464914, |
|
"loss": 2.8393, |
|
"theoretical_loss": 3.8073178468730884, |
|
"tokens_seen": 651427840 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0003873179496879138, |
|
"loss": 2.8258, |
|
"theoretical_loss": 3.806531009428425, |
|
"tokens_seen": 652738560 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.00038708152071117833, |
|
"loss": 2.8412, |
|
"theoretical_loss": 3.805746191782582, |
|
"tokens_seen": 654049280 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.4831024939875183, |
|
"debugging/distinct-1-grams": 0.8154550796195709, |
|
"debugging/distinct-2-grams": 0.9726187101634798, |
|
"debugging/entropy-1-grams": 5.88298844007536, |
|
"debugging/entropy-2-grams": 6.629837829827698, |
|
"debugging/length": 534.5, |
|
"debugging/num_segments": 10, |
|
"epoch": 0.23, |
|
"objective/train/docs_used": 369378, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 3.009995222091675, |
|
"objective/train/theoretical_loss": 3.8049633847186337, |
|
"objective/train/tokens_used": 675820000, |
|
"theoretical_loss": 3.8049633847186337, |
|
"tokens_seen": 655360000 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.000386845091734443, |
|
"loss": 2.8184, |
|
"theoretical_loss": 3.8049633847186337, |
|
"tokens_seen": 655360000 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.00038660866275770764, |
|
"loss": 2.7351, |
|
"theoretical_loss": 3.80418257908004, |
|
"tokens_seen": 656670720 |
|
}, |
|
{ |
|
"epoch": 0.23, |
|
"learning_rate": 0.0003863722337809722, |
|
"loss": 2.8029, |
|
"theoretical_loss": 3.803403765770131, |
|
"tokens_seen": 657981440 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.00038613580480423683, |
|
"loss": 2.7912, |
|
"theoretical_loss": 3.802626935751598, |
|
"tokens_seen": 659292160 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.00038589937582750143, |
|
"loss": 2.7949, |
|
"theoretical_loss": 3.801852080045988, |
|
"tokens_seen": 660602880 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.00038566294685076603, |
|
"loss": 2.7934, |
|
"theoretical_loss": 3.801079189733206, |
|
"tokens_seen": 661913600 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0003854265178740307, |
|
"loss": 2.8019, |
|
"theoretical_loss": 3.8003082559510197, |
|
"tokens_seen": 663224320 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0003851900888972953, |
|
"loss": 2.8398, |
|
"theoretical_loss": 3.799539269894571, |
|
"tokens_seen": 664535040 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0003849536599205599, |
|
"loss": 2.8581, |
|
"theoretical_loss": 3.798772222815893, |
|
"tokens_seen": 665845760 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.00038471723094382447, |
|
"loss": 2.7848, |
|
"theoretical_loss": 3.79800710602343, |
|
"tokens_seen": 667156480 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0003844926234159259, |
|
"loss": 2.8385, |
|
"theoretical_loss": 3.7972439108815657, |
|
"tokens_seen": 668467200 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.00038425619443919047, |
|
"loss": 2.8834, |
|
"theoretical_loss": 3.7964826288101508, |
|
"tokens_seen": 669777920 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0003840315869112919, |
|
"loss": 2.8737, |
|
"theoretical_loss": 3.795723251284044, |
|
"tokens_seen": 671088640 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.00038379515793455647, |
|
"loss": 2.9181, |
|
"theoretical_loss": 3.7949657698326487, |
|
"tokens_seen": 672399360 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.00038355872895782107, |
|
"loss": 2.9011, |
|
"theoretical_loss": 3.79421017603946, |
|
"tokens_seen": 673710080 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0003833222999810857, |
|
"loss": 2.852, |
|
"theoretical_loss": 3.7934564615416146, |
|
"tokens_seen": 675020800 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0003830858710043503, |
|
"loss": 2.8606, |
|
"theoretical_loss": 3.7927046180294455, |
|
"tokens_seen": 676331520 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0003828494420276149, |
|
"loss": 2.8736, |
|
"theoretical_loss": 3.7919546372460418, |
|
"tokens_seen": 677642240 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.0003826130130508795, |
|
"loss": 2.8608, |
|
"theoretical_loss": 3.7912065109868127, |
|
"tokens_seen": 678952960 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.00038237658407414417, |
|
"loss": 2.8368, |
|
"theoretical_loss": 3.790460231099053, |
|
"tokens_seen": 680263680 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.00038214015509740876, |
|
"loss": 2.892, |
|
"theoretical_loss": 3.789715789481521, |
|
"tokens_seen": 681574400 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.00038190372612067336, |
|
"loss": 2.8421, |
|
"theoretical_loss": 3.788973178084011, |
|
"tokens_seen": 682885120 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.00038166729714393796, |
|
"loss": 2.8264, |
|
"theoretical_loss": 3.788232388906935, |
|
"tokens_seen": 684195840 |
|
}, |
|
{ |
|
"epoch": 0.24, |
|
"learning_rate": 0.00038143086816720256, |
|
"loss": 2.8892, |
|
"theoretical_loss": 3.7874934140009096, |
|
"tokens_seen": 685506560 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0003811944391904672, |
|
"loss": 2.8702, |
|
"theoretical_loss": 3.786756245466346, |
|
"tokens_seen": 686817280 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.00038095801021373175, |
|
"loss": 2.8296, |
|
"theoretical_loss": 3.7860208754530422, |
|
"tokens_seen": 688128000 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0003807215812369964, |
|
"loss": 2.8989, |
|
"theoretical_loss": 3.785287296159782, |
|
"tokens_seen": 689438720 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.00038048515226026105, |
|
"loss": 2.8029, |
|
"theoretical_loss": 3.784555499833936, |
|
"tokens_seen": 690749440 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0003802487232835256, |
|
"loss": 2.8094, |
|
"theoretical_loss": 3.7838254787710683, |
|
"tokens_seen": 692060160 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.00038001229430679025, |
|
"loss": 2.8296, |
|
"theoretical_loss": 3.7830972253145463, |
|
"tokens_seen": 693370880 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.00037977586533005485, |
|
"loss": 2.8966, |
|
"theoretical_loss": 3.7823707318551536, |
|
"tokens_seen": 694681600 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.00037953943635331944, |
|
"loss": 2.7936, |
|
"theoretical_loss": 3.7816459908307074, |
|
"tokens_seen": 695992320 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0003793030073765841, |
|
"loss": 2.8499, |
|
"theoretical_loss": 3.7809229947256826, |
|
"tokens_seen": 697303040 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0003790665783998487, |
|
"loss": 2.8423, |
|
"theoretical_loss": 3.7802017360708327, |
|
"tokens_seen": 698613760 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0003788301494231133, |
|
"loss": 2.7822, |
|
"theoretical_loss": 3.779482207442821, |
|
"tokens_seen": 699924480 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0003785937204463779, |
|
"loss": 2.8023, |
|
"theoretical_loss": 3.778764401463854, |
|
"tokens_seen": 701235200 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.00037835729146964254, |
|
"loss": 2.8137, |
|
"theoretical_loss": 3.778048310801316, |
|
"tokens_seen": 702545920 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.00037812086249290714, |
|
"loss": 2.7984, |
|
"theoretical_loss": 3.7773339281674074, |
|
"tokens_seen": 703856640 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.00037788443351617173, |
|
"loss": 2.817, |
|
"theoretical_loss": 3.7766212463187916, |
|
"tokens_seen": 705167360 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0003776480045394364, |
|
"loss": 2.8796, |
|
"theoretical_loss": 3.7759102580562383, |
|
"tokens_seen": 706478080 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.00037741157556270093, |
|
"loss": 2.7898, |
|
"theoretical_loss": 3.7752009562242757, |
|
"tokens_seen": 707788800 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0003771751465859656, |
|
"loss": 2.7628, |
|
"theoretical_loss": 3.7744933337108444, |
|
"tokens_seen": 709099520 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.00037693871760923023, |
|
"loss": 2.8321, |
|
"theoretical_loss": 3.773787383446951, |
|
"tokens_seen": 710410240 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.0003767022886324948, |
|
"loss": 2.8259, |
|
"theoretical_loss": 3.773083098406333, |
|
"tokens_seen": 711720960 |
|
}, |
|
{ |
|
"epoch": 0.25, |
|
"learning_rate": 0.00037646585965575943, |
|
"loss": 2.8023, |
|
"theoretical_loss": 3.7723804716051195, |
|
"tokens_seen": 713031680 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.000376229430679024, |
|
"loss": 2.8084, |
|
"theoretical_loss": 3.7716794961014983, |
|
"tokens_seen": 714342400 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0003759930017022886, |
|
"loss": 2.7766, |
|
"theoretical_loss": 3.770980164995387, |
|
"tokens_seen": 715653120 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0003757565727255533, |
|
"loss": 2.7966, |
|
"theoretical_loss": 3.7702824714281062, |
|
"tokens_seen": 716963840 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.00037552014374881787, |
|
"loss": 2.7776, |
|
"theoretical_loss": 3.7695864085820547, |
|
"tokens_seen": 718274560 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.00037528371477208247, |
|
"loss": 2.7785, |
|
"theoretical_loss": 3.7688919696803898, |
|
"tokens_seen": 719585280 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"objective/train/docs_used": 405203, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.7325472831726074, |
|
"objective/train/theoretical_loss": 3.76819914798671, |
|
"objective/train/tokens_used": 741356000, |
|
"theoretical_loss": 3.76819914798671, |
|
"tokens_seen": 720896000 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.00037504728579534707, |
|
"loss": 2.7356, |
|
"theoretical_loss": 3.76819914798671, |
|
"tokens_seen": 720896000 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0003748108568186117, |
|
"loss": 2.8209, |
|
"theoretical_loss": 3.7675079368047415, |
|
"tokens_seen": 722206720 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.00037457442784187626, |
|
"loss": 2.7781, |
|
"theoretical_loss": 3.7668183294780246, |
|
"tokens_seen": 723517440 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0003743379988651409, |
|
"loss": 2.7413, |
|
"theoretical_loss": 3.7661303193896076, |
|
"tokens_seen": 724828160 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.00037410156988840556, |
|
"loss": 2.7527, |
|
"theoretical_loss": 3.76544389996174, |
|
"tokens_seen": 726138880 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0003738651409116701, |
|
"loss": 2.7117, |
|
"theoretical_loss": 3.7647590646555713, |
|
"tokens_seen": 727449600 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.00037362871193493476, |
|
"loss": 2.714, |
|
"theoretical_loss": 3.7640758069708493, |
|
"tokens_seen": 728760320 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.00037339228295819936, |
|
"loss": 2.7695, |
|
"theoretical_loss": 3.763394120445626, |
|
"tokens_seen": 730071040 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.00037315585398146395, |
|
"loss": 2.784, |
|
"theoretical_loss": 3.7627139986559595, |
|
"tokens_seen": 731381760 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0003729194250047286, |
|
"loss": 2.754, |
|
"theoretical_loss": 3.7620354352156293, |
|
"tokens_seen": 732692480 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0003726829960279932, |
|
"loss": 2.7113, |
|
"theoretical_loss": 3.7613584237758415, |
|
"tokens_seen": 734003200 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0003724465670512578, |
|
"loss": 2.8187, |
|
"theoretical_loss": 3.7606829580249457, |
|
"tokens_seen": 735313920 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0003722101380745224, |
|
"loss": 2.8081, |
|
"theoretical_loss": 3.760009031688152, |
|
"tokens_seen": 736624640 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.00037197370909778705, |
|
"loss": 2.814, |
|
"theoretical_loss": 3.7593366385272513, |
|
"tokens_seen": 737935360 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.00037173728012105165, |
|
"loss": 2.7847, |
|
"theoretical_loss": 3.758665772340336, |
|
"tokens_seen": 739246080 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.00037150085114431625, |
|
"loss": 2.8126, |
|
"theoretical_loss": 3.7579964269615247, |
|
"tokens_seen": 740556800 |
|
}, |
|
{ |
|
"epoch": 0.26, |
|
"learning_rate": 0.0003712644221675809, |
|
"loss": 2.8318, |
|
"theoretical_loss": 3.7573285962606917, |
|
"tokens_seen": 741867520 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.00037102799319084544, |
|
"loss": 2.7802, |
|
"theoretical_loss": 3.7566622741431956, |
|
"tokens_seen": 743178240 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0003707915642141101, |
|
"loss": 2.8111, |
|
"theoretical_loss": 3.75599745454961, |
|
"tokens_seen": 744488960 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.00037055513523737474, |
|
"loss": 2.8358, |
|
"theoretical_loss": 3.7553341314554625, |
|
"tokens_seen": 745799680 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0003703187062606393, |
|
"loss": 2.8251, |
|
"theoretical_loss": 3.754672298870968, |
|
"tokens_seen": 747110400 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.00037008227728390394, |
|
"loss": 2.8304, |
|
"theoretical_loss": 3.754011950840771, |
|
"tokens_seen": 748421120 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.00036984584830716854, |
|
"loss": 2.8102, |
|
"theoretical_loss": 3.753353081443688, |
|
"tokens_seen": 749731840 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.00036960941933043313, |
|
"loss": 2.7869, |
|
"theoretical_loss": 3.752695684792449, |
|
"tokens_seen": 751042560 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.00036937299035369773, |
|
"loss": 2.7899, |
|
"theoretical_loss": 3.752039755033449, |
|
"tokens_seen": 752353280 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0003691365613769624, |
|
"loss": 2.7801, |
|
"theoretical_loss": 3.751385286346493, |
|
"tokens_seen": 753664000 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.000368900132400227, |
|
"loss": 2.7465, |
|
"theoretical_loss": 3.7507322729445525, |
|
"tokens_seen": 754974720 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0003686637034234916, |
|
"loss": 2.7581, |
|
"theoretical_loss": 3.7500807090735115, |
|
"tokens_seen": 756285440 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.00036842727444675623, |
|
"loss": 2.7208, |
|
"theoretical_loss": 3.749430589011933, |
|
"tokens_seen": 757596160 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.00036819084547002077, |
|
"loss": 2.7744, |
|
"theoretical_loss": 3.748781907070807, |
|
"tokens_seen": 758906880 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0003679544164932854, |
|
"loss": 2.8423, |
|
"theoretical_loss": 3.7481346575933197, |
|
"tokens_seen": 760217600 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0003677179875165501, |
|
"loss": 2.8088, |
|
"theoretical_loss": 3.747488834954609, |
|
"tokens_seen": 761528320 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0003674815585398146, |
|
"loss": 2.8077, |
|
"theoretical_loss": 3.7468444335615363, |
|
"tokens_seen": 762839040 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.00036724512956307927, |
|
"loss": 2.7254, |
|
"theoretical_loss": 3.746201447852446, |
|
"tokens_seen": 764149760 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.00036700870058634387, |
|
"loss": 2.7605, |
|
"theoretical_loss": 3.745559872296941, |
|
"tokens_seen": 765460480 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.00036677227160960847, |
|
"loss": 2.7344, |
|
"theoretical_loss": 3.7449197013956503, |
|
"tokens_seen": 766771200 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0003665358426328731, |
|
"loss": 2.6958, |
|
"theoretical_loss": 3.744280929680005, |
|
"tokens_seen": 768081920 |
|
}, |
|
{ |
|
"epoch": 0.27, |
|
"learning_rate": 0.0003662994136561377, |
|
"loss": 2.7527, |
|
"theoretical_loss": 3.743643551712009, |
|
"tokens_seen": 769392640 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0003660629846794023, |
|
"loss": 2.7773, |
|
"theoretical_loss": 3.7430075620840215, |
|
"tokens_seen": 770703360 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0003658265557026669, |
|
"loss": 2.7209, |
|
"theoretical_loss": 3.7423729554185337, |
|
"tokens_seen": 772014080 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.00036559012672593156, |
|
"loss": 2.7133, |
|
"theoretical_loss": 3.7417397263679493, |
|
"tokens_seen": 773324800 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.00036535369774919616, |
|
"loss": 2.799, |
|
"theoretical_loss": 3.7411078696143694, |
|
"tokens_seen": 774635520 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.00036511726877246076, |
|
"loss": 2.8186, |
|
"theoretical_loss": 3.740477379869378, |
|
"tokens_seen": 775946240 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0003648808397957254, |
|
"loss": 2.7996, |
|
"theoretical_loss": 3.739848251873827, |
|
"tokens_seen": 777256960 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.00036464441081898995, |
|
"loss": 2.8178, |
|
"theoretical_loss": 3.739220480397626, |
|
"tokens_seen": 778567680 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0003644079818422546, |
|
"loss": 2.7482, |
|
"theoretical_loss": 3.7385940602395347, |
|
"tokens_seen": 779878400 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.00036417155286551926, |
|
"loss": 2.7766, |
|
"theoretical_loss": 3.737968986226954, |
|
"tokens_seen": 781189120 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0003639351238887838, |
|
"loss": 2.7678, |
|
"theoretical_loss": 3.7373452532157208, |
|
"tokens_seen": 782499840 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.00036369869491204845, |
|
"loss": 2.8746, |
|
"theoretical_loss": 3.736722856089904, |
|
"tokens_seen": 783810560 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.00036346226593531305, |
|
"loss": 2.8423, |
|
"theoretical_loss": 3.736101789761605, |
|
"tokens_seen": 785121280 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"objective/train/docs_used": 440553, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.451080322265625, |
|
"objective/train/theoretical_loss": 3.735482049170753, |
|
"objective/train/tokens_used": 806892000, |
|
"theoretical_loss": 3.735482049170753, |
|
"tokens_seen": 786432000 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.00036322583695857765, |
|
"loss": 2.7965, |
|
"theoretical_loss": 3.735482049170753, |
|
"tokens_seen": 786432000 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.00036298940798184224, |
|
"loss": 2.8513, |
|
"theoretical_loss": 3.734863629284911, |
|
"tokens_seen": 787742720 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0003627529790051069, |
|
"loss": 2.8257, |
|
"theoretical_loss": 3.7342465250990777, |
|
"tokens_seen": 789053440 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0003625165500283715, |
|
"loss": 2.8062, |
|
"theoretical_loss": 3.7336307316354933, |
|
"tokens_seen": 790364160 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0003622801210516361, |
|
"loss": 2.8369, |
|
"theoretical_loss": 3.7330162439434442, |
|
"tokens_seen": 791674880 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.00036204369207490074, |
|
"loss": 2.8066, |
|
"theoretical_loss": 3.7324030570990736, |
|
"tokens_seen": 792985600 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0003618072630981653, |
|
"loss": 2.8327, |
|
"theoretical_loss": 3.7317911662051912, |
|
"tokens_seen": 794296320 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.00036157083412142994, |
|
"loss": 2.7777, |
|
"theoretical_loss": 3.731180566391086, |
|
"tokens_seen": 795607040 |
|
}, |
|
{ |
|
"epoch": 0.28, |
|
"learning_rate": 0.0003613344051446946, |
|
"loss": 2.7566, |
|
"theoretical_loss": 3.7305712528123367, |
|
"tokens_seen": 796917760 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.00036109797616795913, |
|
"loss": 2.8123, |
|
"theoretical_loss": 3.729963220650631, |
|
"tokens_seen": 798228480 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0003608615471912238, |
|
"loss": 2.8019, |
|
"theoretical_loss": 3.729356465113578, |
|
"tokens_seen": 799539200 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0003606251182144884, |
|
"loss": 2.8193, |
|
"theoretical_loss": 3.7287509814345317, |
|
"tokens_seen": 800849920 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.000360388689237753, |
|
"loss": 2.8706, |
|
"theoretical_loss": 3.7281467648724043, |
|
"tokens_seen": 802160640 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.00036015226026101763, |
|
"loss": 2.7808, |
|
"theoretical_loss": 3.727543810711495, |
|
"tokens_seen": 803471360 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0003599158312842822, |
|
"loss": 2.8134, |
|
"theoretical_loss": 3.7269421142613055, |
|
"tokens_seen": 804782080 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0003596794023075468, |
|
"loss": 2.8048, |
|
"theoretical_loss": 3.7263416708563715, |
|
"tokens_seen": 806092800 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0003594429733308114, |
|
"loss": 2.7764, |
|
"theoretical_loss": 3.725742475856083, |
|
"tokens_seen": 807403520 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0003592065443540761, |
|
"loss": 2.8125, |
|
"theoretical_loss": 3.7251445246445147, |
|
"tokens_seen": 808714240 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.00035897011537734067, |
|
"loss": 2.8072, |
|
"theoretical_loss": 3.724547812630255, |
|
"tokens_seen": 810024960 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.00035873368640060527, |
|
"loss": 2.8413, |
|
"theoretical_loss": 3.7239523352462354, |
|
"tokens_seen": 811335680 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0003584972574238699, |
|
"loss": 2.8219, |
|
"theoretical_loss": 3.7233580879495616, |
|
"tokens_seen": 812646400 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.00035826082844713446, |
|
"loss": 2.7908, |
|
"theoretical_loss": 3.72276506622135, |
|
"tokens_seen": 813957120 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0003580243994703991, |
|
"loss": 2.7585, |
|
"theoretical_loss": 3.722173265566559, |
|
"tokens_seen": 815267840 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.00035778797049366366, |
|
"loss": 2.8308, |
|
"theoretical_loss": 3.7215826815138264, |
|
"tokens_seen": 816578560 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0003575515415169283, |
|
"loss": 2.8019, |
|
"theoretical_loss": 3.720993309615307, |
|
"tokens_seen": 817889280 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.00035731511254019296, |
|
"loss": 2.8033, |
|
"theoretical_loss": 3.720405145446512, |
|
"tokens_seen": 819200000 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0003570786835634575, |
|
"loss": 2.793, |
|
"theoretical_loss": 3.719818184606148, |
|
"tokens_seen": 820510720 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.00035684225458672216, |
|
"loss": 2.8288, |
|
"theoretical_loss": 3.719232422715959, |
|
"tokens_seen": 821821440 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.00035660582560998675, |
|
"loss": 2.7718, |
|
"theoretical_loss": 3.7186478554205693, |
|
"tokens_seen": 823132160 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.0003563812180820881, |
|
"loss": 2.7668, |
|
"theoretical_loss": 3.718064478387329, |
|
"tokens_seen": 824442880 |
|
}, |
|
{ |
|
"epoch": 0.29, |
|
"learning_rate": 0.00035614478910535275, |
|
"loss": 2.7649, |
|
"theoretical_loss": 3.717482287306157, |
|
"tokens_seen": 825753600 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0003559083601286174, |
|
"loss": 2.7729, |
|
"theoretical_loss": 3.7169012778893897, |
|
"tokens_seen": 827064320 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.00035567193115188195, |
|
"loss": 2.7556, |
|
"theoretical_loss": 3.71632144587163, |
|
"tokens_seen": 828375040 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0003554355021751466, |
|
"loss": 2.8103, |
|
"theoretical_loss": 3.7157427870095936, |
|
"tokens_seen": 829685760 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0003551990731984112, |
|
"loss": 2.8171, |
|
"theoretical_loss": 3.7151652970819615, |
|
"tokens_seen": 830996480 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0003549626442216758, |
|
"loss": 2.761, |
|
"theoretical_loss": 3.714588971889232, |
|
"tokens_seen": 832307200 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0003547262152449404, |
|
"loss": 2.7281, |
|
"theoretical_loss": 3.7140138072535738, |
|
"tokens_seen": 833617920 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.00035448978626820504, |
|
"loss": 2.7305, |
|
"theoretical_loss": 3.7134397990186794, |
|
"tokens_seen": 834928640 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.00035425335729146964, |
|
"loss": 2.7776, |
|
"theoretical_loss": 3.7128669430496215, |
|
"tokens_seen": 836239360 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.00035401692831473424, |
|
"loss": 2.7347, |
|
"theoretical_loss": 3.71229523523271, |
|
"tokens_seen": 837550080 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0003537804993379989, |
|
"loss": 2.6805, |
|
"theoretical_loss": 3.7117246714753485, |
|
"tokens_seen": 838860800 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.00035354407036126343, |
|
"loss": 2.7383, |
|
"theoretical_loss": 3.7111552477058947, |
|
"tokens_seen": 840171520 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0003533076413845281, |
|
"loss": 2.7158, |
|
"theoretical_loss": 3.7105869598735195, |
|
"tokens_seen": 841482240 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.00035307121240779274, |
|
"loss": 2.7225, |
|
"theoretical_loss": 3.71001980394807, |
|
"tokens_seen": 842792960 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0003528347834310573, |
|
"loss": 2.7274, |
|
"theoretical_loss": 3.709453775919931, |
|
"tokens_seen": 844103680 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.00035259835445432193, |
|
"loss": 2.7399, |
|
"theoretical_loss": 3.7088888717998865, |
|
"tokens_seen": 845414400 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.00035236192547758653, |
|
"loss": 2.7129, |
|
"theoretical_loss": 3.70832508761899, |
|
"tokens_seen": 846725120 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.00035212549650085113, |
|
"loss": 2.7134, |
|
"theoretical_loss": 3.707762419428425, |
|
"tokens_seen": 848035840 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0003518890675241158, |
|
"loss": 2.7313, |
|
"theoretical_loss": 3.7072008632993745, |
|
"tokens_seen": 849346560 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.0003516526385473804, |
|
"loss": 2.8078, |
|
"theoretical_loss": 3.7066404153228873, |
|
"tokens_seen": 850657280 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"objective/train/docs_used": 477195, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.859386920928955, |
|
"objective/train/theoretical_loss": 3.7060810716097503, |
|
"objective/train/tokens_used": 872428000, |
|
"theoretical_loss": 3.7060810716097503, |
|
"tokens_seen": 851968000 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.000351416209570645, |
|
"loss": 2.7659, |
|
"theoretical_loss": 3.7060810716097503, |
|
"tokens_seen": 851968000 |
|
}, |
|
{ |
|
"epoch": 0.3, |
|
"learning_rate": 0.00035117978059390957, |
|
"loss": 2.7479, |
|
"theoretical_loss": 3.7055228282903556, |
|
"tokens_seen": 853278720 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0003509433516171742, |
|
"loss": 2.7731, |
|
"theoretical_loss": 3.704965681514573, |
|
"tokens_seen": 854589440 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0003507069226404388, |
|
"loss": 2.8253, |
|
"theoretical_loss": 3.7044096274516214, |
|
"tokens_seen": 855900160 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0003504704936637034, |
|
"loss": 2.716, |
|
"theoretical_loss": 3.7038546622899435, |
|
"tokens_seen": 857210880 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00035023406468696807, |
|
"loss": 2.7689, |
|
"theoretical_loss": 3.70330078223708, |
|
"tokens_seen": 858521600 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0003499976357102326, |
|
"loss": 2.7538, |
|
"theoretical_loss": 3.7027479835195427, |
|
"tokens_seen": 859832320 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00034976120673349727, |
|
"loss": 2.7398, |
|
"theoretical_loss": 3.702196262382694, |
|
"tokens_seen": 861143040 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0003495247777567619, |
|
"loss": 2.7677, |
|
"theoretical_loss": 3.701645615090622, |
|
"tokens_seen": 862453760 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00034928834878002646, |
|
"loss": 2.6571, |
|
"theoretical_loss": 3.7010960379260185, |
|
"tokens_seen": 863764480 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0003490519198032911, |
|
"loss": 2.7121, |
|
"theoretical_loss": 3.7005475271900616, |
|
"tokens_seen": 865075200 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0003488154908265557, |
|
"loss": 2.7246, |
|
"theoretical_loss": 3.7000000792022916, |
|
"tokens_seen": 866385920 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0003485790618498203, |
|
"loss": 2.6893, |
|
"theoretical_loss": 3.699453690300495, |
|
"tokens_seen": 867696640 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0003483426328730849, |
|
"loss": 2.7052, |
|
"theoretical_loss": 3.698908356840586, |
|
"tokens_seen": 869007360 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00034810620389634956, |
|
"loss": 2.7138, |
|
"theoretical_loss": 3.6983640751964897, |
|
"tokens_seen": 870318080 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00034786977491961415, |
|
"loss": 2.774, |
|
"theoretical_loss": 3.697820841760025, |
|
"tokens_seen": 871628800 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00034763334594287875, |
|
"loss": 2.738, |
|
"theoretical_loss": 3.697278652940791, |
|
"tokens_seen": 872939520 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0003473969169661434, |
|
"loss": 2.74, |
|
"theoretical_loss": 3.6967375051660527, |
|
"tokens_seen": 874250240 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00034716048798940795, |
|
"loss": 2.7524, |
|
"theoretical_loss": 3.6961973948806266, |
|
"tokens_seen": 875560960 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0003469240590126726, |
|
"loss": 2.7544, |
|
"theoretical_loss": 3.6956583185467693, |
|
"tokens_seen": 876871680 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00034668763003593725, |
|
"loss": 2.6895, |
|
"theoretical_loss": 3.6951202726440666, |
|
"tokens_seen": 878182400 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.0003464512010592018, |
|
"loss": 2.7112, |
|
"theoretical_loss": 3.6945832536693204, |
|
"tokens_seen": 879493120 |
|
}, |
|
{ |
|
"epoch": 0.31, |
|
"learning_rate": 0.00034621477208246644, |
|
"loss": 2.7563, |
|
"theoretical_loss": 3.6940472581364423, |
|
"tokens_seen": 880803840 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.00034597834310573104, |
|
"loss": 2.6831, |
|
"theoretical_loss": 3.6935122825763433, |
|
"tokens_seen": 882114560 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.00034574191412899564, |
|
"loss": 2.7634, |
|
"theoretical_loss": 3.692978323536824, |
|
"tokens_seen": 883425280 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0003455054851522603, |
|
"loss": 2.7442, |
|
"theoretical_loss": 3.6924453775824704, |
|
"tokens_seen": 884736000 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0003452690561755249, |
|
"loss": 2.7676, |
|
"theoretical_loss": 3.6919134412945462, |
|
"tokens_seen": 886046720 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0003450326271987895, |
|
"loss": 2.798, |
|
"theoretical_loss": 3.691382511270887, |
|
"tokens_seen": 887357440 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0003447961982220541, |
|
"loss": 2.7513, |
|
"theoretical_loss": 3.6908525841257953, |
|
"tokens_seen": 888668160 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.00034455976924531874, |
|
"loss": 2.7525, |
|
"theoretical_loss": 3.6903236564899395, |
|
"tokens_seen": 889978880 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.00034432334026858333, |
|
"loss": 2.7335, |
|
"theoretical_loss": 3.6897957250102467, |
|
"tokens_seen": 891289600 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.00034408691129184793, |
|
"loss": 2.6977, |
|
"theoretical_loss": 3.6892687863498024, |
|
"tokens_seen": 892600320 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0003438504823151126, |
|
"loss": 2.7333, |
|
"theoretical_loss": 3.6887428371877506, |
|
"tokens_seen": 893911040 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0003436140533383771, |
|
"loss": 2.7406, |
|
"theoretical_loss": 3.688217874219191, |
|
"tokens_seen": 895221760 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0003433776243616418, |
|
"loss": 2.7368, |
|
"theoretical_loss": 3.6876938941550783, |
|
"tokens_seen": 896532480 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.00034314119538490643, |
|
"loss": 2.7568, |
|
"theoretical_loss": 3.6871708937221275, |
|
"tokens_seen": 897843200 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.00034290476640817097, |
|
"loss": 2.7438, |
|
"theoretical_loss": 3.686648869662709, |
|
"tokens_seen": 899153920 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0003426683374314356, |
|
"loss": 2.7176, |
|
"theoretical_loss": 3.6861278187347573, |
|
"tokens_seen": 900464640 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0003424319084547002, |
|
"loss": 2.7278, |
|
"theoretical_loss": 3.6856077377116696, |
|
"tokens_seen": 901775360 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0003421954794779648, |
|
"loss": 2.704, |
|
"theoretical_loss": 3.685088623382213, |
|
"tokens_seen": 903086080 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0003419590505012294, |
|
"loss": 2.6932, |
|
"theoretical_loss": 3.6845704725504254, |
|
"tokens_seen": 904396800 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.00034172262152449407, |
|
"loss": 2.6636, |
|
"theoretical_loss": 3.684053282035526, |
|
"tokens_seen": 905707520 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.00034148619254775866, |
|
"loss": 2.7068, |
|
"theoretical_loss": 3.6835370486718153, |
|
"tokens_seen": 907018240 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.00034124976357102326, |
|
"loss": 2.7572, |
|
"theoretical_loss": 3.6830217693085876, |
|
"tokens_seen": 908328960 |
|
}, |
|
{ |
|
"epoch": 0.32, |
|
"learning_rate": 0.0003410133345942879, |
|
"loss": 2.7177, |
|
"theoretical_loss": 3.682507440810034, |
|
"tokens_seen": 909639680 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00034077690561755246, |
|
"loss": 2.6885, |
|
"theoretical_loss": 3.6819940600551533, |
|
"tokens_seen": 910950400 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.0003405404766408171, |
|
"loss": 2.7888, |
|
"theoretical_loss": 3.6814816239376595, |
|
"tokens_seen": 912261120 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00034030404766408176, |
|
"loss": 2.7462, |
|
"theoretical_loss": 3.6809701293658934, |
|
"tokens_seen": 913571840 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.0003400676186873463, |
|
"loss": 2.7734, |
|
"theoretical_loss": 3.6804595732627297, |
|
"tokens_seen": 914882560 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00033983118971061096, |
|
"loss": 2.7436, |
|
"theoretical_loss": 3.6799499525654915, |
|
"tokens_seen": 916193280 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"objective/train/docs_used": 513163, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.83975887298584, |
|
"objective/train/theoretical_loss": 3.6794412642258583, |
|
"objective/train/tokens_used": 937964000, |
|
"theoretical_loss": 3.6794412642258583, |
|
"tokens_seen": 917504000 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00033959476073387555, |
|
"loss": 2.7804, |
|
"theoretical_loss": 3.6794412642258583, |
|
"tokens_seen": 917504000 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00033935833175714015, |
|
"loss": 2.787, |
|
"theoretical_loss": 3.6789335052097814, |
|
"tokens_seen": 918814720 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.0003391219027804048, |
|
"loss": 2.7717, |
|
"theoretical_loss": 3.6784266724973946, |
|
"tokens_seen": 920125440 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.0003388854738036694, |
|
"loss": 2.7296, |
|
"theoretical_loss": 3.6779207630829296, |
|
"tokens_seen": 921436160 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.000338649044826934, |
|
"loss": 2.7581, |
|
"theoretical_loss": 3.677415773974629, |
|
"tokens_seen": 922746880 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.0003384126158501986, |
|
"loss": 2.726, |
|
"theoretical_loss": 3.6769117021946625, |
|
"tokens_seen": 924057600 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00033817618687346325, |
|
"loss": 2.7613, |
|
"theoretical_loss": 3.676408544779039, |
|
"tokens_seen": 925368320 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00033793975789672784, |
|
"loss": 2.7116, |
|
"theoretical_loss": 3.6759062987775293, |
|
"tokens_seen": 926679040 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00033770332891999244, |
|
"loss": 2.7743, |
|
"theoretical_loss": 3.6754049612535753, |
|
"tokens_seen": 927989760 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.0003374668999432571, |
|
"loss": 2.7662, |
|
"theoretical_loss": 3.6749045292842126, |
|
"tokens_seen": 929300480 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00033723047096652164, |
|
"loss": 2.7676, |
|
"theoretical_loss": 3.674404999959987, |
|
"tokens_seen": 930611200 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.0003369940419897863, |
|
"loss": 2.8034, |
|
"theoretical_loss": 3.6739063703848718, |
|
"tokens_seen": 931921920 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.0003367576130130509, |
|
"loss": 2.7715, |
|
"theoretical_loss": 3.6734086376761907, |
|
"tokens_seen": 933232640 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.0003365211840363155, |
|
"loss": 2.8154, |
|
"theoretical_loss": 3.6729117989645332, |
|
"tokens_seen": 934543360 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00033628475505958013, |
|
"loss": 2.7405, |
|
"theoretical_loss": 3.6724158513936773, |
|
"tokens_seen": 935854080 |
|
}, |
|
{ |
|
"epoch": 0.33, |
|
"learning_rate": 0.00033604832608284473, |
|
"loss": 2.7108, |
|
"theoretical_loss": 3.6719207921205115, |
|
"tokens_seen": 937164800 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.00033581189710610933, |
|
"loss": 2.7059, |
|
"theoretical_loss": 3.6714266183149533, |
|
"tokens_seen": 938475520 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0003355754681293739, |
|
"loss": 2.7555, |
|
"theoretical_loss": 3.670933327159875, |
|
"tokens_seen": 939786240 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0003353390391526386, |
|
"loss": 2.7353, |
|
"theoretical_loss": 3.670440915851024, |
|
"tokens_seen": 941096960 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0003351026101759032, |
|
"loss": 2.73, |
|
"theoretical_loss": 3.6699493815969464, |
|
"tokens_seen": 942407680 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0003348661811991678, |
|
"loss": 2.7755, |
|
"theoretical_loss": 3.6694587216189114, |
|
"tokens_seen": 943718400 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0003346297522224324, |
|
"loss": 2.7286, |
|
"theoretical_loss": 3.6689689331508366, |
|
"tokens_seen": 945029120 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.00033439332324569697, |
|
"loss": 2.7363, |
|
"theoretical_loss": 3.6684800134392104, |
|
"tokens_seen": 946339840 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0003341568942689616, |
|
"loss": 2.7026, |
|
"theoretical_loss": 3.6679919597430213, |
|
"tokens_seen": 947650560 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.00033392046529222627, |
|
"loss": 2.6754, |
|
"theoretical_loss": 3.6675047693336795, |
|
"tokens_seen": 948961280 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0003336840363154908, |
|
"loss": 2.7145, |
|
"theoretical_loss": 3.6670184394949477, |
|
"tokens_seen": 950272000 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.00033344760733875547, |
|
"loss": 2.7392, |
|
"theoretical_loss": 3.6665329675228646, |
|
"tokens_seen": 951582720 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.00033321117836202, |
|
"loss": 2.7079, |
|
"theoretical_loss": 3.666048350725677, |
|
"tokens_seen": 952893440 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.00033297474938528466, |
|
"loss": 2.7194, |
|
"theoretical_loss": 3.6655645864237627, |
|
"tokens_seen": 954204160 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0003327383204085493, |
|
"loss": 2.6784, |
|
"theoretical_loss": 3.6650816719495625, |
|
"tokens_seen": 955514880 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.00033250189143181386, |
|
"loss": 2.6533, |
|
"theoretical_loss": 3.6645996046475107, |
|
"tokens_seen": 956825600 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0003322654624550785, |
|
"loss": 2.6745, |
|
"theoretical_loss": 3.664118381873961, |
|
"tokens_seen": 958136320 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0003320290334783431, |
|
"loss": 2.665, |
|
"theoretical_loss": 3.66363800099712, |
|
"tokens_seen": 959447040 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0003317926045016077, |
|
"loss": 2.7108, |
|
"theoretical_loss": 3.6631584593969757, |
|
"tokens_seen": 960757760 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.0003315561755248723, |
|
"loss": 2.648, |
|
"theoretical_loss": 3.6626797544652305, |
|
"tokens_seen": 962068480 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.00033131974654813695, |
|
"loss": 2.6699, |
|
"theoretical_loss": 3.6622018836052317, |
|
"tokens_seen": 963379200 |
|
}, |
|
{ |
|
"epoch": 0.34, |
|
"learning_rate": 0.00033108331757140155, |
|
"loss": 2.7234, |
|
"theoretical_loss": 3.6617248442319053, |
|
"tokens_seen": 964689920 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00033084688859466615, |
|
"loss": 2.6963, |
|
"theoretical_loss": 3.661248633771687, |
|
"tokens_seen": 966000640 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0003306104596179308, |
|
"loss": 2.6768, |
|
"theoretical_loss": 3.6607732496624568, |
|
"tokens_seen": 967311360 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00033037403064119534, |
|
"loss": 2.6924, |
|
"theoretical_loss": 3.660298689353471, |
|
"tokens_seen": 968622080 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00033013760166446, |
|
"loss": 2.7223, |
|
"theoretical_loss": 3.659824950305297, |
|
"tokens_seen": 969932800 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00032990117268772465, |
|
"loss": 2.7665, |
|
"theoretical_loss": 3.6593520299897513, |
|
"tokens_seen": 971243520 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0003296647437109892, |
|
"loss": 2.7692, |
|
"theoretical_loss": 3.6588799258898277, |
|
"tokens_seen": 972554240 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00032942831473425384, |
|
"loss": 2.7858, |
|
"theoretical_loss": 3.65840863549964, |
|
"tokens_seen": 973864960 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00032919188575751844, |
|
"loss": 2.737, |
|
"theoretical_loss": 3.6579381563243527, |
|
"tokens_seen": 975175680 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00032895545678078304, |
|
"loss": 2.7019, |
|
"theoretical_loss": 3.6574684858801207, |
|
"tokens_seen": 976486400 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0003287190278040477, |
|
"loss": 2.6586, |
|
"theoretical_loss": 3.656999621694025, |
|
"tokens_seen": 977797120 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0003284825988273123, |
|
"loss": 2.6835, |
|
"theoretical_loss": 3.65653156130401, |
|
"tokens_seen": 979107840 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0003282461698505769, |
|
"loss": 2.6699, |
|
"theoretical_loss": 3.6560643022588217, |
|
"tokens_seen": 980418560 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0003280097408738415, |
|
"loss": 2.6888, |
|
"theoretical_loss": 3.655597842117945, |
|
"tokens_seen": 981729280 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.5395508255365915, |
|
"debugging/distinct-1-grams": 0.7683501269789897, |
|
"debugging/distinct-2-grams": 0.9413977022884555, |
|
"debugging/entropy-1-grams": 6.198614472060548, |
|
"debugging/entropy-2-grams": 7.1599602011918995, |
|
"debugging/length": 502.2, |
|
"debugging/num_segments": 20, |
|
"epoch": 0.35, |
|
"objective/train/docs_used": 548797, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.6332430839538574, |
|
"objective/train/theoretical_loss": 3.6551321784515434, |
|
"objective/train/tokens_used": 1003500000, |
|
"theoretical_loss": 3.6551321784515434, |
|
"tokens_seen": 983040000 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00032777331189710613, |
|
"loss": 2.6951, |
|
"theoretical_loss": 3.6551321784515434, |
|
"tokens_seen": 983040000 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.00032753688292037073, |
|
"loss": 2.7164, |
|
"theoretical_loss": 3.6546673088403985, |
|
"tokens_seen": 984350720 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0003273004539436353, |
|
"loss": 2.7035, |
|
"theoretical_loss": 3.654203230875848, |
|
"tokens_seen": 985661440 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0003270640249669, |
|
"loss": 2.6849, |
|
"theoretical_loss": 3.653739942159728, |
|
"tokens_seen": 986972160 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0003268275959901645, |
|
"loss": 2.6898, |
|
"theoretical_loss": 3.65327744030431, |
|
"tokens_seen": 988282880 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0003265911670134292, |
|
"loss": 2.7672, |
|
"theoretical_loss": 3.6528157229322447, |
|
"tokens_seen": 989593600 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0003263547380366938, |
|
"loss": 2.6891, |
|
"theoretical_loss": 3.6523547876765017, |
|
"tokens_seen": 990904320 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0003261419519576319, |
|
"loss": 2.7356, |
|
"theoretical_loss": 3.651894632180312, |
|
"tokens_seen": 992215040 |
|
}, |
|
{ |
|
"epoch": 0.35, |
|
"learning_rate": 0.0003259055229808965, |
|
"loss": 2.8215, |
|
"theoretical_loss": 3.65143525409711, |
|
"tokens_seen": 993525760 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0003256690940041612, |
|
"loss": 2.7848, |
|
"theoretical_loss": 3.650976651090474, |
|
"tokens_seen": 994836480 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.00032543266502742577, |
|
"loss": 2.8296, |
|
"theoretical_loss": 3.650518820834072, |
|
"tokens_seen": 996147200 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.00032519623605069037, |
|
"loss": 2.8453, |
|
"theoretical_loss": 3.650061761011603, |
|
"tokens_seen": 997457920 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.000324959807073955, |
|
"loss": 2.8247, |
|
"theoretical_loss": 3.649605469316742, |
|
"tokens_seen": 998768640 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.00032472337809721956, |
|
"loss": 2.8156, |
|
"theoretical_loss": 3.6491499434530814, |
|
"tokens_seen": 1000079360 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0003244869491204842, |
|
"loss": 2.8627, |
|
"theoretical_loss": 3.648695181134078, |
|
"tokens_seen": 1001390080 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0003242505201437488, |
|
"loss": 2.7878, |
|
"theoretical_loss": 3.648241180082998, |
|
"tokens_seen": 1002700800 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0003240140911670134, |
|
"loss": 2.8126, |
|
"theoretical_loss": 3.64778793803286, |
|
"tokens_seen": 1004011520 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.00032377766219027806, |
|
"loss": 2.8221, |
|
"theoretical_loss": 3.6473354527263813, |
|
"tokens_seen": 1005322240 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.00032354123321354266, |
|
"loss": 2.8165, |
|
"theoretical_loss": 3.6468837219159256, |
|
"tokens_seen": 1006632960 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.00032330480423680726, |
|
"loss": 2.844, |
|
"theoretical_loss": 3.6464327433634467, |
|
"tokens_seen": 1007943680 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.00032306837526007185, |
|
"loss": 2.8321, |
|
"theoretical_loss": 3.645982514840437, |
|
"tokens_seen": 1009254400 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0003228319462833365, |
|
"loss": 2.7851, |
|
"theoretical_loss": 3.6455330341278733, |
|
"tokens_seen": 1010565120 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0003225955173066011, |
|
"loss": 2.8547, |
|
"theoretical_loss": 3.645084299016166, |
|
"tokens_seen": 1011875840 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0003223590883298657, |
|
"loss": 2.8491, |
|
"theoretical_loss": 3.644636307305104, |
|
"tokens_seen": 1013186560 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.00032212265935313035, |
|
"loss": 2.778, |
|
"theoretical_loss": 3.644189056803805, |
|
"tokens_seen": 1014497280 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0003218862303763949, |
|
"loss": 2.8114, |
|
"theoretical_loss": 3.6437425453306638, |
|
"tokens_seen": 1015808000 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.00032164980139965955, |
|
"loss": 2.8036, |
|
"theoretical_loss": 3.643296770713301, |
|
"tokens_seen": 1017118720 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0003214133724229242, |
|
"loss": 2.7573, |
|
"theoretical_loss": 3.642851730788511, |
|
"tokens_seen": 1018429440 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.00032117694344618874, |
|
"loss": 2.7723, |
|
"theoretical_loss": 3.642407423402213, |
|
"tokens_seen": 1019740160 |
|
}, |
|
{ |
|
"epoch": 0.36, |
|
"learning_rate": 0.0003209405144694534, |
|
"loss": 2.7911, |
|
"theoretical_loss": 3.641963846409399, |
|
"tokens_seen": 1021050880 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.000320704085492718, |
|
"loss": 2.7345, |
|
"theoretical_loss": 3.641520997674087, |
|
"tokens_seen": 1022361600 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0003204676565159826, |
|
"loss": 2.796, |
|
"theoretical_loss": 3.641078875069269, |
|
"tokens_seen": 1023672320 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00032023122753924724, |
|
"loss": 2.7403, |
|
"theoretical_loss": 3.640637476476862, |
|
"tokens_seen": 1024983040 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00031999479856251184, |
|
"loss": 2.7171, |
|
"theoretical_loss": 3.64019679978766, |
|
"tokens_seen": 1026293760 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00031975836958577644, |
|
"loss": 2.7919, |
|
"theoretical_loss": 3.639756842901286, |
|
"tokens_seen": 1027604480 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00031952194060904103, |
|
"loss": 2.7861, |
|
"theoretical_loss": 3.639317603726142, |
|
"tokens_seen": 1028915200 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0003192855116323057, |
|
"loss": 2.858, |
|
"theoretical_loss": 3.638879080179364, |
|
"tokens_seen": 1030225920 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00031904908265557023, |
|
"loss": 2.7529, |
|
"theoretical_loss": 3.63844127018677, |
|
"tokens_seen": 1031536640 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0003188126536788349, |
|
"loss": 2.7633, |
|
"theoretical_loss": 3.6380041716828186, |
|
"tokens_seen": 1032847360 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00031857622470209953, |
|
"loss": 2.8117, |
|
"theoretical_loss": 3.637567782610558, |
|
"tokens_seen": 1034158080 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0003183397957253641, |
|
"loss": 2.7669, |
|
"theoretical_loss": 3.637132100921579, |
|
"tokens_seen": 1035468800 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0003181033667486287, |
|
"loss": 2.752, |
|
"theoretical_loss": 3.6366971245759734, |
|
"tokens_seen": 1036779520 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0003178669377718933, |
|
"loss": 2.7997, |
|
"theoretical_loss": 3.6362628515422815, |
|
"tokens_seen": 1038090240 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0003176305087951579, |
|
"loss": 2.8115, |
|
"theoretical_loss": 3.635829279797453, |
|
"tokens_seen": 1039400960 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0003173940798184226, |
|
"loss": 2.811, |
|
"theoretical_loss": 3.6353964073267964, |
|
"tokens_seen": 1040711680 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00031715765084168717, |
|
"loss": 2.814, |
|
"theoretical_loss": 3.6349642321239375, |
|
"tokens_seen": 1042022400 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00031692122186495177, |
|
"loss": 2.7682, |
|
"theoretical_loss": 3.6345327521907724, |
|
"tokens_seen": 1043333120 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00031668479288821637, |
|
"loss": 2.7907, |
|
"theoretical_loss": 3.6341019655374245, |
|
"tokens_seen": 1044643840 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.000316448363911481, |
|
"loss": 2.7818, |
|
"theoretical_loss": 3.6336718701821997, |
|
"tokens_seen": 1045954560 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0003162119349347456, |
|
"loss": 2.7907, |
|
"theoretical_loss": 3.633242464151543, |
|
"tokens_seen": 1047265280 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"objective/train/docs_used": 582102, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.738445281982422, |
|
"objective/train/theoretical_loss": 3.632813745479994, |
|
"objective/train/tokens_used": 1069036000, |
|
"theoretical_loss": 3.632813745479994, |
|
"tokens_seen": 1048576000 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.0003159755059580102, |
|
"loss": 2.8192, |
|
"theoretical_loss": 3.632813745479994, |
|
"tokens_seen": 1048576000 |
|
}, |
|
{ |
|
"epoch": 0.37, |
|
"learning_rate": 0.00031573907698127486, |
|
"loss": 2.7641, |
|
"theoretical_loss": 3.6323857122101444, |
|
"tokens_seen": 1049886720 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0003155026480045394, |
|
"loss": 2.7563, |
|
"theoretical_loss": 3.631958362392595, |
|
"tokens_seen": 1051197440 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.00031526621902780406, |
|
"loss": 2.809, |
|
"theoretical_loss": 3.631531694085914, |
|
"tokens_seen": 1052508160 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0003150297900510687, |
|
"loss": 2.7815, |
|
"theoretical_loss": 3.6311057053565907, |
|
"tokens_seen": 1053818880 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.00031479336107433325, |
|
"loss": 2.7622, |
|
"theoretical_loss": 3.6306803942789987, |
|
"tokens_seen": 1055129600 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0003145569320975979, |
|
"loss": 2.8323, |
|
"theoretical_loss": 3.63025575893535, |
|
"tokens_seen": 1056440320 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0003143205031208625, |
|
"loss": 2.7748, |
|
"theoretical_loss": 3.6298317974156555, |
|
"tokens_seen": 1057751040 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0003140840741441271, |
|
"loss": 2.7607, |
|
"theoretical_loss": 3.629408507817682, |
|
"tokens_seen": 1059061760 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.00031384764516739175, |
|
"loss": 2.7514, |
|
"theoretical_loss": 3.6289858882469135, |
|
"tokens_seen": 1060372480 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.00031361121619065635, |
|
"loss": 2.7373, |
|
"theoretical_loss": 3.6285639368165077, |
|
"tokens_seen": 1061683200 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.00031337478721392095, |
|
"loss": 2.7827, |
|
"theoretical_loss": 3.6281426516472584, |
|
"tokens_seen": 1062993920 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.00031313835823718554, |
|
"loss": 2.7113, |
|
"theoretical_loss": 3.627722030867552, |
|
"tokens_seen": 1064304640 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0003129019292604502, |
|
"loss": 2.7309, |
|
"theoretical_loss": 3.62730207261333, |
|
"tokens_seen": 1065615360 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.00031266550028371474, |
|
"loss": 2.708, |
|
"theoretical_loss": 3.626882775028048, |
|
"tokens_seen": 1066926080 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0003124290713069794, |
|
"loss": 2.7269, |
|
"theoretical_loss": 3.6264641362626375, |
|
"tokens_seen": 1068236800 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.00031219264233024404, |
|
"loss": 2.7027, |
|
"theoretical_loss": 3.626046154475465, |
|
"tokens_seen": 1069547520 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0003119562133535086, |
|
"loss": 2.7199, |
|
"theoretical_loss": 3.625628827832294, |
|
"tokens_seen": 1070858240 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.00031171978437677324, |
|
"loss": 2.7021, |
|
"theoretical_loss": 3.625212154506246, |
|
"tokens_seen": 1072168960 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.00031148335540003784, |
|
"loss": 2.7076, |
|
"theoretical_loss": 3.624796132677763, |
|
"tokens_seen": 1073479680 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.00031124692642330243, |
|
"loss": 2.6884, |
|
"theoretical_loss": 3.624380760534567, |
|
"tokens_seen": 1074790400 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0003110104974465671, |
|
"loss": 2.7631, |
|
"theoretical_loss": 3.6239660362716246, |
|
"tokens_seen": 1076101120 |
|
}, |
|
{ |
|
"epoch": 0.38, |
|
"learning_rate": 0.0003107740684698317, |
|
"loss": 2.7291, |
|
"theoretical_loss": 3.623551958091108, |
|
"tokens_seen": 1077411840 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0003105376394930963, |
|
"loss": 2.7378, |
|
"theoretical_loss": 3.623138524202358, |
|
"tokens_seen": 1078722560 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0003103012105163609, |
|
"loss": 2.7292, |
|
"theoretical_loss": 3.622725732821844, |
|
"tokens_seen": 1080033280 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.00031006478153962553, |
|
"loss": 2.7589, |
|
"theoretical_loss": 3.622313582173133, |
|
"tokens_seen": 1081344000 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0003098283525628901, |
|
"loss": 2.6946, |
|
"theoretical_loss": 3.621902070486846, |
|
"tokens_seen": 1082654720 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0003095919235861547, |
|
"loss": 2.7934, |
|
"theoretical_loss": 3.6214911960006275, |
|
"tokens_seen": 1083965440 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0003093554946094194, |
|
"loss": 2.7683, |
|
"theoretical_loss": 3.621080956959104, |
|
"tokens_seen": 1085276160 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0003091190656326839, |
|
"loss": 2.6777, |
|
"theoretical_loss": 3.6206713516138507, |
|
"tokens_seen": 1086586880 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.00030888263665594857, |
|
"loss": 2.7258, |
|
"theoretical_loss": 3.620262378223357, |
|
"tokens_seen": 1087897600 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0003086462076792132, |
|
"loss": 2.7544, |
|
"theoretical_loss": 3.619854035052987, |
|
"tokens_seen": 1089208320 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.00030840977870247777, |
|
"loss": 2.7211, |
|
"theoretical_loss": 3.619446320374947, |
|
"tokens_seen": 1090519040 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0003081733497257424, |
|
"loss": 2.7215, |
|
"theoretical_loss": 3.61903923246825, |
|
"tokens_seen": 1091829760 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.000307936920749007, |
|
"loss": 2.6715, |
|
"theoretical_loss": 3.6186327696186797, |
|
"tokens_seen": 1093140480 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0003077004917722716, |
|
"loss": 2.7124, |
|
"theoretical_loss": 3.618226930118757, |
|
"tokens_seen": 1094451200 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.00030746406279553626, |
|
"loss": 2.7827, |
|
"theoretical_loss": 3.6178217122677045, |
|
"tokens_seen": 1095761920 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0003072276338188008, |
|
"loss": 2.7143, |
|
"theoretical_loss": 3.6174171143714124, |
|
"tokens_seen": 1097072640 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.00030699120484206546, |
|
"loss": 2.6943, |
|
"theoretical_loss": 3.617013134742405, |
|
"tokens_seen": 1098383360 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.00030675477586533006, |
|
"loss": 2.7524, |
|
"theoretical_loss": 3.616609771699806, |
|
"tokens_seen": 1099694080 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.00030651834688859465, |
|
"loss": 2.7242, |
|
"theoretical_loss": 3.616207023569306, |
|
"tokens_seen": 1101004800 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.00030628191791185925, |
|
"loss": 2.6719, |
|
"theoretical_loss": 3.6158048886831287, |
|
"tokens_seen": 1102315520 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0003060454889351239, |
|
"loss": 2.674, |
|
"theoretical_loss": 3.6154033653799953, |
|
"tokens_seen": 1103626240 |
|
}, |
|
{ |
|
"epoch": 0.39, |
|
"learning_rate": 0.0003058090599583885, |
|
"loss": 2.6836, |
|
"theoretical_loss": 3.615002452005096, |
|
"tokens_seen": 1104936960 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0003055726309816531, |
|
"loss": 2.7231, |
|
"theoretical_loss": 3.614602146910054, |
|
"tokens_seen": 1106247680 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00030533620200491775, |
|
"loss": 2.7444, |
|
"theoretical_loss": 3.614202448452893, |
|
"tokens_seen": 1107558400 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0003050997730281823, |
|
"loss": 2.7074, |
|
"theoretical_loss": 3.613803354998007, |
|
"tokens_seen": 1108869120 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00030486334405144694, |
|
"loss": 2.7037, |
|
"theoretical_loss": 3.6134048649161237, |
|
"tokens_seen": 1110179840 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0003046269150747116, |
|
"loss": 2.6998, |
|
"theoretical_loss": 3.6130069765842787, |
|
"tokens_seen": 1111490560 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00030439048609797614, |
|
"loss": 2.7138, |
|
"theoretical_loss": 3.6126096883857777, |
|
"tokens_seen": 1112801280 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"objective/train/docs_used": 617837, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.8926000595092773, |
|
"objective/train/theoretical_loss": 3.61221299871017, |
|
"objective/train/tokens_used": 1134572000, |
|
"theoretical_loss": 3.61221299871017, |
|
"tokens_seen": 1114112000 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0003041540571212408, |
|
"loss": 2.7617, |
|
"theoretical_loss": 3.61221299871017, |
|
"tokens_seen": 1114112000 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0003039176281445054, |
|
"loss": 2.7269, |
|
"theoretical_loss": 3.611816905953212, |
|
"tokens_seen": 1115422720 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00030368119916777, |
|
"loss": 2.6824, |
|
"theoretical_loss": 3.6114214085168417, |
|
"tokens_seen": 1116733440 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00030344477019103464, |
|
"loss": 2.7501, |
|
"theoretical_loss": 3.611026504809142, |
|
"tokens_seen": 1118044160 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00030320834121429923, |
|
"loss": 2.712, |
|
"theoretical_loss": 3.6106321932443137, |
|
"tokens_seen": 1119354880 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00030297191223756383, |
|
"loss": 2.6591, |
|
"theoretical_loss": 3.6102384722426435, |
|
"tokens_seen": 1120665600 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00030273548326082843, |
|
"loss": 2.7543, |
|
"theoretical_loss": 3.609845340230474, |
|
"tokens_seen": 1121976320 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0003024990542840931, |
|
"loss": 2.7203, |
|
"theoretical_loss": 3.6094527956401734, |
|
"tokens_seen": 1123287040 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0003022626253073577, |
|
"loss": 2.7033, |
|
"theoretical_loss": 3.6090608369101043, |
|
"tokens_seen": 1124597760 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0003020261963306223, |
|
"loss": 2.6814, |
|
"theoretical_loss": 3.6086694624845954, |
|
"tokens_seen": 1125908480 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00030178976735388693, |
|
"loss": 2.669, |
|
"theoretical_loss": 3.6082786708139114, |
|
"tokens_seen": 1127219200 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00030155333837715147, |
|
"loss": 2.6892, |
|
"theoretical_loss": 3.6078884603542223, |
|
"tokens_seen": 1128529920 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0003013169094004161, |
|
"loss": 2.6341, |
|
"theoretical_loss": 3.607498829567576, |
|
"tokens_seen": 1129840640 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0003010804804236807, |
|
"loss": 2.708, |
|
"theoretical_loss": 3.6071097769218676, |
|
"tokens_seen": 1131151360 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.0003008440514469453, |
|
"loss": 2.7243, |
|
"theoretical_loss": 3.606721300890811, |
|
"tokens_seen": 1132462080 |
|
}, |
|
{ |
|
"epoch": 0.4, |
|
"learning_rate": 0.00030060762247020997, |
|
"loss": 2.7242, |
|
"theoretical_loss": 3.6063333999539102, |
|
"tokens_seen": 1133772800 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00030037119349347457, |
|
"loss": 2.6964, |
|
"theoretical_loss": 3.6059460725964305, |
|
"tokens_seen": 1135083520 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00030013476451673916, |
|
"loss": 2.6908, |
|
"theoretical_loss": 3.6055593173093694, |
|
"tokens_seen": 1136394240 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00029989833554000376, |
|
"loss": 2.7061, |
|
"theoretical_loss": 3.60517313258943, |
|
"tokens_seen": 1137704960 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.0002996619065632684, |
|
"loss": 2.7295, |
|
"theoretical_loss": 3.604787516938992, |
|
"tokens_seen": 1139015680 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.000299425477586533, |
|
"loss": 2.6895, |
|
"theoretical_loss": 3.6044024688660823, |
|
"tokens_seen": 1140326400 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.0002991890486097976, |
|
"loss": 2.7346, |
|
"theoretical_loss": 3.604017986884349, |
|
"tokens_seen": 1141637120 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00029895261963306226, |
|
"loss": 2.6686, |
|
"theoretical_loss": 3.603634069513035, |
|
"tokens_seen": 1142947840 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.0002987161906563268, |
|
"loss": 2.6908, |
|
"theoretical_loss": 3.603250715276947, |
|
"tokens_seen": 1144258560 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00029847976167959146, |
|
"loss": 2.6998, |
|
"theoretical_loss": 3.6028679227064306, |
|
"tokens_seen": 1145569280 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.0002982433327028561, |
|
"loss": 2.75, |
|
"theoretical_loss": 3.6024856903373434, |
|
"tokens_seen": 1146880000 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00029800690372612065, |
|
"loss": 2.7409, |
|
"theoretical_loss": 3.602104016711027, |
|
"tokens_seen": 1148190720 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.0002977704747493853, |
|
"loss": 2.7675, |
|
"theoretical_loss": 3.6017229003742806, |
|
"tokens_seen": 1149501440 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.0002975340457726499, |
|
"loss": 2.7389, |
|
"theoretical_loss": 3.6013423398793347, |
|
"tokens_seen": 1150812160 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.0002972976167959145, |
|
"loss": 2.7125, |
|
"theoretical_loss": 3.600962333783824, |
|
"tokens_seen": 1152122880 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00029706118781917915, |
|
"loss": 2.728, |
|
"theoretical_loss": 3.6005828806507623, |
|
"tokens_seen": 1153433600 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00029682475884244375, |
|
"loss": 2.7785, |
|
"theoretical_loss": 3.6002039790485147, |
|
"tokens_seen": 1154744320 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00029658832986570834, |
|
"loss": 2.7339, |
|
"theoretical_loss": 3.5998256275507723, |
|
"tokens_seen": 1156055040 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00029635190088897294, |
|
"loss": 2.7564, |
|
"theoretical_loss": 3.5994478247365276, |
|
"tokens_seen": 1157365760 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.0002961154719122376, |
|
"loss": 2.7336, |
|
"theoretical_loss": 3.5990705691900464, |
|
"tokens_seen": 1158676480 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.00029587904293550214, |
|
"loss": 2.6968, |
|
"theoretical_loss": 3.5986938595008455, |
|
"tokens_seen": 1159987200 |
|
}, |
|
{ |
|
"epoch": 0.41, |
|
"learning_rate": 0.0002956426139587668, |
|
"loss": 2.739, |
|
"theoretical_loss": 3.5983176942636637, |
|
"tokens_seen": 1161297920 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00029540618498203144, |
|
"loss": 2.7562, |
|
"theoretical_loss": 3.597942072078439, |
|
"tokens_seen": 1162608640 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.000295169756005296, |
|
"loss": 2.6762, |
|
"theoretical_loss": 3.5975669915502833, |
|
"tokens_seen": 1163919360 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00029493332702856063, |
|
"loss": 2.693, |
|
"theoretical_loss": 3.597192451289457, |
|
"tokens_seen": 1165230080 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00029469689805182523, |
|
"loss": 2.7433, |
|
"theoretical_loss": 3.5968184499113436, |
|
"tokens_seen": 1166540800 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00029446046907508983, |
|
"loss": 2.7353, |
|
"theoretical_loss": 3.5964449860364263, |
|
"tokens_seen": 1167851520 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0002942240400983545, |
|
"loss": 2.7333, |
|
"theoretical_loss": 3.5960720582902628, |
|
"tokens_seen": 1169162240 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0002939876111216191, |
|
"loss": 2.6914, |
|
"theoretical_loss": 3.595699665303462, |
|
"tokens_seen": 1170472960 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0002937511821448837, |
|
"loss": 2.6491, |
|
"theoretical_loss": 3.595327805711657, |
|
"tokens_seen": 1171783680 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0002935147531681483, |
|
"loss": 2.6961, |
|
"theoretical_loss": 3.5949564781554857, |
|
"tokens_seen": 1173094400 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0002932783241914129, |
|
"loss": 2.7129, |
|
"theoretical_loss": 3.5945856812805626, |
|
"tokens_seen": 1174405120 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0002930418952146775, |
|
"loss": 2.751, |
|
"theoretical_loss": 3.594215413737457, |
|
"tokens_seen": 1175715840 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0002928054662379421, |
|
"loss": 2.7487, |
|
"theoretical_loss": 3.5938456741816682, |
|
"tokens_seen": 1177026560 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00029256903726120677, |
|
"loss": 2.7021, |
|
"theoretical_loss": 3.593476461273605, |
|
"tokens_seen": 1178337280 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"objective/train/docs_used": 651196, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.538501501083374, |
|
"objective/train/theoretical_loss": 3.5931077736785584, |
|
"objective/train/tokens_used": 1200108000, |
|
"theoretical_loss": 3.5931077736785584, |
|
"tokens_seen": 1179648000 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0002923326082844713, |
|
"loss": 2.6904, |
|
"theoretical_loss": 3.5931077736785584, |
|
"tokens_seen": 1179648000 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00029209617930773597, |
|
"loss": 2.7642, |
|
"theoretical_loss": 3.5927396100666806, |
|
"tokens_seen": 1180958720 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0002918597503310006, |
|
"loss": 2.7317, |
|
"theoretical_loss": 3.592371969112962, |
|
"tokens_seen": 1182269440 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00029162332135426516, |
|
"loss": 2.6943, |
|
"theoretical_loss": 3.5920048494972074, |
|
"tokens_seen": 1183580160 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0002913868923775298, |
|
"loss": 2.7779, |
|
"theoretical_loss": 3.5916382499040136, |
|
"tokens_seen": 1184890880 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.0002911504634007944, |
|
"loss": 2.7194, |
|
"theoretical_loss": 3.591272169022747, |
|
"tokens_seen": 1186201600 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.000290914034424059, |
|
"loss": 2.6754, |
|
"theoretical_loss": 3.590906605547521, |
|
"tokens_seen": 1187512320 |
|
}, |
|
{ |
|
"epoch": 0.42, |
|
"learning_rate": 0.00029067760544732366, |
|
"loss": 2.7506, |
|
"theoretical_loss": 3.590541558177172, |
|
"tokens_seen": 1188823040 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.00029044117647058826, |
|
"loss": 2.7132, |
|
"theoretical_loss": 3.5901770256152403, |
|
"tokens_seen": 1190133760 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.00029020474749385285, |
|
"loss": 2.6936, |
|
"theoretical_loss": 3.589813006569945, |
|
"tokens_seen": 1191444480 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.00028996831851711745, |
|
"loss": 2.7217, |
|
"theoretical_loss": 3.589449499754164, |
|
"tokens_seen": 1192755200 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.0002897318895403821, |
|
"loss": 2.7581, |
|
"theoretical_loss": 3.5890865038854103, |
|
"tokens_seen": 1194065920 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.00028949546056364665, |
|
"loss": 2.7031, |
|
"theoretical_loss": 3.5887240176858115, |
|
"tokens_seen": 1195376640 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.0002892590315869113, |
|
"loss": 2.7189, |
|
"theoretical_loss": 3.588362039882088, |
|
"tokens_seen": 1196687360 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.00028902260261017595, |
|
"loss": 2.7182, |
|
"theoretical_loss": 3.5880005692055317, |
|
"tokens_seen": 1197998080 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.0002887861736334405, |
|
"loss": 2.6783, |
|
"theoretical_loss": 3.5876396043919834, |
|
"tokens_seen": 1199308800 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.00028854974465670515, |
|
"loss": 2.7205, |
|
"theoretical_loss": 3.587279144181813, |
|
"tokens_seen": 1200619520 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.00028831331567996974, |
|
"loss": 2.7329, |
|
"theoretical_loss": 3.586919187319897, |
|
"tokens_seen": 1201930240 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.00028807688670323434, |
|
"loss": 2.6913, |
|
"theoretical_loss": 3.5865597325555987, |
|
"tokens_seen": 1203240960 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.000287840457726499, |
|
"loss": 2.7401, |
|
"theoretical_loss": 3.5862007786427474, |
|
"tokens_seen": 1204551680 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.0002876040287497636, |
|
"loss": 2.7276, |
|
"theoretical_loss": 3.585842324339616, |
|
"tokens_seen": 1205862400 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.0002873675997730282, |
|
"loss": 2.7041, |
|
"theoretical_loss": 3.5854843684089017, |
|
"tokens_seen": 1207173120 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.0002871311707962928, |
|
"loss": 2.7244, |
|
"theoretical_loss": 3.5851269096177054, |
|
"tokens_seen": 1208483840 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.00028689474181955744, |
|
"loss": 2.721, |
|
"theoretical_loss": 3.5847699467375103, |
|
"tokens_seen": 1209794560 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.00028665831284282203, |
|
"loss": 2.7419, |
|
"theoretical_loss": 3.584413478544163, |
|
"tokens_seen": 1211105280 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.00028642188386608663, |
|
"loss": 2.729, |
|
"theoretical_loss": 3.584057503817852, |
|
"tokens_seen": 1212416000 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.0002861854548893513, |
|
"loss": 2.6775, |
|
"theoretical_loss": 3.5837020213430884, |
|
"tokens_seen": 1213726720 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.0002859490259126158, |
|
"loss": 2.7225, |
|
"theoretical_loss": 3.583347029908686, |
|
"tokens_seen": 1215037440 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.0002857125969358805, |
|
"loss": 2.7179, |
|
"theoretical_loss": 3.582992528307741, |
|
"tokens_seen": 1216348160 |
|
}, |
|
{ |
|
"epoch": 0.43, |
|
"learning_rate": 0.00028547616795914513, |
|
"loss": 2.7152, |
|
"theoretical_loss": 3.5826385153376124, |
|
"tokens_seen": 1217658880 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00028523973898240967, |
|
"loss": 2.7084, |
|
"theoretical_loss": 3.582284989799902, |
|
"tokens_seen": 1218969600 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0002850033100056743, |
|
"loss": 2.7369, |
|
"theoretical_loss": 3.5819319505004366, |
|
"tokens_seen": 1220280320 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0002847668810289389, |
|
"loss": 2.7269, |
|
"theoretical_loss": 3.581579396249245, |
|
"tokens_seen": 1221591040 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0002845304520522035, |
|
"loss": 2.7058, |
|
"theoretical_loss": 3.5812273258605436, |
|
"tokens_seen": 1222901760 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00028429402307546817, |
|
"loss": 2.7345, |
|
"theoretical_loss": 3.580875738152713, |
|
"tokens_seen": 1224212480 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0002840575940987327, |
|
"loss": 2.7566, |
|
"theoretical_loss": 3.5805246319482817, |
|
"tokens_seen": 1225523200 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00028382116512199737, |
|
"loss": 2.7297, |
|
"theoretical_loss": 3.5801740060739036, |
|
"tokens_seen": 1226833920 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00028358473614526196, |
|
"loss": 2.6699, |
|
"theoretical_loss": 3.579823859360345, |
|
"tokens_seen": 1228144640 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00028334830716852656, |
|
"loss": 2.7415, |
|
"theoretical_loss": 3.5794741906424594, |
|
"tokens_seen": 1229455360 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00028311187819179116, |
|
"loss": 2.7704, |
|
"theoretical_loss": 3.5791249987591733, |
|
"tokens_seen": 1230766080 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0002828754492150558, |
|
"loss": 2.696, |
|
"theoretical_loss": 3.5787762825534664, |
|
"tokens_seen": 1232076800 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0002826390202383204, |
|
"loss": 2.7599, |
|
"theoretical_loss": 3.578428040872351, |
|
"tokens_seen": 1233387520 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.000282402591261585, |
|
"loss": 2.719, |
|
"theoretical_loss": 3.5780802725668575, |
|
"tokens_seen": 1234698240 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00028216616228484966, |
|
"loss": 2.7349, |
|
"theoretical_loss": 3.5777329764920136, |
|
"tokens_seen": 1236008960 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0002819297333081142, |
|
"loss": 2.7247, |
|
"theoretical_loss": 3.5773861515068273, |
|
"tokens_seen": 1237319680 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00028169330433137885, |
|
"loss": 2.7526, |
|
"theoretical_loss": 3.5770397964742675, |
|
"tokens_seen": 1238630400 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0002814568753546435, |
|
"loss": 2.6973, |
|
"theoretical_loss": 3.5766939102612474, |
|
"tokens_seen": 1239941120 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.00028122044637790805, |
|
"loss": 2.7114, |
|
"theoretical_loss": 3.5763484917386057, |
|
"tokens_seen": 1241251840 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0002809840174011727, |
|
"loss": 2.7062, |
|
"theoretical_loss": 3.576003539781091, |
|
"tokens_seen": 1242562560 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0002807475884244373, |
|
"loss": 2.7452, |
|
"theoretical_loss": 3.5756590532673407, |
|
"tokens_seen": 1243873280 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"objective/train/docs_used": 687081, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.7596964836120605, |
|
"objective/train/theoretical_loss": 3.5753150310798665, |
|
"objective/train/tokens_used": 1265644000, |
|
"theoretical_loss": 3.5753150310798665, |
|
"tokens_seen": 1245184000 |
|
}, |
|
{ |
|
"epoch": 0.44, |
|
"learning_rate": 0.0002805111594477019, |
|
"loss": 2.7577, |
|
"theoretical_loss": 3.5753150310798665, |
|
"tokens_seen": 1245184000 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00028027473047096654, |
|
"loss": 2.7681, |
|
"theoretical_loss": 3.5749714721050343, |
|
"tokens_seen": 1246494720 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00028003830149423114, |
|
"loss": 2.7865, |
|
"theoretical_loss": 3.5746283752330505, |
|
"tokens_seen": 1247805440 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00027980187251749574, |
|
"loss": 2.7529, |
|
"theoretical_loss": 3.574285739357941, |
|
"tokens_seen": 1249116160 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00027956544354076034, |
|
"loss": 2.8135, |
|
"theoretical_loss": 3.5739435633775365, |
|
"tokens_seen": 1250426880 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.000279329014564025, |
|
"loss": 2.7906, |
|
"theoretical_loss": 3.573601846193453, |
|
"tokens_seen": 1251737600 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.0002790925855872896, |
|
"loss": 2.7711, |
|
"theoretical_loss": 3.573260586711079, |
|
"tokens_seen": 1253048320 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.0002788561566105542, |
|
"loss": 2.7497, |
|
"theoretical_loss": 3.5729197838395543, |
|
"tokens_seen": 1254359040 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00027861972763381884, |
|
"loss": 2.746, |
|
"theoretical_loss": 3.5725794364917554, |
|
"tokens_seen": 1255669760 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.0002783832986570834, |
|
"loss": 2.7584, |
|
"theoretical_loss": 3.5722395435842778, |
|
"tokens_seen": 1256980480 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00027814686968034803, |
|
"loss": 2.6996, |
|
"theoretical_loss": 3.5719001040374216, |
|
"tokens_seen": 1258291200 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00027791044070361263, |
|
"loss": 2.7221, |
|
"theoretical_loss": 3.5715611167751717, |
|
"tokens_seen": 1259601920 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.0002776740117268772, |
|
"loss": 2.7164, |
|
"theoretical_loss": 3.571222580725184, |
|
"tokens_seen": 1260912640 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.0002774375827501419, |
|
"loss": 2.6775, |
|
"theoretical_loss": 3.5708844948187686, |
|
"tokens_seen": 1262223360 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.0002772011537734065, |
|
"loss": 2.7207, |
|
"theoretical_loss": 3.5705468579908715, |
|
"tokens_seen": 1263534080 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00027696472479667107, |
|
"loss": 2.7102, |
|
"theoretical_loss": 3.5702096691800627, |
|
"tokens_seen": 1264844800 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00027672829581993567, |
|
"loss": 2.7278, |
|
"theoretical_loss": 3.5698729273285146, |
|
"tokens_seen": 1266155520 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.0002764918668432003, |
|
"loss": 2.6994, |
|
"theoretical_loss": 3.5695366313819914, |
|
"tokens_seen": 1267466240 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.0002762554378664649, |
|
"loss": 2.6967, |
|
"theoretical_loss": 3.56920078028983, |
|
"tokens_seen": 1268776960 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.0002760190088897295, |
|
"loss": 2.6664, |
|
"theoretical_loss": 3.5688653730049245, |
|
"tokens_seen": 1270087680 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.00027578257991299417, |
|
"loss": 2.6885, |
|
"theoretical_loss": 3.5685304084837126, |
|
"tokens_seen": 1271398400 |
|
}, |
|
{ |
|
"epoch": 0.45, |
|
"learning_rate": 0.0002755461509362587, |
|
"loss": 2.7197, |
|
"theoretical_loss": 3.5681958856861558, |
|
"tokens_seen": 1272709120 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00027530972195952336, |
|
"loss": 2.7167, |
|
"theoretical_loss": 3.5678618035757297, |
|
"tokens_seen": 1274019840 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.000275073292982788, |
|
"loss": 2.695, |
|
"theoretical_loss": 3.5675281611194025, |
|
"tokens_seen": 1275330560 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00027483686400605256, |
|
"loss": 2.711, |
|
"theoretical_loss": 3.5671949572876245, |
|
"tokens_seen": 1276641280 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0002746004350293172, |
|
"loss": 2.6874, |
|
"theoretical_loss": 3.5668621910543097, |
|
"tokens_seen": 1277952000 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0002743640060525818, |
|
"loss": 2.644, |
|
"theoretical_loss": 3.566529861396822, |
|
"tokens_seen": 1279262720 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0002741275770758464, |
|
"loss": 2.6857, |
|
"theoretical_loss": 3.566197967295959, |
|
"tokens_seen": 1280573440 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00027389114809911106, |
|
"loss": 2.6703, |
|
"theoretical_loss": 3.56586650773594, |
|
"tokens_seen": 1281884160 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00027365471912237565, |
|
"loss": 2.6677, |
|
"theoretical_loss": 3.5655354817043863, |
|
"tokens_seen": 1283194880 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00027341829014564025, |
|
"loss": 2.7047, |
|
"theoretical_loss": 3.5652048881923095, |
|
"tokens_seen": 1284505600 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00027318186116890485, |
|
"loss": 2.6536, |
|
"theoretical_loss": 3.5648747261940974, |
|
"tokens_seen": 1285816320 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0002729454321921695, |
|
"loss": 2.6367, |
|
"theoretical_loss": 3.5645449947074956, |
|
"tokens_seen": 1287127040 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00027270900321543404, |
|
"loss": 2.6546, |
|
"theoretical_loss": 3.5642156927335975, |
|
"tokens_seen": 1288437760 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0002724725742386987, |
|
"loss": 2.6724, |
|
"theoretical_loss": 3.563886819276826, |
|
"tokens_seen": 1289748480 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00027223614526196335, |
|
"loss": 2.6625, |
|
"theoretical_loss": 3.5635583733449203, |
|
"tokens_seen": 1291059200 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0002719997162852279, |
|
"loss": 2.6636, |
|
"theoretical_loss": 3.5632303539489225, |
|
"tokens_seen": 1292369920 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00027176328730849254, |
|
"loss": 2.6695, |
|
"theoretical_loss": 3.562902760103161, |
|
"tokens_seen": 1293680640 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00027152685833175714, |
|
"loss": 2.6111, |
|
"theoretical_loss": 3.56257559082524, |
|
"tokens_seen": 1294991360 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.00027129042935502174, |
|
"loss": 2.7303, |
|
"theoretical_loss": 3.562248845136021, |
|
"tokens_seen": 1296302080 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0002710540003782864, |
|
"loss": 2.6753, |
|
"theoretical_loss": 3.5619225220596116, |
|
"tokens_seen": 1297612800 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.000270817571401551, |
|
"loss": 2.7305, |
|
"theoretical_loss": 3.56159662062335, |
|
"tokens_seen": 1298923520 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0002705811424248156, |
|
"loss": 2.6723, |
|
"theoretical_loss": 3.5612711398577916, |
|
"tokens_seen": 1300234240 |
|
}, |
|
{ |
|
"epoch": 0.46, |
|
"learning_rate": 0.0002703447134480802, |
|
"loss": 2.6636, |
|
"theoretical_loss": 3.5609460787966962, |
|
"tokens_seen": 1301544960 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00027010828447134483, |
|
"loss": 2.6819, |
|
"theoretical_loss": 3.560621436477012, |
|
"tokens_seen": 1302855680 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00026987185549460943, |
|
"loss": 2.6488, |
|
"theoretical_loss": 3.560297211938863, |
|
"tokens_seen": 1304166400 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00026963542651787403, |
|
"loss": 2.6952, |
|
"theoretical_loss": 3.5599734042255373, |
|
"tokens_seen": 1305477120 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.0002693989975411387, |
|
"loss": 2.693, |
|
"theoretical_loss": 3.559650012383469, |
|
"tokens_seen": 1306787840 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.0002691625685644032, |
|
"loss": 2.6732, |
|
"theoretical_loss": 3.559327035462228, |
|
"tokens_seen": 1308098560 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.0002689261395876679, |
|
"loss": 2.6645, |
|
"theoretical_loss": 3.5590044725145074, |
|
"tokens_seen": 1309409280 |
|
}, |
|
{ |
|
"debugging/Self-BLEU-5": 0.5616705212962695, |
|
"debugging/distinct-1-grams": 0.796841979490702, |
|
"debugging/distinct-2-grams": 0.9743537545539217, |
|
"debugging/entropy-1-grams": 6.281224573287751, |
|
"debugging/entropy-2-grams": 7.318801977908639, |
|
"debugging/length": 460.25, |
|
"debugging/num_segments": 24, |
|
"epoch": 0.47, |
|
"objective/train/docs_used": 722210, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 3.034902572631836, |
|
"objective/train/theoretical_loss": 3.5586823225961064, |
|
"objective/train/tokens_used": 1331180000, |
|
"theoretical_loss": 3.5586823225961064, |
|
"tokens_seen": 1310720000 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.0002686897106109325, |
|
"loss": 2.6237, |
|
"theoretical_loss": 3.5586823225961064, |
|
"tokens_seen": 1310720000 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00026845328163419707, |
|
"loss": 2.6581, |
|
"theoretical_loss": 3.5583605847659214, |
|
"tokens_seen": 1312030720 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.0002682168526574617, |
|
"loss": 2.6362, |
|
"theoretical_loss": 3.558039258085929, |
|
"tokens_seen": 1313341440 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.0002679804236807263, |
|
"loss": 2.7213, |
|
"theoretical_loss": 3.5577183416211744, |
|
"tokens_seen": 1314652160 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.0002677439947039909, |
|
"loss": 2.6685, |
|
"theoretical_loss": 3.5573978344397594, |
|
"tokens_seen": 1315962880 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00026750756572725557, |
|
"loss": 2.6144, |
|
"theoretical_loss": 3.5570777356128276, |
|
"tokens_seen": 1317273600 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00026727113675052016, |
|
"loss": 2.6287, |
|
"theoretical_loss": 3.556758044214552, |
|
"tokens_seen": 1318584320 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00026703470777378476, |
|
"loss": 2.6641, |
|
"theoretical_loss": 3.5564387593221225, |
|
"tokens_seen": 1319895040 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00026679827879704936, |
|
"loss": 2.6217, |
|
"theoretical_loss": 3.556119880015733, |
|
"tokens_seen": 1321205760 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.000266561849820314, |
|
"loss": 2.6583, |
|
"theoretical_loss": 3.5558014053785687, |
|
"tokens_seen": 1322516480 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00026632542084357855, |
|
"loss": 2.647, |
|
"theoretical_loss": 3.5554833344967927, |
|
"tokens_seen": 1323827200 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.0002660889918668432, |
|
"loss": 2.6288, |
|
"theoretical_loss": 3.5551656664595344, |
|
"tokens_seen": 1325137920 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00026585256289010786, |
|
"loss": 2.619, |
|
"theoretical_loss": 3.5548484003588756, |
|
"tokens_seen": 1326448640 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.0002656161339133724, |
|
"loss": 2.6547, |
|
"theoretical_loss": 3.554531535289841, |
|
"tokens_seen": 1327759360 |
|
}, |
|
{ |
|
"epoch": 0.47, |
|
"learning_rate": 0.00026537970493663705, |
|
"loss": 2.678, |
|
"theoretical_loss": 3.5542150703503808, |
|
"tokens_seen": 1329070080 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026514327595990165, |
|
"loss": 2.6858, |
|
"theoretical_loss": 3.553899004641363, |
|
"tokens_seen": 1330380800 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026490684698316625, |
|
"loss": 2.6943, |
|
"theoretical_loss": 3.55358333726656, |
|
"tokens_seen": 1331691520 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.0002646704180064309, |
|
"loss": 2.6441, |
|
"theoretical_loss": 3.553268067332634, |
|
"tokens_seen": 1333002240 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.0002644339890296955, |
|
"loss": 2.679, |
|
"theoretical_loss": 3.5529531939491275, |
|
"tokens_seen": 1334312960 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.0002641975600529601, |
|
"loss": 2.6601, |
|
"theoretical_loss": 3.552638716228449, |
|
"tokens_seen": 1335623680 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.0002639611310762247, |
|
"loss": 2.6693, |
|
"theoretical_loss": 3.552324633285865, |
|
"tokens_seen": 1336934400 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026372470209948934, |
|
"loss": 2.6284, |
|
"theoretical_loss": 3.5520109442394827, |
|
"tokens_seen": 1338245120 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026348827312275394, |
|
"loss": 2.6787, |
|
"theoretical_loss": 3.5516976482102414, |
|
"tokens_seen": 1339555840 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026325184414601854, |
|
"loss": 2.7308, |
|
"theoretical_loss": 3.5513847443218998, |
|
"tokens_seen": 1340866560 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.0002630154151692832, |
|
"loss": 2.6488, |
|
"theoretical_loss": 3.551072231701025, |
|
"tokens_seen": 1342177280 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026277898619254773, |
|
"loss": 2.6368, |
|
"theoretical_loss": 3.550760109476979, |
|
"tokens_seen": 1343488000 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.0002625425572158124, |
|
"loss": 2.659, |
|
"theoretical_loss": 3.550448376781909, |
|
"tokens_seen": 1344798720 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026230612823907704, |
|
"loss": 2.6445, |
|
"theoretical_loss": 3.550137032750734, |
|
"tokens_seen": 1346109440 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.0002620696992623416, |
|
"loss": 2.6342, |
|
"theoretical_loss": 3.5498260765211356, |
|
"tokens_seen": 1347420160 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026183327028560623, |
|
"loss": 2.697, |
|
"theoretical_loss": 3.549515507233543, |
|
"tokens_seen": 1348730880 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026159684130887083, |
|
"loss": 2.7078, |
|
"theoretical_loss": 3.5492053240311257, |
|
"tokens_seen": 1350041600 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.0002613604123321354, |
|
"loss": 2.6624, |
|
"theoretical_loss": 3.5488955260597783, |
|
"tokens_seen": 1351352320 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.0002611239833554001, |
|
"loss": 2.7078, |
|
"theoretical_loss": 3.5485861124681124, |
|
"tokens_seen": 1352663040 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.0002608875543786647, |
|
"loss": 2.6611, |
|
"theoretical_loss": 3.5482770824074428, |
|
"tokens_seen": 1353973760 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.0002606511254019293, |
|
"loss": 2.6832, |
|
"theoretical_loss": 3.5479684350317777, |
|
"tokens_seen": 1355284480 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026041469642519387, |
|
"loss": 2.7189, |
|
"theoretical_loss": 3.5476601694978083, |
|
"tokens_seen": 1356595200 |
|
}, |
|
{ |
|
"epoch": 0.48, |
|
"learning_rate": 0.00026017826744845847, |
|
"loss": 2.6972, |
|
"theoretical_loss": 3.5473522849648944, |
|
"tokens_seen": 1357905920 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00025994183847172307, |
|
"loss": 2.6848, |
|
"theoretical_loss": 3.547044780595058, |
|
"tokens_seen": 1359216640 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.0002597054094949877, |
|
"loss": 2.6864, |
|
"theoretical_loss": 3.5467376555529686, |
|
"tokens_seen": 1360527360 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.0002594689805182523, |
|
"loss": 2.6594, |
|
"theoretical_loss": 3.546430909005934, |
|
"tokens_seen": 1361838080 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.0002592325515415169, |
|
"loss": 2.6559, |
|
"theoretical_loss": 3.5461245401238894, |
|
"tokens_seen": 1363148800 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00025899612256478156, |
|
"loss": 2.6862, |
|
"theoretical_loss": 3.545818548079385, |
|
"tokens_seen": 1364459520 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.0002587596935880461, |
|
"loss": 2.6839, |
|
"theoretical_loss": 3.545512932047579, |
|
"tokens_seen": 1365770240 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00025852326461131076, |
|
"loss": 2.667, |
|
"theoretical_loss": 3.5452076912062216, |
|
"tokens_seen": 1367080960 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.0002582868356345754, |
|
"loss": 2.7063, |
|
"theoretical_loss": 3.54490282473565, |
|
"tokens_seen": 1368391680 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00025805040665783995, |
|
"loss": 2.6548, |
|
"theoretical_loss": 3.544598331818772, |
|
"tokens_seen": 1369702400 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.0002578139776811046, |
|
"loss": 2.6861, |
|
"theoretical_loss": 3.5442942116410614, |
|
"tokens_seen": 1371013120 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.0002575775487043692, |
|
"loss": 2.657, |
|
"theoretical_loss": 3.5439904633905424, |
|
"tokens_seen": 1372323840 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.0002573411197276338, |
|
"loss": 2.6873, |
|
"theoretical_loss": 3.5436870862577825, |
|
"tokens_seen": 1373634560 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00025710469075089845, |
|
"loss": 2.6745, |
|
"theoretical_loss": 3.5433840794358806, |
|
"tokens_seen": 1374945280 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"objective/train/docs_used": 759843, |
|
"objective/train/instantaneous_batch_size": 32, |
|
"objective/train/instantaneous_microbatch_size": 32768, |
|
"objective/train/original_loss": 2.5367300510406494, |
|
"objective/train/theoretical_loss": 3.5430814421204575, |
|
"objective/train/tokens_used": 1396716000, |
|
"theoretical_loss": 3.5430814421204575, |
|
"tokens_seen": 1376256000 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00025686826177416305, |
|
"loss": 2.6763, |
|
"theoretical_loss": 3.5430814421204575, |
|
"tokens_seen": 1376256000 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00025663183279742765, |
|
"loss": 2.7077, |
|
"theoretical_loss": 3.5427791735096448, |
|
"tokens_seen": 1377566720 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00025639540382069225, |
|
"loss": 2.6516, |
|
"theoretical_loss": 3.542477272804075, |
|
"tokens_seen": 1378877440 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.0002561589748439569, |
|
"loss": 2.6722, |
|
"theoretical_loss": 3.542175739206872, |
|
"tokens_seen": 1380188160 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.0002559225458672215, |
|
"loss": 2.6098, |
|
"theoretical_loss": 3.541874571923639, |
|
"tokens_seen": 1381498880 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.0002556861168904861, |
|
"loss": 2.6425, |
|
"theoretical_loss": 3.5415737701624517, |
|
"tokens_seen": 1382809600 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.00025544968791375074, |
|
"loss": 2.6648, |
|
"theoretical_loss": 3.5412733331338453, |
|
"tokens_seen": 1384120320 |
|
}, |
|
{ |
|
"epoch": 0.49, |
|
"learning_rate": 0.0002552132589370153, |
|
"loss": 2.6946, |
|
"theoretical_loss": 3.5409732600508055, |
|
"tokens_seen": 1385431040 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.00025497682996027994, |
|
"loss": 2.6316, |
|
"theoretical_loss": 3.5406735501287594, |
|
"tokens_seen": 1386741760 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.00025474040098354454, |
|
"loss": 2.6562, |
|
"theoretical_loss": 3.5403742025855647, |
|
"tokens_seen": 1388052480 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.00025450397200680913, |
|
"loss": 2.6769, |
|
"theoretical_loss": 3.5400752166415, |
|
"tokens_seen": 1389363200 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.0002542675430300738, |
|
"loss": 2.639, |
|
"theoretical_loss": 3.5397765915192547, |
|
"tokens_seen": 1390673920 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.0002540311140533384, |
|
"loss": 2.686, |
|
"theoretical_loss": 3.5394783264439207, |
|
"tokens_seen": 1391984640 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.000253794685076603, |
|
"loss": 2.6641, |
|
"theoretical_loss": 3.5391804206429818, |
|
"tokens_seen": 1393295360 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.0002535582560998676, |
|
"loss": 2.7026, |
|
"theoretical_loss": 3.538882873346303, |
|
"tokens_seen": 1394606080 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.00025332182712313223, |
|
"loss": 2.7189, |
|
"theoretical_loss": 3.5385856837861223, |
|
"tokens_seen": 1395916800 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.0002530853981463968, |
|
"loss": 2.7667, |
|
"theoretical_loss": 3.538288851197042, |
|
"tokens_seen": 1397227520 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.0002528489691696614, |
|
"loss": 2.7162, |
|
"theoretical_loss": 3.537992374816016, |
|
"tokens_seen": 1398538240 |
|
}, |
|
{ |
|
"epoch": 0.5, |
|
"learning_rate": 0.0002526125401929261, |
|
"loss": 2.6978, |
|
"theoretical_loss": 3.537696253882345, |
|
"tokens_seen": 1399848960 |
|
} |
|
], |
|
"max_steps": 42724, |
|
"num_train_epochs": 9223372036854775807, |
|
"total_flos": 7.14460209610752e+17, |
|
"trial_name": null, |
|
"trial_params": null |
|
} |
|
|