| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 3.0, |
| "eval_steps": 500, |
| "global_step": 1875, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "entropy": 1.5365671901141895, |
| "epoch": 0.0016, |
| "grad_norm": 4.21875, |
| "learning_rate": 0.0, |
| "loss": 5.494313716888428, |
| "mean_token_accuracy": 0.6573571685482474, |
| "num_tokens": 348701.0, |
| "step": 1 |
| }, |
| { |
| "entropy": 1.348903712299135, |
| "epoch": 0.016, |
| "grad_norm": 2.59375, |
| "learning_rate": 3.157894736842105e-05, |
| "loss": 5.544558631049262, |
| "mean_token_accuracy": 0.6651431231035126, |
| "num_tokens": 371496.0, |
| "step": 10 |
| }, |
| { |
| "entropy": 1.360134594142437, |
| "epoch": 0.032, |
| "grad_norm": 3.296875, |
| "learning_rate": 6.666666666666667e-05, |
| "loss": 5.742576599121094, |
| "mean_token_accuracy": 0.6662841200828552, |
| "num_tokens": 400839.0, |
| "step": 20 |
| }, |
| { |
| "entropy": 1.375771728157997, |
| "epoch": 0.048, |
| "grad_norm": 3.140625, |
| "learning_rate": 0.0001017543859649123, |
| "loss": 5.714028167724609, |
| "mean_token_accuracy": 0.6566085889935493, |
| "num_tokens": 431025.0, |
| "step": 30 |
| }, |
| { |
| "entropy": 1.2748979568481444, |
| "epoch": 0.064, |
| "grad_norm": 2.828125, |
| "learning_rate": 0.0001368421052631579, |
| "loss": 5.11612548828125, |
| "mean_token_accuracy": 0.6816287323832512, |
| "num_tokens": 459089.0, |
| "step": 40 |
| }, |
| { |
| "entropy": 1.3155348658561707, |
| "epoch": 0.08, |
| "grad_norm": 3.15625, |
| "learning_rate": 0.00017192982456140353, |
| "loss": 5.35040512084961, |
| "mean_token_accuracy": 0.6721501812338829, |
| "num_tokens": 484648.0, |
| "step": 50 |
| }, |
| { |
| "entropy": 1.1791150525212288, |
| "epoch": 0.096, |
| "grad_norm": 2.375, |
| "learning_rate": 0.00019999940277008808, |
| "loss": 4.642951583862304, |
| "mean_token_accuracy": 0.7028968930244446, |
| "num_tokens": 511463.0, |
| "step": 60 |
| }, |
| { |
| "entropy": 1.1177749201655387, |
| "epoch": 0.112, |
| "grad_norm": 2.421875, |
| "learning_rate": 0.0001999785004721968, |
| "loss": 4.523546981811523, |
| "mean_token_accuracy": 0.7057401552796364, |
| "num_tokens": 538438.0, |
| "step": 70 |
| }, |
| { |
| "entropy": 1.1784485399723053, |
| "epoch": 0.128, |
| "grad_norm": 2.25, |
| "learning_rate": 0.00019992774381199778, |
| "loss": 4.712202072143555, |
| "mean_token_accuracy": 0.7004493281245232, |
| "num_tokens": 569421.0, |
| "step": 80 |
| }, |
| { |
| "entropy": 1.0666756927967072, |
| "epoch": 0.144, |
| "grad_norm": 4.25, |
| "learning_rate": 0.00019984714794582683, |
| "loss": 4.181539154052734, |
| "mean_token_accuracy": 0.7218294009566307, |
| "num_tokens": 595806.0, |
| "step": 90 |
| }, |
| { |
| "entropy": 1.1410984337329864, |
| "epoch": 0.16, |
| "grad_norm": 2.59375, |
| "learning_rate": 0.00019973673694024, |
| "loss": 4.578577041625977, |
| "mean_token_accuracy": 0.70914496332407, |
| "num_tokens": 626238.0, |
| "step": 100 |
| }, |
| { |
| "entropy": 1.058976437151432, |
| "epoch": 0.176, |
| "grad_norm": 4.34375, |
| "learning_rate": 0.0001995965437648273, |
| "loss": 4.2924964904785154, |
| "mean_token_accuracy": 0.7180401891469955, |
| "num_tokens": 653441.0, |
| "step": 110 |
| }, |
| { |
| "entropy": 1.2249037250876427, |
| "epoch": 0.192, |
| "grad_norm": 3.96875, |
| "learning_rate": 0.00019942661028236745, |
| "loss": 5.084912490844727, |
| "mean_token_accuracy": 0.6847136527299881, |
| "num_tokens": 682582.0, |
| "step": 120 |
| }, |
| { |
| "entropy": 1.1726351112127305, |
| "epoch": 0.208, |
| "grad_norm": 3.296875, |
| "learning_rate": 0.00019922698723632767, |
| "loss": 5.018433380126953, |
| "mean_token_accuracy": 0.7055773109197616, |
| "num_tokens": 710468.0, |
| "step": 130 |
| }, |
| { |
| "entropy": 1.3469643473625184, |
| "epoch": 0.224, |
| "grad_norm": 2.875, |
| "learning_rate": 0.000198997734235711, |
| "loss": 5.7476848602294925, |
| "mean_token_accuracy": 0.6790369123220443, |
| "num_tokens": 740979.0, |
| "step": 140 |
| }, |
| { |
| "entropy": 1.187085197865963, |
| "epoch": 0.24, |
| "grad_norm": 2.359375, |
| "learning_rate": 0.0001987389197372567, |
| "loss": 5.085608673095703, |
| "mean_token_accuracy": 0.7057169727981091, |
| "num_tokens": 768427.0, |
| "step": 150 |
| }, |
| { |
| "entropy": 1.2938067495822907, |
| "epoch": 0.256, |
| "grad_norm": 2.359375, |
| "learning_rate": 0.0001984506210249986, |
| "loss": 5.570721054077149, |
| "mean_token_accuracy": 0.6840770006179809, |
| "num_tokens": 796744.0, |
| "step": 160 |
| }, |
| { |
| "entropy": 1.1161220327019692, |
| "epoch": 0.272, |
| "grad_norm": 2.265625, |
| "learning_rate": 0.00019813292418718732, |
| "loss": 4.821302795410157, |
| "mean_token_accuracy": 0.7168306574225426, |
| "num_tokens": 822225.0, |
| "step": 170 |
| }, |
| { |
| "entropy": 1.2217196941375732, |
| "epoch": 0.288, |
| "grad_norm": 2.484375, |
| "learning_rate": 0.00019778592409058378, |
| "loss": 5.171672821044922, |
| "mean_token_accuracy": 0.6998881995677948, |
| "num_tokens": 848049.0, |
| "step": 180 |
| }, |
| { |
| "entropy": 1.1091715931892394, |
| "epoch": 0.304, |
| "grad_norm": 3.109375, |
| "learning_rate": 0.00019740972435213115, |
| "loss": 4.743664169311524, |
| "mean_token_accuracy": 0.7203769966959953, |
| "num_tokens": 873172.0, |
| "step": 190 |
| }, |
| { |
| "entropy": 1.2154748886823654, |
| "epoch": 0.32, |
| "grad_norm": 2.21875, |
| "learning_rate": 0.00019700443730801413, |
| "loss": 5.212123107910156, |
| "mean_token_accuracy": 0.7076091229915619, |
| "num_tokens": 901716.0, |
| "step": 200 |
| }, |
| { |
| "entropy": 1.161145742237568, |
| "epoch": 0.336, |
| "grad_norm": 2.078125, |
| "learning_rate": 0.00019657018398011434, |
| "loss": 4.889891052246094, |
| "mean_token_accuracy": 0.7145821407437325, |
| "num_tokens": 928506.0, |
| "step": 210 |
| }, |
| { |
| "entropy": 1.1209837168455123, |
| "epoch": 0.352, |
| "grad_norm": 2.09375, |
| "learning_rate": 0.00019610709403987246, |
| "loss": 4.862021636962891, |
| "mean_token_accuracy": 0.7152845054864884, |
| "num_tokens": 958723.0, |
| "step": 220 |
| }, |
| { |
| "entropy": 1.1322738587856294, |
| "epoch": 0.368, |
| "grad_norm": 2.203125, |
| "learning_rate": 0.00019561530576956703, |
| "loss": 4.911846923828125, |
| "mean_token_accuracy": 0.7134342208504677, |
| "num_tokens": 987751.0, |
| "step": 230 |
| }, |
| { |
| "entropy": 1.1780440196394921, |
| "epoch": 0.384, |
| "grad_norm": 2.421875, |
| "learning_rate": 0.00019509496602102252, |
| "loss": 5.0671440124511715, |
| "mean_token_accuracy": 0.7104542285203934, |
| "num_tokens": 1018315.0, |
| "step": 240 |
| }, |
| { |
| "entropy": 1.1776200592517854, |
| "epoch": 0.4, |
| "grad_norm": 2.359375, |
| "learning_rate": 0.00019454623017175812, |
| "loss": 5.145317459106446, |
| "mean_token_accuracy": 0.7086253471672534, |
| "num_tokens": 1046627.0, |
| "step": 250 |
| }, |
| { |
| "entropy": 1.1513659209012985, |
| "epoch": 0.416, |
| "grad_norm": 2.140625, |
| "learning_rate": 0.00019396926207859084, |
| "loss": 4.8750255584716795, |
| "mean_token_accuracy": 0.7203947469592095, |
| "num_tokens": 1075027.0, |
| "step": 260 |
| }, |
| { |
| "entropy": 1.1364729449152946, |
| "epoch": 0.432, |
| "grad_norm": 2.265625, |
| "learning_rate": 0.00019336423402870653, |
| "loss": 4.866281890869141, |
| "mean_token_accuracy": 0.7150357633829116, |
| "num_tokens": 1102334.0, |
| "step": 270 |
| }, |
| { |
| "entropy": 1.108440762758255, |
| "epoch": 0.448, |
| "grad_norm": 1.9453125, |
| "learning_rate": 0.00019273132668821364, |
| "loss": 4.697020721435547, |
| "mean_token_accuracy": 0.7265092357993126, |
| "num_tokens": 1131415.0, |
| "step": 280 |
| }, |
| { |
| "entropy": 1.1001321360468865, |
| "epoch": 0.464, |
| "grad_norm": 2.125, |
| "learning_rate": 0.00019207072904819486, |
| "loss": 4.747102355957031, |
| "mean_token_accuracy": 0.7231316924095154, |
| "num_tokens": 1158280.0, |
| "step": 290 |
| }, |
| { |
| "entropy": 1.1048941344022751, |
| "epoch": 0.48, |
| "grad_norm": 2.234375, |
| "learning_rate": 0.00019138263836827288, |
| "loss": 4.644550323486328, |
| "mean_token_accuracy": 0.7270571634173393, |
| "num_tokens": 1188651.0, |
| "step": 300 |
| }, |
| { |
| "entropy": 1.0945911705493927, |
| "epoch": 0.496, |
| "grad_norm": 1.984375, |
| "learning_rate": 0.00019066726011770726, |
| "loss": 4.7493640899658205, |
| "mean_token_accuracy": 0.7258608743548394, |
| "num_tokens": 1216504.0, |
| "step": 310 |
| }, |
| { |
| "entropy": 1.0356923416256905, |
| "epoch": 0.512, |
| "grad_norm": 2.1875, |
| "learning_rate": 0.00018992480791403958, |
| "loss": 4.3843944549560545, |
| "mean_token_accuracy": 0.7372462660074234, |
| "num_tokens": 1243732.0, |
| "step": 320 |
| }, |
| { |
| "entropy": 1.0773125097155571, |
| "epoch": 0.528, |
| "grad_norm": 2.328125, |
| "learning_rate": 0.0001891555034593055, |
| "loss": 4.6180767059326175, |
| "mean_token_accuracy": 0.7283408164978027, |
| "num_tokens": 1269272.0, |
| "step": 330 |
| }, |
| { |
| "entropy": 1.050987295806408, |
| "epoch": 0.544, |
| "grad_norm": 1.640625, |
| "learning_rate": 0.00018835957647383303, |
| "loss": 4.4863533020019535, |
| "mean_token_accuracy": 0.7359548002481461, |
| "num_tokens": 1296001.0, |
| "step": 340 |
| }, |
| { |
| "entropy": 1.0461898490786552, |
| "epoch": 0.56, |
| "grad_norm": 2.453125, |
| "learning_rate": 0.000187537264627646, |
| "loss": 4.434906005859375, |
| "mean_token_accuracy": 0.7349108412861824, |
| "num_tokens": 1322180.0, |
| "step": 350 |
| }, |
| { |
| "entropy": 1.0046504065394402, |
| "epoch": 0.576, |
| "grad_norm": 2.125, |
| "learning_rate": 0.00018668881346949417, |
| "loss": 4.309226989746094, |
| "mean_token_accuracy": 0.73991359770298, |
| "num_tokens": 1347532.0, |
| "step": 360 |
| }, |
| { |
| "entropy": 1.0861794739961623, |
| "epoch": 0.592, |
| "grad_norm": 1.890625, |
| "learning_rate": 0.0001858144763535302, |
| "loss": 4.599692535400391, |
| "mean_token_accuracy": 0.7285229310393333, |
| "num_tokens": 1373584.0, |
| "step": 370 |
| }, |
| { |
| "entropy": 1.0600647926330566, |
| "epoch": 0.608, |
| "grad_norm": 2.203125, |
| "learning_rate": 0.00018491451436365627, |
| "loss": 4.5491493225097654, |
| "mean_token_accuracy": 0.7357145607471466, |
| "num_tokens": 1399594.0, |
| "step": 380 |
| }, |
| { |
| "entropy": 1.098224511742592, |
| "epoch": 0.624, |
| "grad_norm": 1.9296875, |
| "learning_rate": 0.00018398919623556238, |
| "loss": 4.8250572204589846, |
| "mean_token_accuracy": 0.7251888766884804, |
| "num_tokens": 1428792.0, |
| "step": 390 |
| }, |
| { |
| "entropy": 1.1232757449150086, |
| "epoch": 0.64, |
| "grad_norm": 2.03125, |
| "learning_rate": 0.00018303879827647975, |
| "loss": 4.712010192871094, |
| "mean_token_accuracy": 0.7287421196699142, |
| "num_tokens": 1457889.0, |
| "step": 400 |
| }, |
| { |
| "entropy": 1.0411028936505318, |
| "epoch": 0.656, |
| "grad_norm": 1.859375, |
| "learning_rate": 0.00018206360428267332, |
| "loss": 4.405958938598633, |
| "mean_token_accuracy": 0.7350577011704444, |
| "num_tokens": 1484479.0, |
| "step": 410 |
| }, |
| { |
| "entropy": 1.1146624743938447, |
| "epoch": 0.672, |
| "grad_norm": 1.734375, |
| "learning_rate": 0.00018106390545469795, |
| "loss": 4.742184448242187, |
| "mean_token_accuracy": 0.7282734125852585, |
| "num_tokens": 1514268.0, |
| "step": 420 |
| }, |
| { |
| "entropy": 1.0480840012431145, |
| "epoch": 0.688, |
| "grad_norm": 1.78125, |
| "learning_rate": 0.0001800400003104436, |
| "loss": 4.4207916259765625, |
| "mean_token_accuracy": 0.7365467861294747, |
| "num_tokens": 1540474.0, |
| "step": 430 |
| }, |
| { |
| "entropy": 1.0067586719989776, |
| "epoch": 0.704, |
| "grad_norm": 1.921875, |
| "learning_rate": 0.0001789921945959958, |
| "loss": 4.31513671875, |
| "mean_token_accuracy": 0.7437789484858512, |
| "num_tokens": 1568557.0, |
| "step": 440 |
| }, |
| { |
| "entropy": 1.0166094586253167, |
| "epoch": 0.72, |
| "grad_norm": 1.6171875, |
| "learning_rate": 0.0001779208011943371, |
| "loss": 4.292739486694336, |
| "mean_token_accuracy": 0.7471857726573944, |
| "num_tokens": 1594403.0, |
| "step": 450 |
| }, |
| { |
| "entropy": 1.0385540708899499, |
| "epoch": 0.736, |
| "grad_norm": 2.75, |
| "learning_rate": 0.00017682614003191807, |
| "loss": 4.415458679199219, |
| "mean_token_accuracy": 0.7400641202926636, |
| "num_tokens": 1624111.0, |
| "step": 460 |
| }, |
| { |
| "entropy": 1.0706985503435136, |
| "epoch": 0.752, |
| "grad_norm": 2.265625, |
| "learning_rate": 0.0001757085379831246, |
| "loss": 4.554729080200195, |
| "mean_token_accuracy": 0.7342943042516709, |
| "num_tokens": 1652609.0, |
| "step": 470 |
| }, |
| { |
| "entropy": 0.9689434483647347, |
| "epoch": 0.768, |
| "grad_norm": 2.03125, |
| "learning_rate": 0.00017456832877267084, |
| "loss": 4.2303211212158205, |
| "mean_token_accuracy": 0.7474748462438583, |
| "num_tokens": 1678589.0, |
| "step": 480 |
| }, |
| { |
| "entropy": 1.1837652444839477, |
| "epoch": 0.784, |
| "grad_norm": 2.078125, |
| "learning_rate": 0.00017340585287594604, |
| "loss": 4.968061447143555, |
| "mean_token_accuracy": 0.7166377156972885, |
| "num_tokens": 1708513.0, |
| "step": 490 |
| }, |
| { |
| "entropy": 1.042286379635334, |
| "epoch": 0.8, |
| "grad_norm": 2.515625, |
| "learning_rate": 0.00017222145741734626, |
| "loss": 4.422880172729492, |
| "mean_token_accuracy": 0.7405225187540054, |
| "num_tokens": 1736283.0, |
| "step": 500 |
| }, |
| { |
| "entropy": 0.9832519263029098, |
| "epoch": 0.816, |
| "grad_norm": 1.6171875, |
| "learning_rate": 0.00017101549606662024, |
| "loss": 4.119276428222657, |
| "mean_token_accuracy": 0.7495438739657402, |
| "num_tokens": 1765017.0, |
| "step": 510 |
| }, |
| { |
| "entropy": 0.9872105062007904, |
| "epoch": 0.832, |
| "grad_norm": 1.71875, |
| "learning_rate": 0.00016978832893326074, |
| "loss": 4.234106826782226, |
| "mean_token_accuracy": 0.7473902180790901, |
| "num_tokens": 1790682.0, |
| "step": 520 |
| }, |
| { |
| "entropy": 0.9845283895730972, |
| "epoch": 0.848, |
| "grad_norm": 1.4921875, |
| "learning_rate": 0.00016854032245897308, |
| "loss": 4.146430969238281, |
| "mean_token_accuracy": 0.7520387843251228, |
| "num_tokens": 1820050.0, |
| "step": 530 |
| }, |
| { |
| "entropy": 1.0613365799188614, |
| "epoch": 0.864, |
| "grad_norm": 2.046875, |
| "learning_rate": 0.00016727184930825288, |
| "loss": 4.48931655883789, |
| "mean_token_accuracy": 0.7396015107631684, |
| "num_tokens": 1847079.0, |
| "step": 540 |
| }, |
| { |
| "entropy": 0.9684726029634476, |
| "epoch": 0.88, |
| "grad_norm": 2.03125, |
| "learning_rate": 0.00016598328825710533, |
| "loss": 4.12475357055664, |
| "mean_token_accuracy": 0.7526269048452378, |
| "num_tokens": 1872383.0, |
| "step": 550 |
| }, |
| { |
| "entropy": 0.9990086019039154, |
| "epoch": 0.896, |
| "grad_norm": 1.8984375, |
| "learning_rate": 0.00016467502407993992, |
| "loss": 4.360863494873047, |
| "mean_token_accuracy": 0.7387492001056671, |
| "num_tokens": 1903630.0, |
| "step": 560 |
| }, |
| { |
| "entropy": 0.9708257809281349, |
| "epoch": 0.912, |
| "grad_norm": 1.6328125, |
| "learning_rate": 0.00016334744743467364, |
| "loss": 4.054442596435547, |
| "mean_token_accuracy": 0.7545965671539306, |
| "num_tokens": 1930592.0, |
| "step": 570 |
| }, |
| { |
| "entropy": 1.0035204842686654, |
| "epoch": 0.928, |
| "grad_norm": 1.8984375, |
| "learning_rate": 0.00016200095474607753, |
| "loss": 4.25194206237793, |
| "mean_token_accuracy": 0.7464351058006287, |
| "num_tokens": 1960212.0, |
| "step": 580 |
| }, |
| { |
| "entropy": 0.9635734960436821, |
| "epoch": 0.944, |
| "grad_norm": 1.53125, |
| "learning_rate": 0.00016063594808740113, |
| "loss": 4.0587310791015625, |
| "mean_token_accuracy": 0.7547481089830399, |
| "num_tokens": 1986049.0, |
| "step": 590 |
| }, |
| { |
| "entropy": 0.9986386775970459, |
| "epoch": 0.96, |
| "grad_norm": 1.703125, |
| "learning_rate": 0.0001592528350603103, |
| "loss": 4.331460952758789, |
| "mean_token_accuracy": 0.7503352165222168, |
| "num_tokens": 2013266.0, |
| "step": 600 |
| }, |
| { |
| "entropy": 1.006168755888939, |
| "epoch": 0.976, |
| "grad_norm": 1.734375, |
| "learning_rate": 0.00015785202867317407, |
| "loss": 4.238505554199219, |
| "mean_token_accuracy": 0.7441465124487877, |
| "num_tokens": 2041824.0, |
| "step": 610 |
| }, |
| { |
| "entropy": 1.0482723653316497, |
| "epoch": 0.992, |
| "grad_norm": 1.9296875, |
| "learning_rate": 0.0001564339472177373, |
| "loss": 4.394336700439453, |
| "mean_token_accuracy": 0.7416493371129036, |
| "num_tokens": 2071693.0, |
| "step": 620 |
| }, |
| { |
| "epoch": 1.0, |
| "eval_entropy": 1.0188530464172363, |
| "eval_loss": 1.0276753902435303, |
| "eval_mean_token_accuracy": 0.7517141411304474, |
| "eval_num_tokens": 2084150.0, |
| "eval_runtime": 72.9791, |
| "eval_samples_per_second": 13.703, |
| "eval_steps_per_second": 3.426, |
| "step": 625 |
| }, |
| { |
| "entropy": 0.9583894208073616, |
| "epoch": 1.008, |
| "grad_norm": 1.40625, |
| "learning_rate": 0.0001549990141442153, |
| "loss": 3.791181945800781, |
| "mean_token_accuracy": 0.760094690322876, |
| "num_tokens": 2099483.0, |
| "step": 630 |
| }, |
| { |
| "entropy": 0.7975272431969642, |
| "epoch": 1.024, |
| "grad_norm": 1.390625, |
| "learning_rate": 0.00015354765793484834, |
| "loss": 3.2353271484375, |
| "mean_token_accuracy": 0.7800843045115471, |
| "num_tokens": 2127344.0, |
| "step": 640 |
| }, |
| { |
| "entropy": 0.8557936266064644, |
| "epoch": 1.04, |
| "grad_norm": 1.921875, |
| "learning_rate": 0.00015208031197595356, |
| "loss": 3.591104507446289, |
| "mean_token_accuracy": 0.769377326965332, |
| "num_tokens": 2157286.0, |
| "step": 650 |
| }, |
| { |
| "entropy": 0.8815567880868912, |
| "epoch": 1.056, |
| "grad_norm": 1.4765625, |
| "learning_rate": 0.0001505974144285124, |
| "loss": 3.509926986694336, |
| "mean_token_accuracy": 0.7711383983492851, |
| "num_tokens": 2187199.0, |
| "step": 660 |
| }, |
| { |
| "entropy": 0.7711537912487983, |
| "epoch": 1.072, |
| "grad_norm": 1.3828125, |
| "learning_rate": 0.00014909940809733222, |
| "loss": 3.108415412902832, |
| "mean_token_accuracy": 0.7885575816035271, |
| "num_tokens": 2213742.0, |
| "step": 670 |
| }, |
| { |
| "entropy": 0.8327512949705124, |
| "epoch": 1.088, |
| "grad_norm": 6.5625, |
| "learning_rate": 0.00014758674029882152, |
| "loss": 3.380535125732422, |
| "mean_token_accuracy": 0.7737385779619217, |
| "num_tokens": 2242448.0, |
| "step": 680 |
| }, |
| { |
| "entropy": 0.8324707329273224, |
| "epoch": 1.104, |
| "grad_norm": 1.3125, |
| "learning_rate": 0.00014605986272741748, |
| "loss": 3.3809837341308593, |
| "mean_token_accuracy": 0.7749153047800064, |
| "num_tokens": 2273149.0, |
| "step": 690 |
| }, |
| { |
| "entropy": 0.8442294985055924, |
| "epoch": 1.12, |
| "grad_norm": 1.578125, |
| "learning_rate": 0.0001445192313207067, |
| "loss": 3.3175960540771485, |
| "mean_token_accuracy": 0.7804568380117416, |
| "num_tokens": 2302497.0, |
| "step": 700 |
| }, |
| { |
| "entropy": 0.7547720111906528, |
| "epoch": 1.1360000000000001, |
| "grad_norm": 1.625, |
| "learning_rate": 0.00014296530612327863, |
| "loss": 3.042662239074707, |
| "mean_token_accuracy": 0.7904633671045304, |
| "num_tokens": 2328935.0, |
| "step": 710 |
| }, |
| { |
| "entropy": 0.7825665086507797, |
| "epoch": 1.152, |
| "grad_norm": 1.625, |
| "learning_rate": 0.00014139855114935252, |
| "loss": 3.1688852310180664, |
| "mean_token_accuracy": 0.7849456086754799, |
| "num_tokens": 2355711.0, |
| "step": 720 |
| }, |
| { |
| "entropy": 0.7932837694883347, |
| "epoch": 1.168, |
| "grad_norm": 1.453125, |
| "learning_rate": 0.00013981943424421932, |
| "loss": 3.217195510864258, |
| "mean_token_accuracy": 0.7835724055767059, |
| "num_tokens": 2383138.0, |
| "step": 730 |
| }, |
| { |
| "entropy": 0.7902459263801574, |
| "epoch": 1.184, |
| "grad_norm": 1.4296875, |
| "learning_rate": 0.00013822842694453924, |
| "loss": 3.1072725296020507, |
| "mean_token_accuracy": 0.7865744397044182, |
| "num_tokens": 2411055.0, |
| "step": 740 |
| }, |
| { |
| "entropy": 0.7994440570473671, |
| "epoch": 1.2, |
| "grad_norm": 3.6875, |
| "learning_rate": 0.00013662600433753745, |
| "loss": 3.352284622192383, |
| "mean_token_accuracy": 0.7770974606275558, |
| "num_tokens": 2437837.0, |
| "step": 750 |
| }, |
| { |
| "entropy": 0.7930429011583329, |
| "epoch": 1.216, |
| "grad_norm": 1.765625, |
| "learning_rate": 0.00013501264491913906, |
| "loss": 3.1913455963134765, |
| "mean_token_accuracy": 0.7834275960922241, |
| "num_tokens": 2463749.0, |
| "step": 760 |
| }, |
| { |
| "entropy": 0.844582536816597, |
| "epoch": 1.232, |
| "grad_norm": 1.7734375, |
| "learning_rate": 0.00013338883045108674, |
| "loss": 3.4333824157714843, |
| "mean_token_accuracy": 0.777095565199852, |
| "num_tokens": 2492017.0, |
| "step": 770 |
| }, |
| { |
| "entropy": 0.7995493680238723, |
| "epoch": 1.248, |
| "grad_norm": 1.3046875, |
| "learning_rate": 0.0001317550458170826, |
| "loss": 3.26405029296875, |
| "mean_token_accuracy": 0.7810850724577904, |
| "num_tokens": 2518076.0, |
| "step": 780 |
| }, |
| { |
| "entropy": 0.8197085373103619, |
| "epoch": 1.264, |
| "grad_norm": 1.359375, |
| "learning_rate": 0.00013011177887799845, |
| "loss": 3.2417884826660157, |
| "mean_token_accuracy": 0.7789853543043137, |
| "num_tokens": 2547590.0, |
| "step": 790 |
| }, |
| { |
| "entropy": 0.8223014809191227, |
| "epoch": 1.28, |
| "grad_norm": 1.3828125, |
| "learning_rate": 0.0001284595203261965, |
| "loss": 3.366427993774414, |
| "mean_token_accuracy": 0.7806236863136291, |
| "num_tokens": 2576314.0, |
| "step": 800 |
| }, |
| { |
| "entropy": 0.8295105174183846, |
| "epoch": 1.296, |
| "grad_norm": 1.5078125, |
| "learning_rate": 0.00012679876353900482, |
| "loss": 3.4096916198730467, |
| "mean_token_accuracy": 0.7710141837596893, |
| "num_tokens": 2604971.0, |
| "step": 810 |
| }, |
| { |
| "entropy": 0.8094971597194671, |
| "epoch": 1.312, |
| "grad_norm": 1.609375, |
| "learning_rate": 0.00012513000443139112, |
| "loss": 3.2491512298583984, |
| "mean_token_accuracy": 0.7876769602298737, |
| "num_tokens": 2631986.0, |
| "step": 820 |
| }, |
| { |
| "entropy": 0.798582112789154, |
| "epoch": 1.328, |
| "grad_norm": 1.984375, |
| "learning_rate": 0.00012345374130787854, |
| "loss": 3.19933967590332, |
| "mean_token_accuracy": 0.7822520866990089, |
| "num_tokens": 2656910.0, |
| "step": 830 |
| }, |
| { |
| "entropy": 0.8112326949834824, |
| "epoch": 1.3439999999999999, |
| "grad_norm": 1.515625, |
| "learning_rate": 0.00012177047471374807, |
| "loss": 3.2226589202880858, |
| "mean_token_accuracy": 0.7859175249934196, |
| "num_tokens": 2686061.0, |
| "step": 840 |
| }, |
| { |
| "entropy": 0.8292336650192738, |
| "epoch": 1.3599999999999999, |
| "grad_norm": 1.8515625, |
| "learning_rate": 0.00012008070728557186, |
| "loss": 3.453116226196289, |
| "mean_token_accuracy": 0.7712548315525055, |
| "num_tokens": 2714961.0, |
| "step": 850 |
| }, |
| { |
| "entropy": 0.8000457942485809, |
| "epoch": 1.376, |
| "grad_norm": 1.4375, |
| "learning_rate": 0.00011838494360112185, |
| "loss": 3.1879623413085936, |
| "mean_token_accuracy": 0.7834105402231216, |
| "num_tokens": 2741521.0, |
| "step": 860 |
| }, |
| { |
| "entropy": 0.7751885265111923, |
| "epoch": 1.392, |
| "grad_norm": 1.2578125, |
| "learning_rate": 0.00011668369002869912, |
| "loss": 3.129037094116211, |
| "mean_token_accuracy": 0.7885062962770462, |
| "num_tokens": 2768309.0, |
| "step": 870 |
| }, |
| { |
| "entropy": 0.8135885834693909, |
| "epoch": 1.408, |
| "grad_norm": 1.765625, |
| "learning_rate": 0.00011497745457592816, |
| "loss": 3.372562789916992, |
| "mean_token_accuracy": 0.7782492652535439, |
| "num_tokens": 2796984.0, |
| "step": 880 |
| }, |
| { |
| "entropy": 0.7925616443157196, |
| "epoch": 1.424, |
| "grad_norm": 1.625, |
| "learning_rate": 0.00011326674673806195, |
| "loss": 3.062746047973633, |
| "mean_token_accuracy": 0.7937909960746765, |
| "num_tokens": 2827932.0, |
| "step": 890 |
| }, |
| { |
| "entropy": 0.7889664500951767, |
| "epoch": 1.44, |
| "grad_norm": 1.375, |
| "learning_rate": 0.00011155207734584263, |
| "loss": 3.2588306427001954, |
| "mean_token_accuracy": 0.7854808464646339, |
| "num_tokens": 2860126.0, |
| "step": 900 |
| }, |
| { |
| "entropy": 0.7500537633895874, |
| "epoch": 1.456, |
| "grad_norm": 1.453125, |
| "learning_rate": 0.00010983395841296348, |
| "loss": 2.9640825271606444, |
| "mean_token_accuracy": 0.7949178665876389, |
| "num_tokens": 2884946.0, |
| "step": 910 |
| }, |
| { |
| "entropy": 0.7727585166692734, |
| "epoch": 1.472, |
| "grad_norm": 1.46875, |
| "learning_rate": 0.00010811290298317755, |
| "loss": 3.129817581176758, |
| "mean_token_accuracy": 0.7860683888196945, |
| "num_tokens": 2908387.0, |
| "step": 920 |
| }, |
| { |
| "entropy": 0.7516519904136658, |
| "epoch": 1.488, |
| "grad_norm": 1.2890625, |
| "learning_rate": 0.0001063894249770989, |
| "loss": 3.1172218322753906, |
| "mean_token_accuracy": 0.7909497246146202, |
| "num_tokens": 2935394.0, |
| "step": 930 |
| }, |
| { |
| "entropy": 0.8111906915903091, |
| "epoch": 1.504, |
| "grad_norm": 1.2890625, |
| "learning_rate": 0.00010466403903874176, |
| "loss": 3.222005844116211, |
| "mean_token_accuracy": 0.7841308936476707, |
| "num_tokens": 2966607.0, |
| "step": 940 |
| }, |
| { |
| "entropy": 0.7766234025359153, |
| "epoch": 1.52, |
| "grad_norm": 1.421875, |
| "learning_rate": 0.00010293726038184393, |
| "loss": 3.127474784851074, |
| "mean_token_accuracy": 0.7897476837038994, |
| "num_tokens": 2993998.0, |
| "step": 950 |
| }, |
| { |
| "entropy": 0.7666798770427704, |
| "epoch": 1.536, |
| "grad_norm": 2.015625, |
| "learning_rate": 0.00010120960463601976, |
| "loss": 3.038217544555664, |
| "mean_token_accuracy": 0.7863978728652, |
| "num_tokens": 3022959.0, |
| "step": 960 |
| }, |
| { |
| "entropy": 0.7193653047084808, |
| "epoch": 1.552, |
| "grad_norm": 1.453125, |
| "learning_rate": 9.948158769278939e-05, |
| "loss": 2.9356218338012696, |
| "mean_token_accuracy": 0.7945682421326637, |
| "num_tokens": 3049100.0, |
| "step": 970 |
| }, |
| { |
| "entropy": 0.7440014734864235, |
| "epoch": 1.568, |
| "grad_norm": 1.3046875, |
| "learning_rate": 9.775372555152912e-05, |
| "loss": 2.9548721313476562, |
| "mean_token_accuracy": 0.7947487100958824, |
| "num_tokens": 3073764.0, |
| "step": 980 |
| }, |
| { |
| "entropy": 0.7881634041666985, |
| "epoch": 1.584, |
| "grad_norm": 1.3828125, |
| "learning_rate": 9.602653416539031e-05, |
| "loss": 3.2630630493164063, |
| "mean_token_accuracy": 0.7804958373308182, |
| "num_tokens": 3101016.0, |
| "step": 990 |
| }, |
| { |
| "entropy": 0.7688482075929641, |
| "epoch": 1.6, |
| "grad_norm": 1.1171875, |
| "learning_rate": 9.430052928723153e-05, |
| "loss": 3.0473140716552733, |
| "mean_token_accuracy": 0.7904627084732055, |
| "num_tokens": 3131323.0, |
| "step": 1000 |
| }, |
| { |
| "entropy": 0.8112111315131187, |
| "epoch": 1.616, |
| "grad_norm": 1.3046875, |
| "learning_rate": 9.257622631561085e-05, |
| "loss": 3.270753860473633, |
| "mean_token_accuracy": 0.7852039277553559, |
| "num_tokens": 3161781.0, |
| "step": 1010 |
| }, |
| { |
| "entropy": 0.7603359699249268, |
| "epoch": 1.6320000000000001, |
| "grad_norm": 1.078125, |
| "learning_rate": 9.085414014088369e-05, |
| "loss": 3.0997121810913084, |
| "mean_token_accuracy": 0.7898457184433937, |
| "num_tokens": 3189668.0, |
| "step": 1020 |
| }, |
| { |
| "entropy": 0.789544765651226, |
| "epoch": 1.6480000000000001, |
| "grad_norm": 1.3203125, |
| "learning_rate": 8.913478499145254e-05, |
| "loss": 3.1730154037475584, |
| "mean_token_accuracy": 0.7893433123826981, |
| "num_tokens": 3217248.0, |
| "step": 1030 |
| }, |
| { |
| "entropy": 0.7715817041695118, |
| "epoch": 1.6640000000000001, |
| "grad_norm": 1.125, |
| "learning_rate": 8.741867428021446e-05, |
| "loss": 3.1480581283569338, |
| "mean_token_accuracy": 0.7871334388852119, |
| "num_tokens": 3244290.0, |
| "step": 1040 |
| }, |
| { |
| "entropy": 0.7311166271567344, |
| "epoch": 1.6800000000000002, |
| "grad_norm": 1.734375, |
| "learning_rate": 8.570632045125185e-05, |
| "loss": 2.9483835220336916, |
| "mean_token_accuracy": 0.7995148777961731, |
| "num_tokens": 3270106.0, |
| "step": 1050 |
| }, |
| { |
| "entropy": 0.8038631252944469, |
| "epoch": 1.696, |
| "grad_norm": 1.296875, |
| "learning_rate": 8.399823482681262e-05, |
| "loss": 3.2476577758789062, |
| "mean_token_accuracy": 0.788225382566452, |
| "num_tokens": 3302183.0, |
| "step": 1060 |
| }, |
| { |
| "entropy": 0.7686532281339169, |
| "epoch": 1.712, |
| "grad_norm": 1.484375, |
| "learning_rate": 8.22949274546255e-05, |
| "loss": 3.1519790649414063, |
| "mean_token_accuracy": 0.7915654450654983, |
| "num_tokens": 3329432.0, |
| "step": 1070 |
| }, |
| { |
| "entropy": 0.7671505004167557, |
| "epoch": 1.728, |
| "grad_norm": 1.375, |
| "learning_rate": 8.059690695559568e-05, |
| "loss": 3.085156059265137, |
| "mean_token_accuracy": 0.7906328395009041, |
| "num_tokens": 3353693.0, |
| "step": 1080 |
| }, |
| { |
| "entropy": 0.7490876868367196, |
| "epoch": 1.744, |
| "grad_norm": 1.8203125, |
| "learning_rate": 7.89046803719267e-05, |
| "loss": 3.077616310119629, |
| "mean_token_accuracy": 0.7907149896025658, |
| "num_tokens": 3379146.0, |
| "step": 1090 |
| }, |
| { |
| "entropy": 0.821656309068203, |
| "epoch": 1.76, |
| "grad_norm": 1.1328125, |
| "learning_rate": 7.721875301571359e-05, |
| "loss": 3.2485275268554688, |
| "mean_token_accuracy": 0.7844672784209251, |
| "num_tokens": 3407799.0, |
| "step": 1100 |
| }, |
| { |
| "entropy": 0.7870353177189827, |
| "epoch": 1.776, |
| "grad_norm": 1.390625, |
| "learning_rate": 7.55396283180529e-05, |
| "loss": 3.2128215789794923, |
| "mean_token_accuracy": 0.7861049249768257, |
| "num_tokens": 3436295.0, |
| "step": 1110 |
| }, |
| { |
| "entropy": 0.7809636801481247, |
| "epoch": 1.792, |
| "grad_norm": 2.59375, |
| "learning_rate": 7.386780767871397e-05, |
| "loss": 3.1558361053466797, |
| "mean_token_accuracy": 0.7923071622848511, |
| "num_tokens": 3463552.0, |
| "step": 1120 |
| }, |
| { |
| "entropy": 0.7963082253932953, |
| "epoch": 1.808, |
| "grad_norm": 1.3515625, |
| "learning_rate": 7.22037903164173e-05, |
| "loss": 3.2051555633544924, |
| "mean_token_accuracy": 0.7853836208581925, |
| "num_tokens": 3490070.0, |
| "step": 1130 |
| }, |
| { |
| "entropy": 0.8024965927004815, |
| "epoch": 1.8239999999999998, |
| "grad_norm": 1.4296875, |
| "learning_rate": 7.054807311976379e-05, |
| "loss": 3.256487274169922, |
| "mean_token_accuracy": 0.7800001233816147, |
| "num_tokens": 3519607.0, |
| "step": 1140 |
| }, |
| { |
| "entropy": 0.7608409076929092, |
| "epoch": 1.8399999999999999, |
| "grad_norm": 1.6640625, |
| "learning_rate": 6.890115049885994e-05, |
| "loss": 3.1335351943969725, |
| "mean_token_accuracy": 0.7930847942829132, |
| "num_tokens": 3549954.0, |
| "step": 1150 |
| }, |
| { |
| "entropy": 0.7995478600263596, |
| "epoch": 1.8559999999999999, |
| "grad_norm": 1.0859375, |
| "learning_rate": 6.726351423768322e-05, |
| "loss": 3.192752647399902, |
| "mean_token_accuracy": 0.7869470104575157, |
| "num_tokens": 3579252.0, |
| "step": 1160 |
| }, |
| { |
| "entropy": 0.7790917068719864, |
| "epoch": 1.8719999999999999, |
| "grad_norm": 1.15625, |
| "learning_rate": 6.563565334723134e-05, |
| "loss": 3.1096052169799804, |
| "mean_token_accuracy": 0.7890855461359024, |
| "num_tokens": 3607698.0, |
| "step": 1170 |
| }, |
| { |
| "entropy": 0.6804191634058953, |
| "epoch": 1.888, |
| "grad_norm": 1.4921875, |
| "learning_rate": 6.40180539194999e-05, |
| "loss": 2.7701562881469726, |
| "mean_token_accuracy": 0.8071593567728996, |
| "num_tokens": 3635417.0, |
| "step": 1180 |
| }, |
| { |
| "entropy": 0.7258283078670502, |
| "epoch": 1.904, |
| "grad_norm": 1.28125, |
| "learning_rate": 6.241119898233144e-05, |
| "loss": 2.969502830505371, |
| "mean_token_accuracy": 0.7979315519332886, |
| "num_tokens": 3661222.0, |
| "step": 1190 |
| }, |
| { |
| "entropy": 0.7636997759342193, |
| "epoch": 1.92, |
| "grad_norm": 1.6484375, |
| "learning_rate": 6.0815568355179556e-05, |
| "loss": 3.1156858444213866, |
| "mean_token_accuracy": 0.7898655593395233, |
| "num_tokens": 3692905.0, |
| "step": 1200 |
| }, |
| { |
| "entropy": 0.80526192933321, |
| "epoch": 1.936, |
| "grad_norm": 1.1796875, |
| "learning_rate": 5.923163850583113e-05, |
| "loss": 3.170664596557617, |
| "mean_token_accuracy": 0.7876248195767402, |
| "num_tokens": 3720018.0, |
| "step": 1210 |
| }, |
| { |
| "entropy": 0.7665807068347931, |
| "epoch": 1.952, |
| "grad_norm": 1.28125, |
| "learning_rate": 5.765988240812921e-05, |
| "loss": 3.141133499145508, |
| "mean_token_accuracy": 0.790970367193222, |
| "num_tokens": 3746413.0, |
| "step": 1220 |
| }, |
| { |
| "entropy": 0.7018338434398175, |
| "epoch": 1.968, |
| "grad_norm": 1.375, |
| "learning_rate": 5.6100769400739383e-05, |
| "loss": 2.782573699951172, |
| "mean_token_accuracy": 0.8073293790221214, |
| "num_tokens": 3770477.0, |
| "step": 1230 |
| }, |
| { |
| "entropy": 0.7392673164606094, |
| "epoch": 1.984, |
| "grad_norm": 1.46875, |
| "learning_rate": 5.4554765047001613e-05, |
| "loss": 2.957522964477539, |
| "mean_token_accuracy": 0.7974923148751258, |
| "num_tokens": 3796914.0, |
| "step": 1240 |
| }, |
| { |
| "entropy": 0.7508858695626259, |
| "epoch": 2.0, |
| "grad_norm": 1.2890625, |
| "learning_rate": 5.302233099590928e-05, |
| "loss": 3.049625205993652, |
| "mean_token_accuracy": 0.7928661614656448, |
| "num_tokens": 3822313.0, |
| "step": 1250 |
| }, |
| { |
| "epoch": 2.0, |
| "eval_entropy": 0.7935182250738144, |
| "eval_loss": 0.9520308971405029, |
| "eval_mean_token_accuracy": 0.7710564243793487, |
| "eval_num_tokens": 3822313.0, |
| "eval_runtime": 72.7945, |
| "eval_samples_per_second": 13.737, |
| "eval_steps_per_second": 3.434, |
| "step": 1250 |
| }, |
| { |
| "entropy": 0.7053400471806526, |
| "epoch": 2.016, |
| "grad_norm": 1.3125, |
| "learning_rate": 5.150392484425728e-05, |
| "loss": 2.6305599212646484, |
| "mean_token_accuracy": 0.8136913523077964, |
| "num_tokens": 3849460.0, |
| "step": 1260 |
| }, |
| { |
| "entropy": 0.6759374007582665, |
| "epoch": 2.032, |
| "grad_norm": 1.1796875, |
| "learning_rate": 5.000000000000002e-05, |
| "loss": 2.740474891662598, |
| "mean_token_accuracy": 0.810079038143158, |
| "num_tokens": 3881699.0, |
| "step": 1270 |
| }, |
| { |
| "entropy": 0.6673750668764115, |
| "epoch": 2.048, |
| "grad_norm": 1.234375, |
| "learning_rate": 4.851100554686021e-05, |
| "loss": 2.5454484939575197, |
| "mean_token_accuracy": 0.8159348770976067, |
| "num_tokens": 3909562.0, |
| "step": 1280 |
| }, |
| { |
| "entropy": 0.5898188591003418, |
| "epoch": 2.064, |
| "grad_norm": 1.2578125, |
| "learning_rate": 4.7037386110228985e-05, |
| "loss": 2.3230581283569336, |
| "mean_token_accuracy": 0.8328281179070472, |
| "num_tokens": 3936217.0, |
| "step": 1290 |
| }, |
| { |
| "entropy": 0.6469230554997921, |
| "epoch": 2.08, |
| "grad_norm": 1.59375, |
| "learning_rate": 4.5579581724397255e-05, |
| "loss": 2.592478942871094, |
| "mean_token_accuracy": 0.8147690609097481, |
| "num_tokens": 3966142.0, |
| "step": 1300 |
| }, |
| { |
| "entropy": 0.6723301410675049, |
| "epoch": 2.096, |
| "grad_norm": 1.125, |
| "learning_rate": 4.413802770115816e-05, |
| "loss": 2.6196125030517576, |
| "mean_token_accuracy": 0.8145923808217048, |
| "num_tokens": 3995663.0, |
| "step": 1310 |
| }, |
| { |
| "entropy": 0.6221730917692184, |
| "epoch": 2.112, |
| "grad_norm": 1.2578125, |
| "learning_rate": 4.271315449981934e-05, |
| "loss": 2.4203453063964844, |
| "mean_token_accuracy": 0.8260695144534111, |
| "num_tokens": 4025537.0, |
| "step": 1320 |
| }, |
| { |
| "entropy": 0.6446548275649547, |
| "epoch": 2.128, |
| "grad_norm": 1.1015625, |
| "learning_rate": 4.130538759866457e-05, |
| "loss": 2.562292289733887, |
| "mean_token_accuracy": 0.8144417598843574, |
| "num_tokens": 4053555.0, |
| "step": 1330 |
| }, |
| { |
| "entropy": 0.6655827894806862, |
| "epoch": 2.144, |
| "grad_norm": 1.2421875, |
| "learning_rate": 3.991514736790258e-05, |
| "loss": 2.6614761352539062, |
| "mean_token_accuracy": 0.8111246332526207, |
| "num_tokens": 4080361.0, |
| "step": 1340 |
| }, |
| { |
| "entropy": 0.6002881981432437, |
| "epoch": 2.16, |
| "grad_norm": 1.078125, |
| "learning_rate": 3.854284894414122e-05, |
| "loss": 2.3379238128662108, |
| "mean_token_accuracy": 0.8291780337691307, |
| "num_tokens": 4108641.0, |
| "step": 1350 |
| }, |
| { |
| "entropy": 0.6322725959122181, |
| "epoch": 2.176, |
| "grad_norm": 1.34375, |
| "learning_rate": 3.7188902106424416e-05, |
| "loss": 2.5648292541503905, |
| "mean_token_accuracy": 0.819225026667118, |
| "num_tokens": 4137407.0, |
| "step": 1360 |
| }, |
| { |
| "entropy": 0.6220526427030564, |
| "epoch": 2.192, |
| "grad_norm": 1.265625, |
| "learning_rate": 3.5853711153868965e-05, |
| "loss": 2.521045112609863, |
| "mean_token_accuracy": 0.8181931138038635, |
| "num_tokens": 4165637.0, |
| "step": 1370 |
| }, |
| { |
| "entropy": 0.59385786652565, |
| "epoch": 2.208, |
| "grad_norm": 1.1484375, |
| "learning_rate": 3.4537674784937614e-05, |
| "loss": 2.256278419494629, |
| "mean_token_accuracy": 0.8328502863645554, |
| "num_tokens": 4190019.0, |
| "step": 1380 |
| }, |
| { |
| "entropy": 0.6182742714881897, |
| "epoch": 2.224, |
| "grad_norm": 1.390625, |
| "learning_rate": 3.324118597838464e-05, |
| "loss": 2.4784978866577148, |
| "mean_token_accuracy": 0.8253998696804047, |
| "num_tokens": 4218065.0, |
| "step": 1390 |
| }, |
| { |
| "entropy": 0.625076462328434, |
| "epoch": 2.24, |
| "grad_norm": 1.375, |
| "learning_rate": 3.196463187590929e-05, |
| "loss": 2.5287513732910156, |
| "mean_token_accuracy": 0.8231741398572922, |
| "num_tokens": 4245504.0, |
| "step": 1400 |
| }, |
| { |
| "entropy": 0.6474951952695847, |
| "epoch": 2.2560000000000002, |
| "grad_norm": 1.65625, |
| "learning_rate": 3.070839366655215e-05, |
| "loss": 2.574743461608887, |
| "mean_token_accuracy": 0.8156981885433197, |
| "num_tokens": 4275439.0, |
| "step": 1410 |
| }, |
| { |
| "entropy": 0.6507719412446022, |
| "epoch": 2.2720000000000002, |
| "grad_norm": 1.3984375, |
| "learning_rate": 2.9472846472869298e-05, |
| "loss": 2.5274782180786133, |
| "mean_token_accuracy": 0.8186039045453072, |
| "num_tokens": 4300521.0, |
| "step": 1420 |
| }, |
| { |
| "entropy": 0.6078799493610859, |
| "epoch": 2.288, |
| "grad_norm": 1.203125, |
| "learning_rate": 2.8258359238917665e-05, |
| "loss": 2.3472341537475585, |
| "mean_token_accuracy": 0.8310479253530503, |
| "num_tokens": 4325611.0, |
| "step": 1430 |
| }, |
| { |
| "entropy": 0.6164450705051422, |
| "epoch": 2.304, |
| "grad_norm": 1.2109375, |
| "learning_rate": 2.7065294620085424e-05, |
| "loss": 2.4567951202392577, |
| "mean_token_accuracy": 0.8222395464777946, |
| "num_tokens": 4351381.0, |
| "step": 1440 |
| }, |
| { |
| "entropy": 0.6051406018435955, |
| "epoch": 2.32, |
| "grad_norm": 1.5078125, |
| "learning_rate": 2.5894008874800325e-05, |
| "loss": 2.4069543838500977, |
| "mean_token_accuracy": 0.8269070222973823, |
| "num_tokens": 4380875.0, |
| "step": 1450 |
| }, |
| { |
| "entropy": 0.6406742617487907, |
| "epoch": 2.336, |
| "grad_norm": 1.5, |
| "learning_rate": 2.4744851758148156e-05, |
| "loss": 2.575749969482422, |
| "mean_token_accuracy": 0.8187732085585594, |
| "num_tokens": 4410021.0, |
| "step": 1460 |
| }, |
| { |
| "entropy": 0.6262726046144962, |
| "epoch": 2.352, |
| "grad_norm": 11.6875, |
| "learning_rate": 2.361816641743303e-05, |
| "loss": 2.5055971145629883, |
| "mean_token_accuracy": 0.8225668385624886, |
| "num_tokens": 4437438.0, |
| "step": 1470 |
| }, |
| { |
| "entropy": 0.6012745849788189, |
| "epoch": 2.368, |
| "grad_norm": 1.265625, |
| "learning_rate": 2.251428928971102e-05, |
| "loss": 2.387685775756836, |
| "mean_token_accuracy": 0.8289849147200584, |
| "num_tokens": 4463678.0, |
| "step": 1480 |
| }, |
| { |
| "entropy": 0.6096144415438175, |
| "epoch": 2.384, |
| "grad_norm": 0.98828125, |
| "learning_rate": 2.1433550001327373e-05, |
| "loss": 2.4208652496337892, |
| "mean_token_accuracy": 0.825342382490635, |
| "num_tokens": 4488019.0, |
| "step": 1490 |
| }, |
| { |
| "entropy": 0.6083463601768017, |
| "epoch": 2.4, |
| "grad_norm": 1.2265625, |
| "learning_rate": 2.0376271269487514e-05, |
| "loss": 2.388985824584961, |
| "mean_token_accuracy": 0.824395664036274, |
| "num_tokens": 4517306.0, |
| "step": 1500 |
| }, |
| { |
| "entropy": 0.6329524978995323, |
| "epoch": 2.416, |
| "grad_norm": 1.671875, |
| "learning_rate": 1.9342768805891178e-05, |
| "loss": 2.51919002532959, |
| "mean_token_accuracy": 0.8179876819252968, |
| "num_tokens": 4545893.0, |
| "step": 1510 |
| }, |
| { |
| "entropy": 0.5982788041234016, |
| "epoch": 2.432, |
| "grad_norm": 1.2265625, |
| "learning_rate": 1.8333351222458407e-05, |
| "loss": 2.3680145263671877, |
| "mean_token_accuracy": 0.8285414636135101, |
| "num_tokens": 4573653.0, |
| "step": 1520 |
| }, |
| { |
| "entropy": 0.680705762654543, |
| "epoch": 2.448, |
| "grad_norm": 1.140625, |
| "learning_rate": 1.7348319939175637e-05, |
| "loss": 2.6735370635986326, |
| "mean_token_accuracy": 0.8125541433691978, |
| "num_tokens": 4601669.0, |
| "step": 1530 |
| }, |
| { |
| "entropy": 0.563636127859354, |
| "epoch": 2.464, |
| "grad_norm": 1.2265625, |
| "learning_rate": 1.6387969094089316e-05, |
| "loss": 2.165860176086426, |
| "mean_token_accuracy": 0.838480469584465, |
| "num_tokens": 4627149.0, |
| "step": 1540 |
| }, |
| { |
| "entropy": 0.5980882495641708, |
| "epoch": 2.48, |
| "grad_norm": 1.046875, |
| "learning_rate": 1.5452585455473977e-05, |
| "loss": 2.3885900497436525, |
| "mean_token_accuracy": 0.8232199549674988, |
| "num_tokens": 4653051.0, |
| "step": 1550 |
| }, |
| { |
| "entropy": 0.5893117256462574, |
| "epoch": 2.496, |
| "grad_norm": 1.375, |
| "learning_rate": 1.454244833620102e-05, |
| "loss": 2.361526107788086, |
| "mean_token_accuracy": 0.8287293270230294, |
| "num_tokens": 4679592.0, |
| "step": 1560 |
| }, |
| { |
| "entropy": 0.6897482767701149, |
| "epoch": 2.512, |
| "grad_norm": 1.1328125, |
| "learning_rate": 1.3657829510333654e-05, |
| "loss": 2.8343505859375, |
| "mean_token_accuracy": 0.8045855790376664, |
| "num_tokens": 4712580.0, |
| "step": 1570 |
| }, |
| { |
| "entropy": 0.5980210550129413, |
| "epoch": 2.528, |
| "grad_norm": 1.390625, |
| "learning_rate": 1.2798993131973091e-05, |
| "loss": 2.393186569213867, |
| "mean_token_accuracy": 0.8249497979879379, |
| "num_tokens": 4740546.0, |
| "step": 1580 |
| }, |
| { |
| "entropy": 0.6140701808035374, |
| "epoch": 2.544, |
| "grad_norm": 1.3359375, |
| "learning_rate": 1.1966195656380031e-05, |
| "loss": 2.440609169006348, |
| "mean_token_accuracy": 0.8237545773386955, |
| "num_tokens": 4767938.0, |
| "step": 1590 |
| }, |
| { |
| "entropy": 0.6063477545976639, |
| "epoch": 2.56, |
| "grad_norm": 1.1875, |
| "learning_rate": 1.1159685763395111e-05, |
| "loss": 2.3470132827758787, |
| "mean_token_accuracy": 0.8265559047460556, |
| "num_tokens": 4794286.0, |
| "step": 1600 |
| }, |
| { |
| "entropy": 0.6648287683725357, |
| "epoch": 2.576, |
| "grad_norm": 1.1328125, |
| "learning_rate": 1.0379704283181179e-05, |
| "loss": 2.7020286560058593, |
| "mean_token_accuracy": 0.808851246535778, |
| "num_tokens": 4823293.0, |
| "step": 1610 |
| }, |
| { |
| "entropy": 0.6474792867898941, |
| "epoch": 2.592, |
| "grad_norm": 1.1953125, |
| "learning_rate": 9.62648412430951e-06, |
| "loss": 2.606777000427246, |
| "mean_token_accuracy": 0.8219101890921593, |
| "num_tokens": 4852606.0, |
| "step": 1620 |
| }, |
| { |
| "entropy": 0.6176077917218208, |
| "epoch": 2.608, |
| "grad_norm": 1.5625, |
| "learning_rate": 8.900250204211514e-06, |
| "loss": 2.4370376586914064, |
| "mean_token_accuracy": 0.8261922746896744, |
| "num_tokens": 4880168.0, |
| "step": 1630 |
| }, |
| { |
| "entropy": 0.6533839575946331, |
| "epoch": 2.624, |
| "grad_norm": 1.15625, |
| "learning_rate": 8.201219382016556e-06, |
| "loss": 2.6114227294921877, |
| "mean_token_accuracy": 0.815986056625843, |
| "num_tokens": 4909334.0, |
| "step": 1640 |
| }, |
| { |
| "entropy": 0.617949029058218, |
| "epoch": 2.64, |
| "grad_norm": 1.2265625, |
| "learning_rate": 7.529600393796232e-06, |
| "loss": 2.4090667724609376, |
| "mean_token_accuracy": 0.8269602835178376, |
| "num_tokens": 4936371.0, |
| "step": 1650 |
| }, |
| { |
| "entropy": 0.6316794902086258, |
| "epoch": 2.656, |
| "grad_norm": 1.1640625, |
| "learning_rate": 6.8855937902340576e-06, |
| "loss": 2.516379165649414, |
| "mean_token_accuracy": 0.819202584028244, |
| "num_tokens": 4966014.0, |
| "step": 1660 |
| }, |
| { |
| "entropy": 0.6159006580710411, |
| "epoch": 2.672, |
| "grad_norm": 1.34375, |
| "learning_rate": 6.269391876739495e-06, |
| "loss": 2.430217170715332, |
| "mean_token_accuracy": 0.8262986287474632, |
| "num_tokens": 4991945.0, |
| "step": 1670 |
| }, |
| { |
| "entropy": 0.6038173116743565, |
| "epoch": 2.6879999999999997, |
| "grad_norm": 0.9765625, |
| "learning_rate": 5.681178656024055e-06, |
| "loss": 2.3973724365234377, |
| "mean_token_accuracy": 0.8248073413968087, |
| "num_tokens": 5019869.0, |
| "step": 1680 |
| }, |
| { |
| "entropy": 0.6135182313621044, |
| "epoch": 2.7039999999999997, |
| "grad_norm": 1.2578125, |
| "learning_rate": 5.121129773156663e-06, |
| "loss": 2.4443332672119142, |
| "mean_token_accuracy": 0.824867345392704, |
| "num_tokens": 5048093.0, |
| "step": 1690 |
| }, |
| { |
| "entropy": 0.6408688805997371, |
| "epoch": 2.7199999999999998, |
| "grad_norm": 1.1640625, |
| "learning_rate": 4.58941246311464e-06, |
| "loss": 2.571330261230469, |
| "mean_token_accuracy": 0.8133804589509964, |
| "num_tokens": 5078113.0, |
| "step": 1700 |
| }, |
| { |
| "entropy": 0.6208270855247975, |
| "epoch": 2.7359999999999998, |
| "grad_norm": 1.1953125, |
| "learning_rate": 4.0861855008460405e-06, |
| "loss": 2.4142126083374023, |
| "mean_token_accuracy": 0.8225093990564346, |
| "num_tokens": 5106452.0, |
| "step": 1710 |
| }, |
| { |
| "entropy": 0.6045813173055649, |
| "epoch": 2.752, |
| "grad_norm": 1.1875, |
| "learning_rate": 3.611599153858214e-06, |
| "loss": 2.3853626251220703, |
| "mean_token_accuracy": 0.8237989127635956, |
| "num_tokens": 5133206.0, |
| "step": 1720 |
| }, |
| { |
| "entropy": 0.5914519041776657, |
| "epoch": 2.768, |
| "grad_norm": 1.328125, |
| "learning_rate": 3.1657951373467497e-06, |
| "loss": 2.298448181152344, |
| "mean_token_accuracy": 0.8312044337391853, |
| "num_tokens": 5156203.0, |
| "step": 1730 |
| }, |
| { |
| "entropy": 0.6053797014057636, |
| "epoch": 2.784, |
| "grad_norm": 1.2734375, |
| "learning_rate": 2.748906571878207e-06, |
| "loss": 2.4279769897460937, |
| "mean_token_accuracy": 0.8242192730307579, |
| "num_tokens": 5183714.0, |
| "step": 1740 |
| }, |
| { |
| "entropy": 0.6214097492396832, |
| "epoch": 2.8, |
| "grad_norm": 1.0078125, |
| "learning_rate": 2.3610579436393e-06, |
| "loss": 2.4572864532470704, |
| "mean_token_accuracy": 0.8246369242668152, |
| "num_tokens": 5212940.0, |
| "step": 1750 |
| }, |
| { |
| "entropy": 0.6008337244391442, |
| "epoch": 2.816, |
| "grad_norm": 1.1875, |
| "learning_rate": 2.002365067264289e-06, |
| "loss": 2.364995002746582, |
| "mean_token_accuracy": 0.8302600085735321, |
| "num_tokens": 5236919.0, |
| "step": 1760 |
| }, |
| { |
| "entropy": 0.5900142095983029, |
| "epoch": 2.832, |
| "grad_norm": 1.1875, |
| "learning_rate": 1.6729350512519005e-06, |
| "loss": 2.3344017028808595, |
| "mean_token_accuracy": 0.8315937608480454, |
| "num_tokens": 5264983.0, |
| "step": 1770 |
| }, |
| { |
| "entropy": 0.6705081954598426, |
| "epoch": 2.848, |
| "grad_norm": 1.2734375, |
| "learning_rate": 1.3728662659818204e-06, |
| "loss": 2.7036811828613283, |
| "mean_token_accuracy": 0.8103000715374946, |
| "num_tokens": 5294775.0, |
| "step": 1780 |
| }, |
| { |
| "entropy": 0.6234683454036712, |
| "epoch": 2.864, |
| "grad_norm": 1.3046875, |
| "learning_rate": 1.1022483143405705e-06, |
| "loss": 2.4741338729858398, |
| "mean_token_accuracy": 0.8195253252983093, |
| "num_tokens": 5322219.0, |
| "step": 1790 |
| }, |
| { |
| "entropy": 0.6028523907065392, |
| "epoch": 2.88, |
| "grad_norm": 1.875, |
| "learning_rate": 8.611620049653879e-07, |
| "loss": 2.3432369232177734, |
| "mean_token_accuracy": 0.8303986221551896, |
| "num_tokens": 5348043.0, |
| "step": 1800 |
| }, |
| { |
| "entropy": 0.6217740766704083, |
| "epoch": 2.896, |
| "grad_norm": 1.3515625, |
| "learning_rate": 6.496793281141056e-07, |
| "loss": 2.472747230529785, |
| "mean_token_accuracy": 0.8228062316775322, |
| "num_tokens": 5377321.0, |
| "step": 1810 |
| }, |
| { |
| "entropy": 0.651834636926651, |
| "epoch": 2.912, |
| "grad_norm": 1.390625, |
| "learning_rate": 4.678634341683252e-07, |
| "loss": 2.596288871765137, |
| "mean_token_accuracy": 0.8179462268948555, |
| "num_tokens": 5407751.0, |
| "step": 1820 |
| }, |
| { |
| "entropy": 0.602143744379282, |
| "epoch": 2.928, |
| "grad_norm": 1.4453125, |
| "learning_rate": 3.1576861477621287e-07, |
| "loss": 2.424236297607422, |
| "mean_token_accuracy": 0.8246050551533699, |
| "num_tokens": 5435212.0, |
| "step": 1830 |
| }, |
| { |
| "entropy": 0.6053410112857819, |
| "epoch": 2.944, |
| "grad_norm": 1.1796875, |
| "learning_rate": 1.9344028664056713e-07, |
| "loss": 2.388152313232422, |
| "mean_token_accuracy": 0.8283460915088654, |
| "num_tokens": 5462690.0, |
| "step": 1840 |
| }, |
| { |
| "entropy": 0.6017510555684566, |
| "epoch": 2.96, |
| "grad_norm": 1.4765625, |
| "learning_rate": 1.0091497795706728e-07, |
| "loss": 2.410584831237793, |
| "mean_token_accuracy": 0.8255392298102379, |
| "num_tokens": 5489364.0, |
| "step": 1850 |
| }, |
| { |
| "entropy": 0.6333115980029106, |
| "epoch": 2.976, |
| "grad_norm": 1.2734375, |
| "learning_rate": 3.8220317506654226e-08, |
| "loss": 2.480808067321777, |
| "mean_token_accuracy": 0.8206309258937836, |
| "num_tokens": 5518093.0, |
| "step": 1860 |
| }, |
| { |
| "entropy": 0.6766974046826363, |
| "epoch": 2.992, |
| "grad_norm": 1.2421875, |
| "learning_rate": 5.375026405352035e-09, |
| "loss": 2.673609161376953, |
| "mean_token_accuracy": 0.8086716338992119, |
| "num_tokens": 5547228.0, |
| "step": 1870 |
| }, |
| { |
| "epoch": 3.0, |
| "eval_entropy": 0.6697398825883866, |
| "eval_loss": 0.986240029335022, |
| "eval_mean_token_accuracy": 0.7718619239330292, |
| "eval_num_tokens": 5560476.0, |
| "eval_runtime": 72.585, |
| "eval_samples_per_second": 13.777, |
| "eval_steps_per_second": 3.444, |
| "step": 1875 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 1875, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 3, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 1.618119414518085e+17, |
| "train_batch_size": 4, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
