| { | |
| "best_metric": 0.38729745149612427, | |
| "best_model_checkpoint": "mikhail_panzo/zlm_b64_le5_s12000/checkpoint-4500", | |
| "epoch": 3.769633507853403, | |
| "eval_steps": 500, | |
| "global_step": 4500, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.041884816753926704, | |
| "grad_norm": 12.821951866149902, | |
| "learning_rate": 2.4000000000000003e-07, | |
| "loss": 1.1152, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.08376963350785341, | |
| "grad_norm": 6.186221122741699, | |
| "learning_rate": 4.900000000000001e-07, | |
| "loss": 1.0584, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.1256544502617801, | |
| "grad_norm": 13.08674430847168, | |
| "learning_rate": 7.4e-07, | |
| "loss": 0.9996, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.16753926701570682, | |
| "grad_norm": 4.10632848739624, | |
| "learning_rate": 9.9e-07, | |
| "loss": 0.9355, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.2094240837696335, | |
| "grad_norm": 3.2428064346313477, | |
| "learning_rate": 1.2400000000000002e-06, | |
| "loss": 0.852, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.2513089005235602, | |
| "grad_norm": 2.0861401557922363, | |
| "learning_rate": 1.4900000000000001e-06, | |
| "loss": 0.8241, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.2931937172774869, | |
| "grad_norm": 8.602317810058594, | |
| "learning_rate": 1.74e-06, | |
| "loss": 0.7629, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 0.33507853403141363, | |
| "grad_norm": 2.7594990730285645, | |
| "learning_rate": 1.9900000000000004e-06, | |
| "loss": 0.7418, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 0.3769633507853403, | |
| "grad_norm": 2.7047348022460938, | |
| "learning_rate": 2.24e-06, | |
| "loss": 0.7166, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 0.418848167539267, | |
| "grad_norm": 2.0666158199310303, | |
| "learning_rate": 2.4900000000000003e-06, | |
| "loss": 0.6876, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.418848167539267, | |
| "eval_loss": 0.5937426090240479, | |
| "eval_runtime": 263.0955, | |
| "eval_samples_per_second": 32.266, | |
| "eval_steps_per_second": 4.037, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.4607329842931937, | |
| "grad_norm": 2.955193519592285, | |
| "learning_rate": 2.7400000000000004e-06, | |
| "loss": 0.673, | |
| "step": 550 | |
| }, | |
| { | |
| "epoch": 0.5026178010471204, | |
| "grad_norm": 4.856322765350342, | |
| "learning_rate": 2.99e-06, | |
| "loss": 0.6539, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 0.5445026178010471, | |
| "grad_norm": 7.631103992462158, | |
| "learning_rate": 3.2400000000000003e-06, | |
| "loss": 0.6588, | |
| "step": 650 | |
| }, | |
| { | |
| "epoch": 0.5863874345549738, | |
| "grad_norm": 2.715771436691284, | |
| "learning_rate": 3.49e-06, | |
| "loss": 0.624, | |
| "step": 700 | |
| }, | |
| { | |
| "epoch": 0.6282722513089005, | |
| "grad_norm": 2.7684273719787598, | |
| "learning_rate": 3.74e-06, | |
| "loss": 0.6116, | |
| "step": 750 | |
| }, | |
| { | |
| "epoch": 0.6701570680628273, | |
| "grad_norm": 2.219599723815918, | |
| "learning_rate": 3.990000000000001e-06, | |
| "loss": 0.6097, | |
| "step": 800 | |
| }, | |
| { | |
| "epoch": 0.7120418848167539, | |
| "grad_norm": 3.749188184738159, | |
| "learning_rate": 4.24e-06, | |
| "loss": 0.614, | |
| "step": 850 | |
| }, | |
| { | |
| "epoch": 0.7539267015706806, | |
| "grad_norm": 2.120619773864746, | |
| "learning_rate": 4.49e-06, | |
| "loss": 0.5814, | |
| "step": 900 | |
| }, | |
| { | |
| "epoch": 0.7958115183246073, | |
| "grad_norm": 2.7725327014923096, | |
| "learning_rate": 4.74e-06, | |
| "loss": 0.5895, | |
| "step": 950 | |
| }, | |
| { | |
| "epoch": 0.837696335078534, | |
| "grad_norm": 9.851961135864258, | |
| "learning_rate": 4.9900000000000005e-06, | |
| "loss": 0.5623, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 0.837696335078534, | |
| "eval_loss": 0.4933945834636688, | |
| "eval_runtime": 258.739, | |
| "eval_samples_per_second": 32.809, | |
| "eval_steps_per_second": 4.105, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 0.8795811518324608, | |
| "grad_norm": 5.074089527130127, | |
| "learning_rate": 5.240000000000001e-06, | |
| "loss": 0.5748, | |
| "step": 1050 | |
| }, | |
| { | |
| "epoch": 0.9214659685863874, | |
| "grad_norm": 5.884639739990234, | |
| "learning_rate": 5.485e-06, | |
| "loss": 0.5395, | |
| "step": 1100 | |
| }, | |
| { | |
| "epoch": 0.9633507853403142, | |
| "grad_norm": 3.1588447093963623, | |
| "learning_rate": 5.735e-06, | |
| "loss": 0.5478, | |
| "step": 1150 | |
| }, | |
| { | |
| "epoch": 1.0052356020942408, | |
| "grad_norm": 2.4452970027923584, | |
| "learning_rate": 5.985000000000001e-06, | |
| "loss": 0.538, | |
| "step": 1200 | |
| }, | |
| { | |
| "epoch": 1.0471204188481675, | |
| "grad_norm": 8.290769577026367, | |
| "learning_rate": 6.235000000000001e-06, | |
| "loss": 0.5411, | |
| "step": 1250 | |
| }, | |
| { | |
| "epoch": 1.0890052356020943, | |
| "grad_norm": 4.080046653747559, | |
| "learning_rate": 6.485000000000001e-06, | |
| "loss": 0.5387, | |
| "step": 1300 | |
| }, | |
| { | |
| "epoch": 1.130890052356021, | |
| "grad_norm": 2.7520201206207275, | |
| "learning_rate": 6.735000000000001e-06, | |
| "loss": 0.5426, | |
| "step": 1350 | |
| }, | |
| { | |
| "epoch": 1.1727748691099475, | |
| "grad_norm": 3.8192107677459717, | |
| "learning_rate": 6.985000000000001e-06, | |
| "loss": 0.5315, | |
| "step": 1400 | |
| }, | |
| { | |
| "epoch": 1.2146596858638743, | |
| "grad_norm": 9.535676956176758, | |
| "learning_rate": 7.235000000000001e-06, | |
| "loss": 0.5159, | |
| "step": 1450 | |
| }, | |
| { | |
| "epoch": 1.256544502617801, | |
| "grad_norm": 2.5731022357940674, | |
| "learning_rate": 7.485000000000001e-06, | |
| "loss": 0.5165, | |
| "step": 1500 | |
| }, | |
| { | |
| "epoch": 1.256544502617801, | |
| "eval_loss": 0.4628298878669739, | |
| "eval_runtime": 257.0265, | |
| "eval_samples_per_second": 33.028, | |
| "eval_steps_per_second": 4.132, | |
| "step": 1500 | |
| }, | |
| { | |
| "epoch": 1.2984293193717278, | |
| "grad_norm": 3.662090539932251, | |
| "learning_rate": 7.735e-06, | |
| "loss": 0.5188, | |
| "step": 1550 | |
| }, | |
| { | |
| "epoch": 1.3403141361256545, | |
| "grad_norm": 3.1680572032928467, | |
| "learning_rate": 7.985e-06, | |
| "loss": 0.5148, | |
| "step": 1600 | |
| }, | |
| { | |
| "epoch": 1.3821989528795813, | |
| "grad_norm": 3.601471185684204, | |
| "learning_rate": 8.235e-06, | |
| "loss": 0.5127, | |
| "step": 1650 | |
| }, | |
| { | |
| "epoch": 1.4240837696335078, | |
| "grad_norm": 4.497046947479248, | |
| "learning_rate": 8.485000000000001e-06, | |
| "loss": 0.5039, | |
| "step": 1700 | |
| }, | |
| { | |
| "epoch": 1.4659685863874345, | |
| "grad_norm": 3.6496424674987793, | |
| "learning_rate": 8.735000000000002e-06, | |
| "loss": 0.5114, | |
| "step": 1750 | |
| }, | |
| { | |
| "epoch": 1.5078534031413613, | |
| "grad_norm": 2.7080061435699463, | |
| "learning_rate": 8.985000000000001e-06, | |
| "loss": 0.4953, | |
| "step": 1800 | |
| }, | |
| { | |
| "epoch": 1.5497382198952878, | |
| "grad_norm": 2.5439274311065674, | |
| "learning_rate": 9.235e-06, | |
| "loss": 0.497, | |
| "step": 1850 | |
| }, | |
| { | |
| "epoch": 1.5916230366492146, | |
| "grad_norm": 2.004573106765747, | |
| "learning_rate": 9.485000000000002e-06, | |
| "loss": 0.4889, | |
| "step": 1900 | |
| }, | |
| { | |
| "epoch": 1.6335078534031413, | |
| "grad_norm": 4.699184894561768, | |
| "learning_rate": 9.735e-06, | |
| "loss": 0.4925, | |
| "step": 1950 | |
| }, | |
| { | |
| "epoch": 1.675392670157068, | |
| "grad_norm": 3.2080323696136475, | |
| "learning_rate": 9.985000000000002e-06, | |
| "loss": 0.4851, | |
| "step": 2000 | |
| }, | |
| { | |
| "epoch": 1.675392670157068, | |
| "eval_loss": 0.43240800499916077, | |
| "eval_runtime": 256.4745, | |
| "eval_samples_per_second": 33.099, | |
| "eval_steps_per_second": 4.141, | |
| "step": 2000 | |
| }, | |
| { | |
| "epoch": 1.7172774869109948, | |
| "grad_norm": 2.5463037490844727, | |
| "learning_rate": 9.953000000000001e-06, | |
| "loss": 0.4802, | |
| "step": 2050 | |
| }, | |
| { | |
| "epoch": 1.7591623036649215, | |
| "grad_norm": 3.6697189807891846, | |
| "learning_rate": 9.903e-06, | |
| "loss": 0.485, | |
| "step": 2100 | |
| }, | |
| { | |
| "epoch": 1.8010471204188483, | |
| "grad_norm": 2.6455466747283936, | |
| "learning_rate": 9.853e-06, | |
| "loss": 0.4863, | |
| "step": 2150 | |
| }, | |
| { | |
| "epoch": 1.8429319371727748, | |
| "grad_norm": 2.849780321121216, | |
| "learning_rate": 9.803e-06, | |
| "loss": 0.4824, | |
| "step": 2200 | |
| }, | |
| { | |
| "epoch": 1.8848167539267016, | |
| "grad_norm": 2.037757158279419, | |
| "learning_rate": 9.753e-06, | |
| "loss": 0.4762, | |
| "step": 2250 | |
| }, | |
| { | |
| "epoch": 1.9267015706806283, | |
| "grad_norm": 2.060755491256714, | |
| "learning_rate": 9.703000000000002e-06, | |
| "loss": 0.4793, | |
| "step": 2300 | |
| }, | |
| { | |
| "epoch": 1.9685863874345548, | |
| "grad_norm": 2.3721907138824463, | |
| "learning_rate": 9.653e-06, | |
| "loss": 0.4744, | |
| "step": 2350 | |
| }, | |
| { | |
| "epoch": 2.0104712041884816, | |
| "grad_norm": 2.605724573135376, | |
| "learning_rate": 9.603000000000001e-06, | |
| "loss": 0.4665, | |
| "step": 2400 | |
| }, | |
| { | |
| "epoch": 2.0523560209424083, | |
| "grad_norm": 1.9419960975646973, | |
| "learning_rate": 9.553000000000002e-06, | |
| "loss": 0.4701, | |
| "step": 2450 | |
| }, | |
| { | |
| "epoch": 2.094240837696335, | |
| "grad_norm": 2.3074657917022705, | |
| "learning_rate": 9.503e-06, | |
| "loss": 0.4672, | |
| "step": 2500 | |
| }, | |
| { | |
| "epoch": 2.094240837696335, | |
| "eval_loss": 0.4174647927284241, | |
| "eval_runtime": 259.1813, | |
| "eval_samples_per_second": 32.753, | |
| "eval_steps_per_second": 4.098, | |
| "step": 2500 | |
| }, | |
| { | |
| "epoch": 2.136125654450262, | |
| "grad_norm": 20.03424644470215, | |
| "learning_rate": 9.453e-06, | |
| "loss": 0.4676, | |
| "step": 2550 | |
| }, | |
| { | |
| "epoch": 2.1780104712041886, | |
| "grad_norm": 3.410041093826294, | |
| "learning_rate": 9.403000000000001e-06, | |
| "loss": 0.471, | |
| "step": 2600 | |
| }, | |
| { | |
| "epoch": 2.2198952879581153, | |
| "grad_norm": 2.1835362911224365, | |
| "learning_rate": 9.353000000000002e-06, | |
| "loss": 0.4639, | |
| "step": 2650 | |
| }, | |
| { | |
| "epoch": 2.261780104712042, | |
| "grad_norm": 2.245004177093506, | |
| "learning_rate": 9.303e-06, | |
| "loss": 0.4642, | |
| "step": 2700 | |
| }, | |
| { | |
| "epoch": 2.303664921465969, | |
| "grad_norm": 2.2452590465545654, | |
| "learning_rate": 9.253000000000001e-06, | |
| "loss": 0.4638, | |
| "step": 2750 | |
| }, | |
| { | |
| "epoch": 2.345549738219895, | |
| "grad_norm": 3.5843396186828613, | |
| "learning_rate": 9.203000000000002e-06, | |
| "loss": 0.4546, | |
| "step": 2800 | |
| }, | |
| { | |
| "epoch": 2.387434554973822, | |
| "grad_norm": 1.689903974533081, | |
| "learning_rate": 9.153e-06, | |
| "loss": 0.4607, | |
| "step": 2850 | |
| }, | |
| { | |
| "epoch": 2.4293193717277486, | |
| "grad_norm": 2.26529860496521, | |
| "learning_rate": 9.103e-06, | |
| "loss": 0.4732, | |
| "step": 2900 | |
| }, | |
| { | |
| "epoch": 2.4712041884816753, | |
| "grad_norm": 1.669374942779541, | |
| "learning_rate": 9.053000000000001e-06, | |
| "loss": 0.4578, | |
| "step": 2950 | |
| }, | |
| { | |
| "epoch": 2.513089005235602, | |
| "grad_norm": 2.6614444255828857, | |
| "learning_rate": 9.003e-06, | |
| "loss": 0.4615, | |
| "step": 3000 | |
| }, | |
| { | |
| "epoch": 2.513089005235602, | |
| "eval_loss": 0.40531307458877563, | |
| "eval_runtime": 260.7294, | |
| "eval_samples_per_second": 32.559, | |
| "eval_steps_per_second": 4.073, | |
| "step": 3000 | |
| }, | |
| { | |
| "epoch": 2.554973821989529, | |
| "grad_norm": 2.6586270332336426, | |
| "learning_rate": 8.953e-06, | |
| "loss": 0.4486, | |
| "step": 3050 | |
| }, | |
| { | |
| "epoch": 2.5968586387434556, | |
| "grad_norm": 1.5337822437286377, | |
| "learning_rate": 8.903000000000001e-06, | |
| "loss": 0.4474, | |
| "step": 3100 | |
| }, | |
| { | |
| "epoch": 2.6387434554973823, | |
| "grad_norm": 5.568012714385986, | |
| "learning_rate": 8.853e-06, | |
| "loss": 0.4466, | |
| "step": 3150 | |
| }, | |
| { | |
| "epoch": 2.680628272251309, | |
| "grad_norm": 1.8297160863876343, | |
| "learning_rate": 8.803e-06, | |
| "loss": 0.454, | |
| "step": 3200 | |
| }, | |
| { | |
| "epoch": 2.7225130890052354, | |
| "grad_norm": 3.0298142433166504, | |
| "learning_rate": 8.753e-06, | |
| "loss": 0.4513, | |
| "step": 3250 | |
| }, | |
| { | |
| "epoch": 2.7643979057591626, | |
| "grad_norm": 2.6482794284820557, | |
| "learning_rate": 8.703e-06, | |
| "loss": 0.4461, | |
| "step": 3300 | |
| }, | |
| { | |
| "epoch": 2.806282722513089, | |
| "grad_norm": 1.9304325580596924, | |
| "learning_rate": 8.653e-06, | |
| "loss": 0.4464, | |
| "step": 3350 | |
| }, | |
| { | |
| "epoch": 2.8481675392670156, | |
| "grad_norm": 2.3229286670684814, | |
| "learning_rate": 8.603e-06, | |
| "loss": 0.4475, | |
| "step": 3400 | |
| }, | |
| { | |
| "epoch": 2.8900523560209423, | |
| "grad_norm": 1.8658236265182495, | |
| "learning_rate": 8.553000000000001e-06, | |
| "loss": 0.4476, | |
| "step": 3450 | |
| }, | |
| { | |
| "epoch": 2.931937172774869, | |
| "grad_norm": 1.574404239654541, | |
| "learning_rate": 8.503e-06, | |
| "loss": 0.4426, | |
| "step": 3500 | |
| }, | |
| { | |
| "epoch": 2.931937172774869, | |
| "eval_loss": 0.39671415090560913, | |
| "eval_runtime": 259.1033, | |
| "eval_samples_per_second": 32.763, | |
| "eval_steps_per_second": 4.099, | |
| "step": 3500 | |
| }, | |
| { | |
| "epoch": 2.973821989528796, | |
| "grad_norm": 2.0523784160614014, | |
| "learning_rate": 8.453000000000002e-06, | |
| "loss": 0.4545, | |
| "step": 3550 | |
| }, | |
| { | |
| "epoch": 3.0157068062827226, | |
| "grad_norm": 2.5431597232818604, | |
| "learning_rate": 8.403e-06, | |
| "loss": 0.4416, | |
| "step": 3600 | |
| }, | |
| { | |
| "epoch": 3.0575916230366493, | |
| "grad_norm": 1.7480727434158325, | |
| "learning_rate": 8.353000000000001e-06, | |
| "loss": 0.4396, | |
| "step": 3650 | |
| }, | |
| { | |
| "epoch": 3.099476439790576, | |
| "grad_norm": 2.19262433052063, | |
| "learning_rate": 8.303000000000002e-06, | |
| "loss": 0.4403, | |
| "step": 3700 | |
| }, | |
| { | |
| "epoch": 3.141361256544503, | |
| "grad_norm": 1.6949037313461304, | |
| "learning_rate": 8.253e-06, | |
| "loss": 0.4345, | |
| "step": 3750 | |
| }, | |
| { | |
| "epoch": 3.183246073298429, | |
| "grad_norm": 3.506704807281494, | |
| "learning_rate": 8.203000000000001e-06, | |
| "loss": 0.4364, | |
| "step": 3800 | |
| }, | |
| { | |
| "epoch": 3.225130890052356, | |
| "grad_norm": 2.155595064163208, | |
| "learning_rate": 8.153000000000001e-06, | |
| "loss": 0.4361, | |
| "step": 3850 | |
| }, | |
| { | |
| "epoch": 3.2670157068062826, | |
| "grad_norm": 1.7618342638015747, | |
| "learning_rate": 8.103e-06, | |
| "loss": 0.4348, | |
| "step": 3900 | |
| }, | |
| { | |
| "epoch": 3.3089005235602094, | |
| "grad_norm": 2.2834861278533936, | |
| "learning_rate": 8.053e-06, | |
| "loss": 0.4435, | |
| "step": 3950 | |
| }, | |
| { | |
| "epoch": 3.350785340314136, | |
| "grad_norm": 1.7081154584884644, | |
| "learning_rate": 8.003000000000001e-06, | |
| "loss": 0.4295, | |
| "step": 4000 | |
| }, | |
| { | |
| "epoch": 3.350785340314136, | |
| "eval_loss": 0.389825701713562, | |
| "eval_runtime": 254.7511, | |
| "eval_samples_per_second": 33.323, | |
| "eval_steps_per_second": 4.169, | |
| "step": 4000 | |
| }, | |
| { | |
| "epoch": 3.392670157068063, | |
| "grad_norm": 2.6197593212127686, | |
| "learning_rate": 7.953e-06, | |
| "loss": 0.4312, | |
| "step": 4050 | |
| }, | |
| { | |
| "epoch": 3.4345549738219896, | |
| "grad_norm": 1.742619514465332, | |
| "learning_rate": 7.903e-06, | |
| "loss": 0.432, | |
| "step": 4100 | |
| }, | |
| { | |
| "epoch": 3.4764397905759163, | |
| "grad_norm": 1.730943202972412, | |
| "learning_rate": 7.853000000000001e-06, | |
| "loss": 0.4376, | |
| "step": 4150 | |
| }, | |
| { | |
| "epoch": 3.518324607329843, | |
| "grad_norm": 2.4986751079559326, | |
| "learning_rate": 7.803000000000001e-06, | |
| "loss": 0.4344, | |
| "step": 4200 | |
| }, | |
| { | |
| "epoch": 3.5602094240837694, | |
| "grad_norm": 1.707566499710083, | |
| "learning_rate": 7.753e-06, | |
| "loss": 0.435, | |
| "step": 4250 | |
| }, | |
| { | |
| "epoch": 3.6020942408376966, | |
| "grad_norm": 2.230375051498413, | |
| "learning_rate": 7.703e-06, | |
| "loss": 0.4286, | |
| "step": 4300 | |
| }, | |
| { | |
| "epoch": 3.643979057591623, | |
| "grad_norm": 2.0870070457458496, | |
| "learning_rate": 7.653000000000001e-06, | |
| "loss": 0.4302, | |
| "step": 4350 | |
| }, | |
| { | |
| "epoch": 3.6858638743455496, | |
| "grad_norm": 2.5888140201568604, | |
| "learning_rate": 7.603000000000001e-06, | |
| "loss": 0.4333, | |
| "step": 4400 | |
| }, | |
| { | |
| "epoch": 3.7277486910994764, | |
| "grad_norm": 3.0323987007141113, | |
| "learning_rate": 7.553e-06, | |
| "loss": 0.4296, | |
| "step": 4450 | |
| }, | |
| { | |
| "epoch": 3.769633507853403, | |
| "grad_norm": 2.430225133895874, | |
| "learning_rate": 7.503e-06, | |
| "loss": 0.4323, | |
| "step": 4500 | |
| }, | |
| { | |
| "epoch": 3.769633507853403, | |
| "eval_loss": 0.38729745149612427, | |
| "eval_runtime": 254.688, | |
| "eval_samples_per_second": 33.331, | |
| "eval_steps_per_second": 4.17, | |
| "step": 4500 | |
| } | |
| ], | |
| "logging_steps": 50, | |
| "max_steps": 12000, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 11, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": false | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 4.04828814811703e+16, | |
| "train_batch_size": 16, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |