zlm_b64_le5_s12000 / checkpoint-4000 /trainer_state.json
mikhail-panzo's picture
Training in progress, step 4000, checkpoint
5f6c04a verified
raw
history blame
15.9 kB
{
"best_metric": 0.389825701713562,
"best_model_checkpoint": "mikhail_panzo/zlm_b64_le5_s12000/checkpoint-4000",
"epoch": 3.350785340314136,
"eval_steps": 500,
"global_step": 4000,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.041884816753926704,
"grad_norm": 12.821951866149902,
"learning_rate": 2.4000000000000003e-07,
"loss": 1.1152,
"step": 50
},
{
"epoch": 0.08376963350785341,
"grad_norm": 6.186221122741699,
"learning_rate": 4.900000000000001e-07,
"loss": 1.0584,
"step": 100
},
{
"epoch": 0.1256544502617801,
"grad_norm": 13.08674430847168,
"learning_rate": 7.4e-07,
"loss": 0.9996,
"step": 150
},
{
"epoch": 0.16753926701570682,
"grad_norm": 4.10632848739624,
"learning_rate": 9.9e-07,
"loss": 0.9355,
"step": 200
},
{
"epoch": 0.2094240837696335,
"grad_norm": 3.2428064346313477,
"learning_rate": 1.2400000000000002e-06,
"loss": 0.852,
"step": 250
},
{
"epoch": 0.2513089005235602,
"grad_norm": 2.0861401557922363,
"learning_rate": 1.4900000000000001e-06,
"loss": 0.8241,
"step": 300
},
{
"epoch": 0.2931937172774869,
"grad_norm": 8.602317810058594,
"learning_rate": 1.74e-06,
"loss": 0.7629,
"step": 350
},
{
"epoch": 0.33507853403141363,
"grad_norm": 2.7594990730285645,
"learning_rate": 1.9900000000000004e-06,
"loss": 0.7418,
"step": 400
},
{
"epoch": 0.3769633507853403,
"grad_norm": 2.7047348022460938,
"learning_rate": 2.24e-06,
"loss": 0.7166,
"step": 450
},
{
"epoch": 0.418848167539267,
"grad_norm": 2.0666158199310303,
"learning_rate": 2.4900000000000003e-06,
"loss": 0.6876,
"step": 500
},
{
"epoch": 0.418848167539267,
"eval_loss": 0.5937426090240479,
"eval_runtime": 263.0955,
"eval_samples_per_second": 32.266,
"eval_steps_per_second": 4.037,
"step": 500
},
{
"epoch": 0.4607329842931937,
"grad_norm": 2.955193519592285,
"learning_rate": 2.7400000000000004e-06,
"loss": 0.673,
"step": 550
},
{
"epoch": 0.5026178010471204,
"grad_norm": 4.856322765350342,
"learning_rate": 2.99e-06,
"loss": 0.6539,
"step": 600
},
{
"epoch": 0.5445026178010471,
"grad_norm": 7.631103992462158,
"learning_rate": 3.2400000000000003e-06,
"loss": 0.6588,
"step": 650
},
{
"epoch": 0.5863874345549738,
"grad_norm": 2.715771436691284,
"learning_rate": 3.49e-06,
"loss": 0.624,
"step": 700
},
{
"epoch": 0.6282722513089005,
"grad_norm": 2.7684273719787598,
"learning_rate": 3.74e-06,
"loss": 0.6116,
"step": 750
},
{
"epoch": 0.6701570680628273,
"grad_norm": 2.219599723815918,
"learning_rate": 3.990000000000001e-06,
"loss": 0.6097,
"step": 800
},
{
"epoch": 0.7120418848167539,
"grad_norm": 3.749188184738159,
"learning_rate": 4.24e-06,
"loss": 0.614,
"step": 850
},
{
"epoch": 0.7539267015706806,
"grad_norm": 2.120619773864746,
"learning_rate": 4.49e-06,
"loss": 0.5814,
"step": 900
},
{
"epoch": 0.7958115183246073,
"grad_norm": 2.7725327014923096,
"learning_rate": 4.74e-06,
"loss": 0.5895,
"step": 950
},
{
"epoch": 0.837696335078534,
"grad_norm": 9.851961135864258,
"learning_rate": 4.9900000000000005e-06,
"loss": 0.5623,
"step": 1000
},
{
"epoch": 0.837696335078534,
"eval_loss": 0.4933945834636688,
"eval_runtime": 258.739,
"eval_samples_per_second": 32.809,
"eval_steps_per_second": 4.105,
"step": 1000
},
{
"epoch": 0.8795811518324608,
"grad_norm": 5.074089527130127,
"learning_rate": 5.240000000000001e-06,
"loss": 0.5748,
"step": 1050
},
{
"epoch": 0.9214659685863874,
"grad_norm": 5.884639739990234,
"learning_rate": 5.485e-06,
"loss": 0.5395,
"step": 1100
},
{
"epoch": 0.9633507853403142,
"grad_norm": 3.1588447093963623,
"learning_rate": 5.735e-06,
"loss": 0.5478,
"step": 1150
},
{
"epoch": 1.0052356020942408,
"grad_norm": 2.4452970027923584,
"learning_rate": 5.985000000000001e-06,
"loss": 0.538,
"step": 1200
},
{
"epoch": 1.0471204188481675,
"grad_norm": 8.290769577026367,
"learning_rate": 6.235000000000001e-06,
"loss": 0.5411,
"step": 1250
},
{
"epoch": 1.0890052356020943,
"grad_norm": 4.080046653747559,
"learning_rate": 6.485000000000001e-06,
"loss": 0.5387,
"step": 1300
},
{
"epoch": 1.130890052356021,
"grad_norm": 2.7520201206207275,
"learning_rate": 6.735000000000001e-06,
"loss": 0.5426,
"step": 1350
},
{
"epoch": 1.1727748691099475,
"grad_norm": 3.8192107677459717,
"learning_rate": 6.985000000000001e-06,
"loss": 0.5315,
"step": 1400
},
{
"epoch": 1.2146596858638743,
"grad_norm": 9.535676956176758,
"learning_rate": 7.235000000000001e-06,
"loss": 0.5159,
"step": 1450
},
{
"epoch": 1.256544502617801,
"grad_norm": 2.5731022357940674,
"learning_rate": 7.485000000000001e-06,
"loss": 0.5165,
"step": 1500
},
{
"epoch": 1.256544502617801,
"eval_loss": 0.4628298878669739,
"eval_runtime": 257.0265,
"eval_samples_per_second": 33.028,
"eval_steps_per_second": 4.132,
"step": 1500
},
{
"epoch": 1.2984293193717278,
"grad_norm": 3.662090539932251,
"learning_rate": 7.735e-06,
"loss": 0.5188,
"step": 1550
},
{
"epoch": 1.3403141361256545,
"grad_norm": 3.1680572032928467,
"learning_rate": 7.985e-06,
"loss": 0.5148,
"step": 1600
},
{
"epoch": 1.3821989528795813,
"grad_norm": 3.601471185684204,
"learning_rate": 8.235e-06,
"loss": 0.5127,
"step": 1650
},
{
"epoch": 1.4240837696335078,
"grad_norm": 4.497046947479248,
"learning_rate": 8.485000000000001e-06,
"loss": 0.5039,
"step": 1700
},
{
"epoch": 1.4659685863874345,
"grad_norm": 3.6496424674987793,
"learning_rate": 8.735000000000002e-06,
"loss": 0.5114,
"step": 1750
},
{
"epoch": 1.5078534031413613,
"grad_norm": 2.7080061435699463,
"learning_rate": 8.985000000000001e-06,
"loss": 0.4953,
"step": 1800
},
{
"epoch": 1.5497382198952878,
"grad_norm": 2.5439274311065674,
"learning_rate": 9.235e-06,
"loss": 0.497,
"step": 1850
},
{
"epoch": 1.5916230366492146,
"grad_norm": 2.004573106765747,
"learning_rate": 9.485000000000002e-06,
"loss": 0.4889,
"step": 1900
},
{
"epoch": 1.6335078534031413,
"grad_norm": 4.699184894561768,
"learning_rate": 9.735e-06,
"loss": 0.4925,
"step": 1950
},
{
"epoch": 1.675392670157068,
"grad_norm": 3.2080323696136475,
"learning_rate": 9.985000000000002e-06,
"loss": 0.4851,
"step": 2000
},
{
"epoch": 1.675392670157068,
"eval_loss": 0.43240800499916077,
"eval_runtime": 256.4745,
"eval_samples_per_second": 33.099,
"eval_steps_per_second": 4.141,
"step": 2000
},
{
"epoch": 1.7172774869109948,
"grad_norm": 2.5463037490844727,
"learning_rate": 9.953000000000001e-06,
"loss": 0.4802,
"step": 2050
},
{
"epoch": 1.7591623036649215,
"grad_norm": 3.6697189807891846,
"learning_rate": 9.903e-06,
"loss": 0.485,
"step": 2100
},
{
"epoch": 1.8010471204188483,
"grad_norm": 2.6455466747283936,
"learning_rate": 9.853e-06,
"loss": 0.4863,
"step": 2150
},
{
"epoch": 1.8429319371727748,
"grad_norm": 2.849780321121216,
"learning_rate": 9.803e-06,
"loss": 0.4824,
"step": 2200
},
{
"epoch": 1.8848167539267016,
"grad_norm": 2.037757158279419,
"learning_rate": 9.753e-06,
"loss": 0.4762,
"step": 2250
},
{
"epoch": 1.9267015706806283,
"grad_norm": 2.060755491256714,
"learning_rate": 9.703000000000002e-06,
"loss": 0.4793,
"step": 2300
},
{
"epoch": 1.9685863874345548,
"grad_norm": 2.3721907138824463,
"learning_rate": 9.653e-06,
"loss": 0.4744,
"step": 2350
},
{
"epoch": 2.0104712041884816,
"grad_norm": 2.605724573135376,
"learning_rate": 9.603000000000001e-06,
"loss": 0.4665,
"step": 2400
},
{
"epoch": 2.0523560209424083,
"grad_norm": 1.9419960975646973,
"learning_rate": 9.553000000000002e-06,
"loss": 0.4701,
"step": 2450
},
{
"epoch": 2.094240837696335,
"grad_norm": 2.3074657917022705,
"learning_rate": 9.503e-06,
"loss": 0.4672,
"step": 2500
},
{
"epoch": 2.094240837696335,
"eval_loss": 0.4174647927284241,
"eval_runtime": 259.1813,
"eval_samples_per_second": 32.753,
"eval_steps_per_second": 4.098,
"step": 2500
},
{
"epoch": 2.136125654450262,
"grad_norm": 20.03424644470215,
"learning_rate": 9.453e-06,
"loss": 0.4676,
"step": 2550
},
{
"epoch": 2.1780104712041886,
"grad_norm": 3.410041093826294,
"learning_rate": 9.403000000000001e-06,
"loss": 0.471,
"step": 2600
},
{
"epoch": 2.2198952879581153,
"grad_norm": 2.1835362911224365,
"learning_rate": 9.353000000000002e-06,
"loss": 0.4639,
"step": 2650
},
{
"epoch": 2.261780104712042,
"grad_norm": 2.245004177093506,
"learning_rate": 9.303e-06,
"loss": 0.4642,
"step": 2700
},
{
"epoch": 2.303664921465969,
"grad_norm": 2.2452590465545654,
"learning_rate": 9.253000000000001e-06,
"loss": 0.4638,
"step": 2750
},
{
"epoch": 2.345549738219895,
"grad_norm": 3.5843396186828613,
"learning_rate": 9.203000000000002e-06,
"loss": 0.4546,
"step": 2800
},
{
"epoch": 2.387434554973822,
"grad_norm": 1.689903974533081,
"learning_rate": 9.153e-06,
"loss": 0.4607,
"step": 2850
},
{
"epoch": 2.4293193717277486,
"grad_norm": 2.26529860496521,
"learning_rate": 9.103e-06,
"loss": 0.4732,
"step": 2900
},
{
"epoch": 2.4712041884816753,
"grad_norm": 1.669374942779541,
"learning_rate": 9.053000000000001e-06,
"loss": 0.4578,
"step": 2950
},
{
"epoch": 2.513089005235602,
"grad_norm": 2.6614444255828857,
"learning_rate": 9.003e-06,
"loss": 0.4615,
"step": 3000
},
{
"epoch": 2.513089005235602,
"eval_loss": 0.40531307458877563,
"eval_runtime": 260.7294,
"eval_samples_per_second": 32.559,
"eval_steps_per_second": 4.073,
"step": 3000
},
{
"epoch": 2.554973821989529,
"grad_norm": 2.6586270332336426,
"learning_rate": 8.953e-06,
"loss": 0.4486,
"step": 3050
},
{
"epoch": 2.5968586387434556,
"grad_norm": 1.5337822437286377,
"learning_rate": 8.903000000000001e-06,
"loss": 0.4474,
"step": 3100
},
{
"epoch": 2.6387434554973823,
"grad_norm": 5.568012714385986,
"learning_rate": 8.853e-06,
"loss": 0.4466,
"step": 3150
},
{
"epoch": 2.680628272251309,
"grad_norm": 1.8297160863876343,
"learning_rate": 8.803e-06,
"loss": 0.454,
"step": 3200
},
{
"epoch": 2.7225130890052354,
"grad_norm": 3.0298142433166504,
"learning_rate": 8.753e-06,
"loss": 0.4513,
"step": 3250
},
{
"epoch": 2.7643979057591626,
"grad_norm": 2.6482794284820557,
"learning_rate": 8.703e-06,
"loss": 0.4461,
"step": 3300
},
{
"epoch": 2.806282722513089,
"grad_norm": 1.9304325580596924,
"learning_rate": 8.653e-06,
"loss": 0.4464,
"step": 3350
},
{
"epoch": 2.8481675392670156,
"grad_norm": 2.3229286670684814,
"learning_rate": 8.603e-06,
"loss": 0.4475,
"step": 3400
},
{
"epoch": 2.8900523560209423,
"grad_norm": 1.8658236265182495,
"learning_rate": 8.553000000000001e-06,
"loss": 0.4476,
"step": 3450
},
{
"epoch": 2.931937172774869,
"grad_norm": 1.574404239654541,
"learning_rate": 8.503e-06,
"loss": 0.4426,
"step": 3500
},
{
"epoch": 2.931937172774869,
"eval_loss": 0.39671415090560913,
"eval_runtime": 259.1033,
"eval_samples_per_second": 32.763,
"eval_steps_per_second": 4.099,
"step": 3500
},
{
"epoch": 2.973821989528796,
"grad_norm": 2.0523784160614014,
"learning_rate": 8.453000000000002e-06,
"loss": 0.4545,
"step": 3550
},
{
"epoch": 3.0157068062827226,
"grad_norm": 2.5431597232818604,
"learning_rate": 8.403e-06,
"loss": 0.4416,
"step": 3600
},
{
"epoch": 3.0575916230366493,
"grad_norm": 1.7480727434158325,
"learning_rate": 8.353000000000001e-06,
"loss": 0.4396,
"step": 3650
},
{
"epoch": 3.099476439790576,
"grad_norm": 2.19262433052063,
"learning_rate": 8.303000000000002e-06,
"loss": 0.4403,
"step": 3700
},
{
"epoch": 3.141361256544503,
"grad_norm": 1.6949037313461304,
"learning_rate": 8.253e-06,
"loss": 0.4345,
"step": 3750
},
{
"epoch": 3.183246073298429,
"grad_norm": 3.506704807281494,
"learning_rate": 8.203000000000001e-06,
"loss": 0.4364,
"step": 3800
},
{
"epoch": 3.225130890052356,
"grad_norm": 2.155595064163208,
"learning_rate": 8.153000000000001e-06,
"loss": 0.4361,
"step": 3850
},
{
"epoch": 3.2670157068062826,
"grad_norm": 1.7618342638015747,
"learning_rate": 8.103e-06,
"loss": 0.4348,
"step": 3900
},
{
"epoch": 3.3089005235602094,
"grad_norm": 2.2834861278533936,
"learning_rate": 8.053e-06,
"loss": 0.4435,
"step": 3950
},
{
"epoch": 3.350785340314136,
"grad_norm": 1.7081154584884644,
"learning_rate": 8.003000000000001e-06,
"loss": 0.4295,
"step": 4000
},
{
"epoch": 3.350785340314136,
"eval_loss": 0.389825701713562,
"eval_runtime": 254.7511,
"eval_samples_per_second": 33.323,
"eval_steps_per_second": 4.169,
"step": 4000
}
],
"logging_steps": 50,
"max_steps": 12000,
"num_input_tokens_seen": 0,
"num_train_epochs": 11,
"save_steps": 500,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 3.597539584997952e+16,
"train_batch_size": 16,
"trial_name": null,
"trial_params": null
}