File size: 14,659 Bytes
88a4513 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 |
{
"best_metric": 7.4231791496276855,
"best_model_checkpoint": "./qwen2.5-0.5b/qwen2.5-0.5b-expo-DPO-ES-0.1/checkpoint-300",
"epoch": 1.5588096362777515,
"eval_steps": 50,
"global_step": 550,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"dpo_loss": 0.6931471824645996,
"epoch": 0.002834199338686821,
"grad_norm": 18.44253347826331,
"learning_rate": 2.840909090909091e-08,
"logits": -1.359458565711975,
"logps": -84.69721221923828,
"loss": 0.6931,
"objective": 0.6931471824645996,
"ranking_idealized": 0.6458333134651184,
"ranking_idealized_expo": 0.5833333134651184,
"ranking_simple": 0.5833333134651184,
"regularize": 0.6931471824645996,
"step": 1,
"wo_beta": 5.271125316619873
},
{
"dpo_loss": 0.6822353601455688,
"epoch": 0.14170996693434104,
"grad_norm": 18.641365531241362,
"learning_rate": 1.4204545454545458e-06,
"logits": -1.508646011352539,
"logps": -82.822021484375,
"loss": 0.6785,
"objective": 0.6822353601455688,
"ranking_idealized": 0.608418345451355,
"ranking_idealized_expo": 0.5229591727256775,
"ranking_simple": 0.5267857313156128,
"regularize": 0.6822353601455688,
"step": 50,
"wo_beta": 7.122643947601318
},
{
"epoch": 0.14170996693434104,
"eval_dpo_loss": 0.6842507719993591,
"eval_logits": -1.6022367477416992,
"eval_logps": -90.87205505371094,
"eval_loss": 0.6813501715660095,
"eval_objective": 0.6842507719993591,
"eval_ranking_idealized": 0.6030020713806152,
"eval_ranking_idealized_expo": 0.5222567319869995,
"eval_ranking_simple": 0.5258799195289612,
"eval_regularize": 0.6842507719993591,
"eval_runtime": 308.2905,
"eval_samples_per_second": 18.781,
"eval_steps_per_second": 1.567,
"eval_wo_beta": 7.874889850616455,
"step": 50
},
{
"dpo_loss": 0.61030513048172,
"epoch": 0.2834199338686821,
"grad_norm": 19.29211957833529,
"learning_rate": 2.8409090909090916e-06,
"logits": -1.6967747211456299,
"logps": -88.03068542480469,
"loss": 0.618,
"objective": 0.61030513048172,
"ranking_idealized": 0.6016666889190674,
"ranking_idealized_expo": 0.5141666531562805,
"ranking_simple": 0.5729166865348816,
"regularize": 0.61030513048172,
"step": 100,
"wo_beta": 6.357723236083984
},
{
"epoch": 0.2834199338686821,
"eval_dpo_loss": 0.6765881776809692,
"eval_logits": -1.7799152135849,
"eval_logps": -98.88995361328125,
"eval_loss": 0.6732921004295349,
"eval_objective": 0.6765881776809692,
"eval_ranking_idealized": 0.6030020713806152,
"eval_ranking_idealized_expo": 0.5222567319869995,
"eval_ranking_simple": 0.5398550629615784,
"eval_regularize": 0.6765881776809692,
"eval_runtime": 307.7771,
"eval_samples_per_second": 18.812,
"eval_steps_per_second": 1.569,
"eval_wo_beta": 7.7840118408203125,
"step": 100
},
{
"dpo_loss": 0.5696364045143127,
"epoch": 0.42512990080302315,
"grad_norm": 18.142654654279198,
"learning_rate": 4.2613636363636365e-06,
"logits": -1.7447518110275269,
"logps": -90.71894073486328,
"loss": 0.5667,
"objective": 0.5696364045143127,
"ranking_idealized": 0.6066666841506958,
"ranking_idealized_expo": 0.5287500023841858,
"ranking_simple": 0.6016666889190674,
"regularize": 0.5696364045143127,
"step": 150,
"wo_beta": 5.971243858337402
},
{
"epoch": 0.42512990080302315,
"eval_dpo_loss": 0.6829443573951721,
"eval_logits": -1.807220697402954,
"eval_logps": -99.12174987792969,
"eval_loss": 0.686660647392273,
"eval_objective": 0.6829443573951721,
"eval_ranking_idealized": 0.6030020713806152,
"eval_ranking_idealized_expo": 0.5222567319869995,
"eval_ranking_simple": 0.5408902764320374,
"eval_regularize": 0.6829443573951721,
"eval_runtime": 308.0058,
"eval_samples_per_second": 18.798,
"eval_steps_per_second": 1.568,
"eval_wo_beta": 7.853672981262207,
"step": 150
},
{
"dpo_loss": 0.5278546214103699,
"epoch": 0.5668398677373642,
"grad_norm": 15.355172339669656,
"learning_rate": 4.997168347957521e-06,
"logits": -1.9084746837615967,
"logps": -92.50672149658203,
"loss": 0.5214,
"objective": 0.5278546214103699,
"ranking_idealized": 0.5924999713897705,
"ranking_idealized_expo": 0.5166666507720947,
"ranking_simple": 0.6362500190734863,
"regularize": 0.5278546214103699,
"step": 200,
"wo_beta": 5.543394565582275
},
{
"epoch": 0.5668398677373642,
"eval_dpo_loss": 0.69049072265625,
"eval_logits": -1.8895256519317627,
"eval_logps": -99.51531982421875,
"eval_loss": 0.6901801824569702,
"eval_objective": 0.69049072265625,
"eval_ranking_idealized": 0.6030020713806152,
"eval_ranking_idealized_expo": 0.5222567319869995,
"eval_ranking_simple": 0.544513463973999,
"eval_regularize": 0.69049072265625,
"eval_runtime": 308.0169,
"eval_samples_per_second": 18.798,
"eval_steps_per_second": 1.568,
"eval_wo_beta": 7.701313495635986,
"step": 200
},
{
"dpo_loss": 0.4935472011566162,
"epoch": 0.7085498346717053,
"grad_norm": 14.324676993767012,
"learning_rate": 4.973122855144066e-06,
"logits": -1.9081355333328247,
"logps": -88.3556137084961,
"loss": 0.4922,
"objective": 0.4935472011566162,
"ranking_idealized": 0.5991666913032532,
"ranking_idealized_expo": 0.5170833468437195,
"ranking_simple": 0.6608333587646484,
"regularize": 0.4935472011566162,
"step": 250,
"wo_beta": 5.325418472290039
},
{
"epoch": 0.7085498346717053,
"eval_dpo_loss": 0.6914450526237488,
"eval_logits": -1.9887231588363647,
"eval_logps": -82.8383560180664,
"eval_loss": 0.6975522041320801,
"eval_objective": 0.6914450526237488,
"eval_ranking_idealized": 0.6030020713806152,
"eval_ranking_idealized_expo": 0.5222567319869995,
"eval_ranking_simple": 0.5481366515159607,
"eval_regularize": 0.6914450526237488,
"eval_runtime": 307.641,
"eval_samples_per_second": 18.821,
"eval_steps_per_second": 1.57,
"eval_wo_beta": 7.878448486328125,
"step": 250
},
{
"dpo_loss": 0.4521400034427643,
"epoch": 0.8502598016060463,
"grad_norm": 13.428528764338076,
"learning_rate": 4.924776641419513e-06,
"logits": -2.0954272747039795,
"logps": -81.39044952392578,
"loss": 0.4535,
"objective": 0.4521400034427643,
"ranking_idealized": 0.5799999833106995,
"ranking_idealized_expo": 0.4970833361148834,
"ranking_simple": 0.6837499737739563,
"regularize": 0.4521400034427643,
"step": 300,
"wo_beta": 5.026640892028809
},
{
"epoch": 0.8502598016060463,
"eval_dpo_loss": 0.6893911957740784,
"eval_logits": -2.1208713054656982,
"eval_logps": -90.9490966796875,
"eval_loss": 0.6922824382781982,
"eval_objective": 0.6893911957740784,
"eval_ranking_idealized": 0.6030020713806152,
"eval_ranking_idealized_expo": 0.5222567319869995,
"eval_ranking_simple": 0.556418240070343,
"eval_regularize": 0.6893911957740784,
"eval_runtime": 307.9716,
"eval_samples_per_second": 18.8,
"eval_steps_per_second": 1.568,
"eval_wo_beta": 7.4231791496276855,
"step": 300
},
{
"dpo_loss": 0.40847164392471313,
"epoch": 0.9919697685403873,
"grad_norm": 15.041732370189118,
"learning_rate": 4.8526047530778175e-06,
"logits": -1.9279303550720215,
"logps": -84.23763275146484,
"loss": 0.4228,
"objective": 0.40847164392471313,
"ranking_idealized": 0.60916668176651,
"ranking_idealized_expo": 0.5270833373069763,
"ranking_simple": 0.7191666960716248,
"regularize": 0.40847164392471313,
"step": 350,
"wo_beta": 3.9990389347076416
},
{
"epoch": 0.9919697685403873,
"eval_dpo_loss": 0.6968410611152649,
"eval_logits": -1.9802873134613037,
"eval_logps": -87.72307586669922,
"eval_loss": 0.7063526511192322,
"eval_objective": 0.6968410611152649,
"eval_ranking_idealized": 0.6030020713806152,
"eval_ranking_idealized_expo": 0.5222567319869995,
"eval_ranking_simple": 0.5538302063941956,
"eval_regularize": 0.6968410611152649,
"eval_runtime": 307.977,
"eval_samples_per_second": 18.8,
"eval_steps_per_second": 1.568,
"eval_wo_beta": 8.025344848632812,
"step": 350
},
{
"dpo_loss": 0.28143200278282166,
"epoch": 1.1336797354747283,
"grad_norm": 12.112116162894052,
"learning_rate": 4.757316345716554e-06,
"logits": -2.0127880573272705,
"logps": -91.47932434082031,
"loss": 0.2845,
"objective": 0.28143200278282166,
"ranking_idealized": 0.6087499856948853,
"ranking_idealized_expo": 0.5337499976158142,
"ranking_simple": 0.8141666650772095,
"regularize": 0.28143200278282166,
"step": 400,
"wo_beta": 2.649115562438965
},
{
"epoch": 1.1336797354747283,
"eval_dpo_loss": 0.7269378900527954,
"eval_logits": -2.080526828765869,
"eval_logps": -101.31802368164062,
"eval_loss": 0.7304782867431641,
"eval_objective": 0.7269378900527954,
"eval_ranking_idealized": 0.6030020713806152,
"eval_ranking_idealized_expo": 0.5222567319869995,
"eval_ranking_simple": 0.5429606437683105,
"eval_regularize": 0.7269378900527954,
"eval_runtime": 308.0503,
"eval_samples_per_second": 18.796,
"eval_steps_per_second": 1.568,
"eval_wo_beta": 8.616350173950195,
"step": 400
},
{
"dpo_loss": 0.3042532503604889,
"epoch": 1.2753897024090695,
"grad_norm": 12.171625264502326,
"learning_rate": 4.639847716126855e-06,
"logits": -1.9084649085998535,
"logps": -92.91566467285156,
"loss": 0.2989,
"objective": 0.3042532503604889,
"ranking_idealized": 0.5975000262260437,
"ranking_idealized_expo": 0.5199999809265137,
"ranking_simple": 0.8075000047683716,
"regularize": 0.3042532503604889,
"step": 450,
"wo_beta": 3.341869592666626
},
{
"epoch": 1.2753897024090695,
"eval_dpo_loss": 0.6973706483840942,
"eval_logits": -1.8646337985992432,
"eval_logps": -93.1955337524414,
"eval_loss": 0.7005103826522827,
"eval_objective": 0.6973706483840942,
"eval_ranking_idealized": 0.6030020713806152,
"eval_ranking_idealized_expo": 0.5222567319869995,
"eval_ranking_simple": 0.5605590343475342,
"eval_regularize": 0.6973706483840942,
"eval_runtime": 308.4071,
"eval_samples_per_second": 18.774,
"eval_steps_per_second": 1.566,
"eval_wo_beta": 8.238639831542969,
"step": 450
},
{
"dpo_loss": 0.29817140102386475,
"epoch": 1.4170996693434104,
"grad_norm": 15.895661459470155,
"learning_rate": 4.501353102310901e-06,
"logits": -1.9325114488601685,
"logps": -91.53684997558594,
"loss": 0.3065,
"objective": 0.29817140102386475,
"ranking_idealized": 0.57833331823349,
"ranking_idealized_expo": 0.4983333349227905,
"ranking_simple": 0.7975000143051147,
"regularize": 0.29817140102386475,
"step": 500,
"wo_beta": 3.1243510246276855
},
{
"epoch": 1.4170996693434104,
"eval_dpo_loss": 0.7146824598312378,
"eval_logits": -1.9982556104660034,
"eval_logps": -97.01371002197266,
"eval_loss": 0.717850923538208,
"eval_objective": 0.7146824598312378,
"eval_ranking_idealized": 0.6030020713806152,
"eval_ranking_idealized_expo": 0.5222567319869995,
"eval_ranking_simple": 0.5548654198646545,
"eval_regularize": 0.7146824598312378,
"eval_runtime": 308.8392,
"eval_samples_per_second": 18.748,
"eval_steps_per_second": 1.564,
"eval_wo_beta": 8.27602767944336,
"step": 500
},
{
"dpo_loss": 0.2845906615257263,
"epoch": 1.5588096362777515,
"grad_norm": 10.66218965165015,
"learning_rate": 4.34319334202531e-06,
"logits": -1.9555292129516602,
"logps": -97.59473419189453,
"loss": 0.2885,
"objective": 0.2845906615257263,
"ranking_idealized": 0.5945833325386047,
"ranking_idealized_expo": 0.5116666555404663,
"ranking_simple": 0.8041666746139526,
"regularize": 0.2845906615257263,
"step": 550,
"wo_beta": 2.6915340423583984
},
{
"epoch": 1.5588096362777515,
"eval_dpo_loss": 0.7133627533912659,
"eval_logits": -1.9041162729263306,
"eval_logps": -107.96095275878906,
"eval_loss": 0.7091230750083923,
"eval_objective": 0.7133627533912659,
"eval_ranking_idealized": 0.6030020713806152,
"eval_ranking_idealized_expo": 0.5222567319869995,
"eval_ranking_simple": 0.5615941882133484,
"eval_regularize": 0.7133627533912659,
"eval_runtime": 308.0507,
"eval_samples_per_second": 18.796,
"eval_steps_per_second": 1.568,
"eval_wo_beta": 8.196797370910645,
"step": 550
},
{
"epoch": 1.5588096362777515,
"step": 550,
"total_flos": 0.0,
"train_loss": 0.4483427975394509,
"train_runtime": 15087.4424,
"train_samples_per_second": 16.836,
"train_steps_per_second": 0.117
}
],
"logging_steps": 50,
"max_steps": 1760,
"num_input_tokens_seen": 0,
"num_train_epochs": 5,
"save_steps": 50,
"stateful_callbacks": {
"EarlyStoppingCallback": {
"args": {
"early_stopping_patience": 5,
"early_stopping_threshold": 0.0
},
"attributes": {
"early_stopping_patience_counter": 0
}
},
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": true
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 4,
"trial_name": null,
"trial_params": null
}
|