File size: 13,969 Bytes
2219c65 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 |
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.31391014322150285,
"eval_steps": 50,
"global_step": 200,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.015695507161075144,
"grad_norm": 0.07061820477247238,
"learning_rate": 4.999451708687114e-06,
"logits/chosen": 15.001790046691895,
"logits/rejected": 14.624488830566406,
"logps/chosen": -0.33085882663726807,
"logps/rejected": -0.24924471974372864,
"loss": 1.0519,
"rewards/accuracies": 0.23749999701976776,
"rewards/chosen": -0.4962882399559021,
"rewards/margins": -0.12242116779088974,
"rewards/rejected": -0.37386709451675415,
"step": 10
},
{
"epoch": 0.03139101432215029,
"grad_norm": 0.060370542109012604,
"learning_rate": 4.997807075247147e-06,
"logits/chosen": 15.272351264953613,
"logits/rejected": 14.801017761230469,
"logps/chosen": -0.3379867672920227,
"logps/rejected": -0.24759705364704132,
"loss": 1.0494,
"rewards/accuracies": 0.15000000596046448,
"rewards/chosen": -0.5069801211357117,
"rewards/margins": -0.13558456301689148,
"rewards/rejected": -0.3713955581188202,
"step": 20
},
{
"epoch": 0.047086521483225424,
"grad_norm": 0.050332240760326385,
"learning_rate": 4.9950668210706795e-06,
"logits/chosen": 15.569372177124023,
"logits/rejected": 15.249380111694336,
"logps/chosen": -0.3347483277320862,
"logps/rejected": -0.275604784488678,
"loss": 1.0375,
"rewards/accuracies": 0.21250000596046448,
"rewards/chosen": -0.5021225214004517,
"rewards/margins": -0.08871528506278992,
"rewards/rejected": -0.41340717673301697,
"step": 30
},
{
"epoch": 0.06278202864430057,
"grad_norm": 0.054660990834236145,
"learning_rate": 4.9912321481237616e-06,
"logits/chosen": 15.344067573547363,
"logits/rejected": 14.881669998168945,
"logps/chosen": -0.3336474299430847,
"logps/rejected": -0.27252259850502014,
"loss": 1.0375,
"rewards/accuracies": 0.20000000298023224,
"rewards/chosen": -0.5004712343215942,
"rewards/margins": -0.09168727695941925,
"rewards/rejected": -0.40878385305404663,
"step": 40
},
{
"epoch": 0.07847753580537571,
"grad_norm": 0.060405269265174866,
"learning_rate": 4.986304738420684e-06,
"logits/chosen": 15.5701904296875,
"logits/rejected": 15.60998821258545,
"logps/chosen": -0.3075069785118103,
"logps/rejected": -0.24850216507911682,
"loss": 1.0341,
"rewards/accuracies": 0.20000000298023224,
"rewards/chosen": -0.46126049757003784,
"rewards/margins": -0.08850721269845963,
"rewards/rejected": -0.3727532923221588,
"step": 50
},
{
"epoch": 0.07847753580537571,
"eval_logits/chosen": 15.719181060791016,
"eval_logits/rejected": 15.20205307006836,
"eval_logps/chosen": -0.32490479946136475,
"eval_logps/rejected": -0.26058924198150635,
"eval_loss": 1.0295902490615845,
"eval_rewards/accuracies": 0.26923078298568726,
"eval_rewards/chosen": -0.48735716938972473,
"eval_rewards/margins": -0.09647335112094879,
"eval_rewards/rejected": -0.39088380336761475,
"eval_runtime": 14.6102,
"eval_samples_per_second": 28.199,
"eval_steps_per_second": 3.559,
"step": 50
},
{
"epoch": 0.09417304296645085,
"grad_norm": 0.06511708348989487,
"learning_rate": 4.980286753286196e-06,
"logits/chosen": 15.571348190307617,
"logits/rejected": 15.418828010559082,
"logps/chosen": -0.3217025101184845,
"logps/rejected": -0.2813836932182312,
"loss": 1.0385,
"rewards/accuracies": 0.23749999701976776,
"rewards/chosen": -0.48255378007888794,
"rewards/margins": -0.060478221625089645,
"rewards/rejected": -0.4220755696296692,
"step": 60
},
{
"epoch": 0.109868550127526,
"grad_norm": 0.06922140717506409,
"learning_rate": 4.973180832407471e-06,
"logits/chosen": 15.926614761352539,
"logits/rejected": 15.90565299987793,
"logps/chosen": -0.34742942452430725,
"logps/rejected": -0.26853513717651367,
"loss": 1.0417,
"rewards/accuracies": 0.21250000596046448,
"rewards/chosen": -0.5211440324783325,
"rewards/margins": -0.11834144592285156,
"rewards/rejected": -0.40280264616012573,
"step": 70
},
{
"epoch": 0.12556405728860115,
"grad_norm": 0.07853339612483978,
"learning_rate": 4.964990092676263e-06,
"logits/chosen": 15.876957893371582,
"logits/rejected": 15.636863708496094,
"logps/chosen": -0.34999752044677734,
"logps/rejected": -0.2652502655982971,
"loss": 1.0255,
"rewards/accuracies": 0.1875,
"rewards/chosen": -0.524996280670166,
"rewards/margins": -0.12712089717388153,
"rewards/rejected": -0.3978753685951233,
"step": 80
},
{
"epoch": 0.14125956444967627,
"grad_norm": 0.07262148708105087,
"learning_rate": 4.9557181268217225e-06,
"logits/chosen": 16.037456512451172,
"logits/rejected": 15.742408752441406,
"logps/chosen": -0.32482510805130005,
"logps/rejected": -0.2520079016685486,
"loss": 1.0297,
"rewards/accuracies": 0.21250000596046448,
"rewards/chosen": -0.48723769187927246,
"rewards/margins": -0.1092257872223854,
"rewards/rejected": -0.3780118525028229,
"step": 90
},
{
"epoch": 0.15695507161075142,
"grad_norm": 0.08407289534807205,
"learning_rate": 4.9453690018345144e-06,
"logits/chosen": 16.34904670715332,
"logits/rejected": 16.30767059326172,
"logps/chosen": -0.3302846848964691,
"logps/rejected": -0.2799247205257416,
"loss": 1.0285,
"rewards/accuracies": 0.2750000059604645,
"rewards/chosen": -0.4954269826412201,
"rewards/margins": -0.07553993165493011,
"rewards/rejected": -0.41988706588745117,
"step": 100
},
{
"epoch": 0.15695507161075142,
"eval_logits/chosen": 16.43848419189453,
"eval_logits/rejected": 15.978095054626465,
"eval_logps/chosen": -0.3256901502609253,
"eval_logps/rejected": -0.2794351279735565,
"eval_loss": 1.0174708366394043,
"eval_rewards/accuracies": 0.2884615361690521,
"eval_rewards/chosen": -0.4885352551937103,
"eval_rewards/margins": -0.06938254088163376,
"eval_rewards/rejected": -0.41915270686149597,
"eval_runtime": 14.607,
"eval_samples_per_second": 28.206,
"eval_steps_per_second": 3.56,
"step": 100
},
{
"epoch": 0.17265057877182657,
"grad_norm": 0.07448805868625641,
"learning_rate": 4.933947257182901e-06,
"logits/chosen": 16.681476593017578,
"logits/rejected": 16.208908081054688,
"logps/chosen": -0.3410753309726715,
"logps/rejected": -0.28177398443222046,
"loss": 1.0364,
"rewards/accuracies": 0.2874999940395355,
"rewards/chosen": -0.5116130113601685,
"rewards/margins": -0.08895199000835419,
"rewards/rejected": -0.4226610064506531,
"step": 110
},
{
"epoch": 0.1883460859329017,
"grad_norm": 0.17266640067100525,
"learning_rate": 4.921457902821578e-06,
"logits/chosen": 16.73276710510254,
"logits/rejected": 16.46255111694336,
"logps/chosen": -0.30808666348457336,
"logps/rejected": -0.2663067877292633,
"loss": 1.0242,
"rewards/accuracies": 0.23749999701976776,
"rewards/chosen": -0.46213001012802124,
"rewards/margins": -0.0626697838306427,
"rewards/rejected": -0.39946022629737854,
"step": 120
},
{
"epoch": 0.20404159309397685,
"grad_norm": 0.1035998985171318,
"learning_rate": 4.907906416994146e-06,
"logits/chosen": 16.380239486694336,
"logits/rejected": 16.233612060546875,
"logps/chosen": -0.32250356674194336,
"logps/rejected": -0.27460020780563354,
"loss": 1.0162,
"rewards/accuracies": 0.2874999940395355,
"rewards/chosen": -0.48375529050827026,
"rewards/margins": -0.0718550831079483,
"rewards/rejected": -0.41190028190612793,
"step": 130
},
{
"epoch": 0.219737100255052,
"grad_norm": 0.0961030125617981,
"learning_rate": 4.893298743830168e-06,
"logits/chosen": 16.440303802490234,
"logits/rejected": 16.233903884887695,
"logps/chosen": -0.33421438932418823,
"logps/rejected": -0.2857271134853363,
"loss": 1.0008,
"rewards/accuracies": 0.3125,
"rewards/chosen": -0.5013214945793152,
"rewards/margins": -0.07273083180189133,
"rewards/rejected": -0.42859068512916565,
"step": 140
},
{
"epoch": 0.23543260741612712,
"grad_norm": 0.14799565076828003,
"learning_rate": 4.8776412907378845e-06,
"logits/chosen": 16.600738525390625,
"logits/rejected": 16.536447525024414,
"logps/chosen": -0.3243326246738434,
"logps/rejected": -0.3061785101890564,
"loss": 1.0085,
"rewards/accuracies": 0.3375000059604645,
"rewards/chosen": -0.4864989221096039,
"rewards/margins": -0.027231160551309586,
"rewards/rejected": -0.459267795085907,
"step": 150
},
{
"epoch": 0.23543260741612712,
"eval_logits/chosen": 16.8078556060791,
"eval_logits/rejected": 16.311922073364258,
"eval_logps/chosen": -0.326242595911026,
"eval_logps/rejected": -0.3398977816104889,
"eval_loss": 0.9791463017463684,
"eval_rewards/accuracies": 0.4038461446762085,
"eval_rewards/chosen": -0.4893638789653778,
"eval_rewards/margins": 0.020482787862420082,
"eval_rewards/rejected": -0.5098467469215393,
"eval_runtime": 14.6072,
"eval_samples_per_second": 28.205,
"eval_steps_per_second": 3.56,
"step": 150
},
{
"epoch": 0.2511281145772023,
"grad_norm": 0.11464422941207886,
"learning_rate": 4.860940925593703e-06,
"logits/chosen": 16.586620330810547,
"logits/rejected": 16.253028869628906,
"logps/chosen": -0.33331993222236633,
"logps/rejected": -0.35580095648765564,
"loss": 0.9827,
"rewards/accuracies": 0.42500001192092896,
"rewards/chosen": -0.49997982382774353,
"rewards/margins": 0.03372158855199814,
"rewards/rejected": -0.5337014198303223,
"step": 160
},
{
"epoch": 0.2668236217382774,
"grad_norm": 0.09462621062994003,
"learning_rate": 4.84320497372973e-06,
"logits/chosen": 16.220312118530273,
"logits/rejected": 16.105356216430664,
"logps/chosen": -0.2957116663455963,
"logps/rejected": -0.3528788089752197,
"loss": 0.9716,
"rewards/accuracies": 0.4625000059604645,
"rewards/chosen": -0.44356757402420044,
"rewards/margins": 0.08575066924095154,
"rewards/rejected": -0.5293182134628296,
"step": 170
},
{
"epoch": 0.28251912889935255,
"grad_norm": 0.10995513945817947,
"learning_rate": 4.824441214720629e-06,
"logits/chosen": 16.811044692993164,
"logits/rejected": 16.43205451965332,
"logps/chosen": -0.3263992369174957,
"logps/rejected": -0.35878634452819824,
"loss": 0.9546,
"rewards/accuracies": 0.4124999940395355,
"rewards/chosen": -0.4895988404750824,
"rewards/margins": 0.04858064278960228,
"rewards/rejected": -0.5381795167922974,
"step": 180
},
{
"epoch": 0.2982146360604277,
"grad_norm": 0.1348036825656891,
"learning_rate": 4.804657878971252e-06,
"logits/chosen": 16.816158294677734,
"logits/rejected": 16.55413055419922,
"logps/chosen": -0.31842875480651855,
"logps/rejected": -0.3791810870170593,
"loss": 0.9472,
"rewards/accuracies": 0.5249999761581421,
"rewards/chosen": -0.47764310240745544,
"rewards/margins": 0.09112847596406937,
"rewards/rejected": -0.5687715411186218,
"step": 190
},
{
"epoch": 0.31391014322150285,
"grad_norm": 0.13393694162368774,
"learning_rate": 4.783863644106502e-06,
"logits/chosen": 17.075885772705078,
"logits/rejected": 16.982004165649414,
"logps/chosen": -0.3368823528289795,
"logps/rejected": -0.3636801540851593,
"loss": 0.9455,
"rewards/accuracies": 0.375,
"rewards/chosen": -0.5053235292434692,
"rewards/margins": 0.04019671678543091,
"rewards/rejected": -0.5455202460289001,
"step": 200
},
{
"epoch": 0.31391014322150285,
"eval_logits/chosen": 17.17239761352539,
"eval_logits/rejected": 16.609193801879883,
"eval_logps/chosen": -0.33508703112602234,
"eval_logps/rejected": -0.4493505656719208,
"eval_loss": 0.9283667802810669,
"eval_rewards/accuracies": 0.4615384638309479,
"eval_rewards/chosen": -0.5026305913925171,
"eval_rewards/margins": 0.17139528691768646,
"eval_rewards/rejected": -0.67402583360672,
"eval_runtime": 14.5913,
"eval_samples_per_second": 28.236,
"eval_steps_per_second": 3.564,
"step": 200
}
],
"logging_steps": 10,
"max_steps": 1500,
"num_input_tokens_seen": 0,
"num_train_epochs": 3,
"save_steps": 50,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 4.7951015917807206e+17,
"train_batch_size": 1,
"trial_name": null,
"trial_params": null
}
|