File size: 13,626 Bytes
328813a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 |
{
"best_metric": 0.004782966338098049,
"best_model_checkpoint": "./mistral/22-04-24-Weni-WeniGPT-Agents-Mistral-1.0.0-SFT-1.0.25-DPO_Experiment on DPO with other hyperparameters and best SFT model of WeniGPT-2_max_steps-180_batch_8_2024-04-22_ppid_9/checkpoint-180",
"epoch": 5.806451612903226,
"eval_steps": 30,
"global_step": 180,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.3225806451612903,
"grad_norm": 24.6901912689209,
"learning_rate": 5e-06,
"logits/chosen": -1.8365566730499268,
"logits/rejected": -1.8444175720214844,
"logps/chosen": -195.56666564941406,
"logps/rejected": -191.41812133789062,
"loss": 0.653,
"rewards/accuracies": 0.4000000059604645,
"rewards/chosen": 0.09110366553068161,
"rewards/margins": 0.10920021682977676,
"rewards/rejected": -0.018096545711159706,
"step": 10
},
{
"epoch": 0.6451612903225806,
"grad_norm": 13.191404342651367,
"learning_rate": 4.71264367816092e-06,
"logits/chosen": -1.7314943075180054,
"logits/rejected": -1.766556978225708,
"logps/chosen": -137.64279174804688,
"logps/rejected": -220.5197296142578,
"loss": 0.3701,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": 1.1793702840805054,
"rewards/margins": 1.4335505962371826,
"rewards/rejected": -0.2541801631450653,
"step": 20
},
{
"epoch": 0.967741935483871,
"grad_norm": 20.181856155395508,
"learning_rate": 4.42528735632184e-06,
"logits/chosen": -1.8526036739349365,
"logits/rejected": -1.86077880859375,
"logps/chosen": -121.78877258300781,
"logps/rejected": -151.2501678466797,
"loss": 0.225,
"rewards/accuracies": 1.0,
"rewards/chosen": 1.4271174669265747,
"rewards/margins": 1.875509262084961,
"rewards/rejected": -0.4483918249607086,
"step": 30
},
{
"epoch": 0.967741935483871,
"eval_logits/chosen": -1.9326385259628296,
"eval_logits/rejected": -1.9565868377685547,
"eval_logps/chosen": -123.71134185791016,
"eval_logps/rejected": -215.0025634765625,
"eval_loss": 0.22270560264587402,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": 2.6680667400360107,
"eval_rewards/margins": 3.676541566848755,
"eval_rewards/rejected": -1.008474588394165,
"eval_runtime": 15.7165,
"eval_samples_per_second": 1.782,
"eval_steps_per_second": 0.445,
"step": 30
},
{
"epoch": 1.2903225806451613,
"grad_norm": 1.0404773950576782,
"learning_rate": 4.137931034482759e-06,
"logits/chosen": -1.9101307392120361,
"logits/rejected": -1.9370834827423096,
"logps/chosen": -146.01133728027344,
"logps/rejected": -237.4972686767578,
"loss": 0.0848,
"rewards/accuracies": 1.0,
"rewards/chosen": 3.0706539154052734,
"rewards/margins": 5.080114364624023,
"rewards/rejected": -2.009460926055908,
"step": 40
},
{
"epoch": 1.6129032258064515,
"grad_norm": 17.344457626342773,
"learning_rate": 3.850574712643678e-06,
"logits/chosen": -1.8972657918930054,
"logits/rejected": -1.9069604873657227,
"logps/chosen": -195.85150146484375,
"logps/rejected": -158.05677795410156,
"loss": 0.0755,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": 3.0010573863983154,
"rewards/margins": 4.110104560852051,
"rewards/rejected": -1.1090469360351562,
"step": 50
},
{
"epoch": 1.935483870967742,
"grad_norm": 0.7190752029418945,
"learning_rate": 3.563218390804598e-06,
"logits/chosen": -1.8297414779663086,
"logits/rejected": -1.857456922531128,
"logps/chosen": -144.957275390625,
"logps/rejected": -232.25784301757812,
"loss": 0.0853,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": 3.4944889545440674,
"rewards/margins": 5.610640525817871,
"rewards/rejected": -2.116152286529541,
"step": 60
},
{
"epoch": 1.935483870967742,
"eval_logits/chosen": -1.9170362949371338,
"eval_logits/rejected": -1.9400348663330078,
"eval_logps/chosen": -118.40864562988281,
"eval_logps/rejected": -221.06422424316406,
"eval_loss": 0.10952840745449066,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": 3.7286033630371094,
"eval_rewards/margins": 5.949405193328857,
"eval_rewards/rejected": -2.220801830291748,
"eval_runtime": 15.4022,
"eval_samples_per_second": 1.818,
"eval_steps_per_second": 0.454,
"step": 60
},
{
"epoch": 2.258064516129032,
"grad_norm": 6.895229339599609,
"learning_rate": 3.2758620689655175e-06,
"logits/chosen": -1.7604904174804688,
"logits/rejected": -1.7763350009918213,
"logps/chosen": -94.25442504882812,
"logps/rejected": -200.06292724609375,
"loss": 0.0336,
"rewards/accuracies": 1.0,
"rewards/chosen": 3.892274856567383,
"rewards/margins": 7.140706539154053,
"rewards/rejected": -3.248431444168091,
"step": 70
},
{
"epoch": 2.5806451612903225,
"grad_norm": 3.1107871532440186,
"learning_rate": 2.988505747126437e-06,
"logits/chosen": -1.773847222328186,
"logits/rejected": -1.7859563827514648,
"logps/chosen": -134.7067108154297,
"logps/rejected": -175.28102111816406,
"loss": 0.0191,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": 3.6734061241149902,
"rewards/margins": 6.411807060241699,
"rewards/rejected": -2.738400936126709,
"step": 80
},
{
"epoch": 2.903225806451613,
"grad_norm": 0.4654656946659088,
"learning_rate": 2.7011494252873567e-06,
"logits/chosen": -1.8632606267929077,
"logits/rejected": -1.8870786428451538,
"logps/chosen": -152.00164794921875,
"logps/rejected": -194.2272186279297,
"loss": 0.0285,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": 3.7533652782440186,
"rewards/margins": 6.942251682281494,
"rewards/rejected": -3.1888861656188965,
"step": 90
},
{
"epoch": 2.903225806451613,
"eval_logits/chosen": -1.9063053131103516,
"eval_logits/rejected": -1.930100679397583,
"eval_logps/chosen": -116.32177734375,
"eval_logps/rejected": -229.86549377441406,
"eval_loss": 0.05446601286530495,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": 4.145977973937988,
"eval_rewards/margins": 8.12703800201416,
"eval_rewards/rejected": -3.9810593128204346,
"eval_runtime": 15.401,
"eval_samples_per_second": 1.818,
"eval_steps_per_second": 0.455,
"step": 90
},
{
"epoch": 3.225806451612903,
"grad_norm": 0.023089677095413208,
"learning_rate": 2.4137931034482762e-06,
"logits/chosen": -1.8187129497528076,
"logits/rejected": -1.8321533203125,
"logps/chosen": -118.27799987792969,
"logps/rejected": -238.4324493408203,
"loss": 0.0031,
"rewards/accuracies": 1.0,
"rewards/chosen": 4.572004318237305,
"rewards/margins": 9.180875778198242,
"rewards/rejected": -4.608872413635254,
"step": 100
},
{
"epoch": 3.5483870967741935,
"grad_norm": 0.9019374251365662,
"learning_rate": 2.1264367816091954e-06,
"logits/chosen": -1.903515100479126,
"logits/rejected": -1.9107078313827515,
"logps/chosen": -154.8245849609375,
"logps/rejected": -195.23399353027344,
"loss": 0.0127,
"rewards/accuracies": 1.0,
"rewards/chosen": 2.906761646270752,
"rewards/margins": 8.078901290893555,
"rewards/rejected": -5.172140121459961,
"step": 110
},
{
"epoch": 3.870967741935484,
"grad_norm": 0.11791550368070602,
"learning_rate": 1.839080459770115e-06,
"logits/chosen": -1.7719767093658447,
"logits/rejected": -1.7978557348251343,
"logps/chosen": -135.73764038085938,
"logps/rejected": -233.47396850585938,
"loss": 0.001,
"rewards/accuracies": 1.0,
"rewards/chosen": 3.7881596088409424,
"rewards/margins": 10.058608055114746,
"rewards/rejected": -6.270449161529541,
"step": 120
},
{
"epoch": 3.870967741935484,
"eval_logits/chosen": -1.8902080059051514,
"eval_logits/rejected": -1.9141491651535034,
"eval_logps/chosen": -116.14888763427734,
"eval_logps/rejected": -235.0476531982422,
"eval_loss": 0.04676542058587074,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": 4.18055534362793,
"eval_rewards/margins": 9.198049545288086,
"eval_rewards/rejected": -5.01749324798584,
"eval_runtime": 15.4024,
"eval_samples_per_second": 1.818,
"eval_steps_per_second": 0.454,
"step": 120
},
{
"epoch": 4.193548387096774,
"grad_norm": 0.062105003744363785,
"learning_rate": 1.5517241379310346e-06,
"logits/chosen": -1.7840442657470703,
"logits/rejected": -1.7999883890151978,
"logps/chosen": -116.44026184082031,
"logps/rejected": -252.85220336914062,
"loss": 0.0192,
"rewards/accuracies": 1.0,
"rewards/chosen": 2.862762451171875,
"rewards/margins": 9.177450180053711,
"rewards/rejected": -6.314688205718994,
"step": 130
},
{
"epoch": 4.516129032258064,
"grad_norm": 0.3371919095516205,
"learning_rate": 1.2643678160919542e-06,
"logits/chosen": -1.7637265920639038,
"logits/rejected": -1.7790952920913696,
"logps/chosen": -149.11972045898438,
"logps/rejected": -200.7391357421875,
"loss": 0.0156,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": 1.933764100074768,
"rewards/margins": 7.311190128326416,
"rewards/rejected": -5.377426624298096,
"step": 140
},
{
"epoch": 4.838709677419355,
"grad_norm": 0.4724487066268921,
"learning_rate": 9.770114942528738e-07,
"logits/chosen": -1.7254791259765625,
"logits/rejected": -1.754163384437561,
"logps/chosen": -142.09568786621094,
"logps/rejected": -264.6880798339844,
"loss": 0.0021,
"rewards/accuracies": 1.0,
"rewards/chosen": 3.6306958198547363,
"rewards/margins": 9.091299057006836,
"rewards/rejected": -5.460603713989258,
"step": 150
},
{
"epoch": 4.838709677419355,
"eval_logits/chosen": -1.881463885307312,
"eval_logits/rejected": -1.9056099653244019,
"eval_logps/chosen": -117.07276916503906,
"eval_logps/rejected": -239.6072235107422,
"eval_loss": 0.00868917815387249,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": 3.995779514312744,
"eval_rewards/margins": 9.925186157226562,
"eval_rewards/rejected": -5.929406642913818,
"eval_runtime": 15.4035,
"eval_samples_per_second": 1.818,
"eval_steps_per_second": 0.454,
"step": 150
},
{
"epoch": 5.161290322580645,
"grad_norm": 0.4406607449054718,
"learning_rate": 6.896551724137931e-07,
"logits/chosen": -1.8259941339492798,
"logits/rejected": -1.8320337533950806,
"logps/chosen": -178.92930603027344,
"logps/rejected": -274.9721984863281,
"loss": 0.0028,
"rewards/accuracies": 1.0,
"rewards/chosen": 2.2369651794433594,
"rewards/margins": 9.267016410827637,
"rewards/rejected": -7.030051231384277,
"step": 160
},
{
"epoch": 5.483870967741936,
"grad_norm": 0.02007269859313965,
"learning_rate": 4.0229885057471266e-07,
"logits/chosen": -1.7874090671539307,
"logits/rejected": -1.8095813989639282,
"logps/chosen": -149.6767120361328,
"logps/rejected": -237.3863067626953,
"loss": 0.0006,
"rewards/accuracies": 1.0,
"rewards/chosen": 4.129965782165527,
"rewards/margins": 11.051237106323242,
"rewards/rejected": -6.921273708343506,
"step": 170
},
{
"epoch": 5.806451612903226,
"grad_norm": 0.5593944787979126,
"learning_rate": 1.1494252873563219e-07,
"logits/chosen": -1.8051351308822632,
"logits/rejected": -1.8224855661392212,
"logps/chosen": -168.2032928466797,
"logps/rejected": -221.9829864501953,
"loss": 0.0014,
"rewards/accuracies": 1.0,
"rewards/chosen": 2.834609270095825,
"rewards/margins": 9.291410446166992,
"rewards/rejected": -6.456801414489746,
"step": 180
},
{
"epoch": 5.806451612903226,
"eval_logits/chosen": -1.8786267042160034,
"eval_logits/rejected": -1.9027243852615356,
"eval_logps/chosen": -117.46734619140625,
"eval_logps/rejected": -240.9935760498047,
"eval_loss": 0.004782966338098049,
"eval_rewards/accuracies": 1.0,
"eval_rewards/chosen": 3.9168648719787598,
"eval_rewards/margins": 10.123543739318848,
"eval_rewards/rejected": -6.206678867340088,
"eval_runtime": 15.4023,
"eval_samples_per_second": 1.818,
"eval_steps_per_second": 0.454,
"step": 180
}
],
"logging_steps": 10,
"max_steps": 180,
"num_input_tokens_seen": 0,
"num_train_epochs": 6,
"save_steps": 90,
"total_flos": 0.0,
"train_batch_size": 1,
"trial_name": null,
"trial_params": null
}
|