zephyr-7b-gpo-gen-i1 / trainer_state.json
lole25's picture
Model save
79d4aca verified
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 1.0,
"eval_steps": 100,
"global_step": 1250,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.0,
"learning_rate": 4e-08,
"logits/chosen": -1.9152824878692627,
"logits/rejected": -1.9152824878692627,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3334,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1
},
{
"epoch": 0.01,
"learning_rate": 4.0000000000000003e-07,
"logits/chosen": -2.035064220428467,
"logits/rejected": -2.035064220428467,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3197,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 10
},
{
"epoch": 0.02,
"learning_rate": 8.000000000000001e-07,
"logits/chosen": -2.0738348960876465,
"logits/rejected": -2.0738348960876465,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3507,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 20
},
{
"epoch": 0.02,
"learning_rate": 1.2000000000000002e-06,
"logits/chosen": -1.9596742391586304,
"logits/rejected": -1.9596742391586304,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3334,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 30
},
{
"epoch": 0.03,
"learning_rate": 1.6000000000000001e-06,
"logits/chosen": -1.9157626628875732,
"logits/rejected": -1.9157626628875732,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.368,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 40
},
{
"epoch": 0.04,
"learning_rate": 2.0000000000000003e-06,
"logits/chosen": -1.9507601261138916,
"logits/rejected": -1.9507601261138916,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3285,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 50
},
{
"epoch": 0.05,
"learning_rate": 2.4000000000000003e-06,
"logits/chosen": -1.9532804489135742,
"logits/rejected": -1.9532804489135742,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.368,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 60
},
{
"epoch": 0.06,
"learning_rate": 2.8000000000000003e-06,
"logits/chosen": -1.8357080221176147,
"logits/rejected": -1.8357080221176147,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.2717,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 70
},
{
"epoch": 0.06,
"learning_rate": 3.2000000000000003e-06,
"logits/chosen": -1.8322147130966187,
"logits/rejected": -1.8322147130966187,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3457,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 80
},
{
"epoch": 0.07,
"learning_rate": 3.6000000000000003e-06,
"logits/chosen": -1.8425270318984985,
"logits/rejected": -1.8425270318984985,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3902,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 90
},
{
"epoch": 0.08,
"learning_rate": 4.000000000000001e-06,
"logits/chosen": -1.8621313571929932,
"logits/rejected": -1.8621313571929932,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3754,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 100
},
{
"epoch": 0.08,
"eval_logits/chosen": -1.9617642164230347,
"eval_logits/rejected": -1.8066532611846924,
"eval_logps/chosen": -266.6976013183594,
"eval_logps/rejected": -254.9398193359375,
"eval_loss": 0.053734518587589264,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": 0.0,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": 0.0,
"eval_runtime": 727.871,
"eval_samples_per_second": 2.748,
"eval_steps_per_second": 1.374,
"step": 100
},
{
"epoch": 0.09,
"learning_rate": 4.4e-06,
"logits/chosen": -1.909868597984314,
"logits/rejected": -1.909868597984314,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3754,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 110
},
{
"epoch": 0.1,
"learning_rate": 4.800000000000001e-06,
"logits/chosen": -1.7610801458358765,
"logits/rejected": -1.7610801458358765,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3062,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 120
},
{
"epoch": 0.1,
"learning_rate": 4.999756310023261e-06,
"logits/chosen": -2.119793653488159,
"logits/rejected": -2.119793653488159,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.326,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 130
},
{
"epoch": 0.11,
"learning_rate": 4.997807075247147e-06,
"logits/chosen": -1.986210584640503,
"logits/rejected": -1.986210584640503,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3334,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 140
},
{
"epoch": 0.12,
"learning_rate": 4.993910125649561e-06,
"logits/chosen": -2.0813775062561035,
"logits/rejected": -2.0813775062561035,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3556,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 150
},
{
"epoch": 0.13,
"learning_rate": 4.988068499954578e-06,
"logits/chosen": -2.0440070629119873,
"logits/rejected": -2.0440070629119873,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3433,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 160
},
{
"epoch": 0.14,
"learning_rate": 4.980286753286196e-06,
"logits/chosen": -1.6871118545532227,
"logits/rejected": -1.6871118545532227,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3235,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 170
},
{
"epoch": 0.14,
"learning_rate": 4.970570953616383e-06,
"logits/chosen": -1.906797170639038,
"logits/rejected": -1.906797170639038,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3507,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 180
},
{
"epoch": 0.15,
"learning_rate": 4.958928677033465e-06,
"logits/chosen": -1.7077922821044922,
"logits/rejected": -1.7077922821044922,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3433,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 190
},
{
"epoch": 0.16,
"learning_rate": 4.9453690018345144e-06,
"logits/chosen": -1.7370048761367798,
"logits/rejected": -1.7370048761367798,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3556,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 200
},
{
"epoch": 0.16,
"eval_logits/chosen": -1.9617642164230347,
"eval_logits/rejected": -1.8066532611846924,
"eval_logps/chosen": -266.6976013183594,
"eval_logps/rejected": -254.9398193359375,
"eval_loss": 0.053734518587589264,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": 0.0,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": 0.0,
"eval_runtime": 730.7556,
"eval_samples_per_second": 2.737,
"eval_steps_per_second": 1.368,
"step": 200
},
{
"epoch": 0.17,
"learning_rate": 4.9299025014463665e-06,
"logits/chosen": -1.7182317972183228,
"logits/rejected": -1.7182317972183228,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3482,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 210
},
{
"epoch": 0.18,
"learning_rate": 4.912541236180779e-06,
"logits/chosen": -1.905134916305542,
"logits/rejected": -1.905134916305542,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3457,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 220
},
{
"epoch": 0.18,
"learning_rate": 4.893298743830168e-06,
"logits/chosen": -2.071720600128174,
"logits/rejected": -2.071720600128174,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3507,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 230
},
{
"epoch": 0.19,
"learning_rate": 4.8721900291112415e-06,
"logits/chosen": -2.025371789932251,
"logits/rejected": -2.025371789932251,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3482,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 240
},
{
"epoch": 0.2,
"learning_rate": 4.849231551964771e-06,
"logits/chosen": -1.8689197301864624,
"logits/rejected": -1.8689197301864624,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.363,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 250
},
{
"epoch": 0.21,
"learning_rate": 4.824441214720629e-06,
"logits/chosen": -1.937535285949707,
"logits/rejected": -1.937535285949707,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3556,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 260
},
{
"epoch": 0.22,
"learning_rate": 4.7978383481380865e-06,
"logits/chosen": -2.058104991912842,
"logits/rejected": -2.058104991912842,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3581,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 270
},
{
"epoch": 0.22,
"learning_rate": 4.769443696332272e-06,
"logits/chosen": -1.8560088872909546,
"logits/rejected": -1.8560088872909546,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3902,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 280
},
{
"epoch": 0.23,
"learning_rate": 4.7392794005985324e-06,
"logits/chosen": -1.8305753469467163,
"logits/rejected": -1.8305753469467163,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.363,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 290
},
{
"epoch": 0.24,
"learning_rate": 4.707368982147318e-06,
"logits/chosen": -1.7557750940322876,
"logits/rejected": -1.7557750940322876,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3556,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 300
},
{
"epoch": 0.24,
"eval_logits/chosen": -1.9617642164230347,
"eval_logits/rejected": -1.8066532611846924,
"eval_logps/chosen": -266.6976013183594,
"eval_logps/rejected": -254.9398193359375,
"eval_loss": 0.053734518587589264,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": 0.0,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": 0.0,
"eval_runtime": 730.4441,
"eval_samples_per_second": 2.738,
"eval_steps_per_second": 1.369,
"step": 300
},
{
"epoch": 0.25,
"learning_rate": 4.673737323763048e-06,
"logits/chosen": -2.111258029937744,
"logits/rejected": -2.111258029937744,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3655,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 310
},
{
"epoch": 0.26,
"learning_rate": 4.638410650401267e-06,
"logits/chosen": -1.8872848749160767,
"logits/rejected": -1.8872848749160767,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3112,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 320
},
{
"epoch": 0.26,
"learning_rate": 4.601416508739211e-06,
"logits/chosen": -1.9492517709732056,
"logits/rejected": -1.9492517709732056,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3433,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 330
},
{
"epoch": 0.27,
"learning_rate": 4.562783745695738e-06,
"logits/chosen": -1.8537830114364624,
"logits/rejected": -1.8537830114364624,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3704,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 340
},
{
"epoch": 0.28,
"learning_rate": 4.522542485937369e-06,
"logits/chosen": -2.1256401538848877,
"logits/rejected": -2.1256401538848877,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3285,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 350
},
{
"epoch": 0.29,
"learning_rate": 4.4807241083879774e-06,
"logits/chosen": -1.9888694286346436,
"logits/rejected": -1.9888694286346436,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3433,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 360
},
{
"epoch": 0.3,
"learning_rate": 4.437361221760449e-06,
"logits/chosen": -2.064541816711426,
"logits/rejected": -2.064541816711426,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3136,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 370
},
{
"epoch": 0.3,
"learning_rate": 4.3924876391293915e-06,
"logits/chosen": -2.1313929557800293,
"logits/rejected": -2.1313929557800293,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3482,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 380
},
{
"epoch": 0.31,
"learning_rate": 4.346138351564711e-06,
"logits/chosen": -2.0341124534606934,
"logits/rejected": -2.0341124534606934,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.2914,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 390
},
{
"epoch": 0.32,
"learning_rate": 4.2983495008466285e-06,
"logits/chosen": -1.7964853048324585,
"logits/rejected": -1.7964853048324585,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3606,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 400
},
{
"epoch": 0.32,
"eval_logits/chosen": -1.9617642164230347,
"eval_logits/rejected": -1.8066532611846924,
"eval_logps/chosen": -266.6976013183594,
"eval_logps/rejected": -254.9398193359375,
"eval_loss": 0.053734518587589264,
"eval_rewards/accuracies": 0.0,
"eval_rewards/chosen": 0.0,
"eval_rewards/margins": 0.0,
"eval_rewards/rejected": 0.0,
"eval_runtime": 725.8132,
"eval_samples_per_second": 2.756,
"eval_steps_per_second": 1.378,
"step": 400
},
{
"epoch": 0.33,
"learning_rate": 4.249158351283414e-06,
"logits/chosen": -1.9106441736221313,
"logits/rejected": -1.9106441736221313,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3359,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 410
},
{
"epoch": 0.34,
"learning_rate": 4.198603260653792e-06,
"logits/chosen": -1.9656860828399658,
"logits/rejected": -1.9656860828399658,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3729,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 420
},
{
"epoch": 0.34,
"learning_rate": 4.146723650296701e-06,
"logits/chosen": -1.9669355154037476,
"logits/rejected": -1.9669355154037476,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3211,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 430
},
{
"epoch": 0.35,
"learning_rate": 4.093559974371725e-06,
"logits/chosen": -1.8838989734649658,
"logits/rejected": -1.8838989734649658,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3976,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 440
},
{
"epoch": 0.36,
"learning_rate": 4.039153688314146e-06,
"logits/chosen": -1.890795111656189,
"logits/rejected": -1.890795111656189,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3828,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 450
},
{
"epoch": 0.37,
"learning_rate": 3.983547216509254e-06,
"logits/chosen": -1.9264202117919922,
"logits/rejected": -1.9264202117919922,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3531,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 460
},
{
"epoch": 0.38,
"learning_rate": 3.92678391921108e-06,
"logits/chosen": -1.8948800563812256,
"logits/rejected": -1.8948800563812256,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3211,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 470
},
{
"epoch": 0.38,
"learning_rate": 3.868908058731376e-06,
"logits/chosen": -1.9177709817886353,
"logits/rejected": -1.8989362716674805,
"logps/chosen": -10.277307510375977,
"logps/rejected": -4.643392562866211,
"loss": 0.3284,
"rewards/accuracies": 0.02500000037252903,
"rewards/chosen": 9.182358189718798e-05,
"rewards/margins": 2.3801421775715426e-05,
"rewards/rejected": 6.802215648349375e-05,
"step": 480
},
{
"epoch": 0.39,
"learning_rate": 3.8099647649251984e-06,
"logits/chosen": -1.9777803421020508,
"logits/rejected": -1.9777803421020508,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3359,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 490
},
{
"epoch": 0.4,
"learning_rate": 3.7500000000000005e-06,
"logits/chosen": -1.8519232273101807,
"logits/rejected": -1.8519232273101807,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3606,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 500
},
{
"epoch": 0.4,
"eval_logits/chosen": -1.9201807975769043,
"eval_logits/rejected": -1.7627112865447998,
"eval_logps/chosen": -300.9842834472656,
"eval_logps/rejected": -281.2868957519531,
"eval_loss": 0.058566804975271225,
"eval_rewards/accuracies": 0.3125,
"eval_rewards/chosen": -0.034286707639694214,
"eval_rewards/margins": -0.00793963111937046,
"eval_rewards/rejected": -0.026347076520323753,
"eval_runtime": 729.5568,
"eval_samples_per_second": 2.741,
"eval_steps_per_second": 1.371,
"step": 500
},
{
"epoch": 0.41,
"learning_rate": 3.689060522675689e-06,
"logits/chosen": -1.7138468027114868,
"logits/rejected": -1.7138468027114868,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3211,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 510
},
{
"epoch": 0.42,
"learning_rate": 3.627193851723577e-06,
"logits/chosen": -1.802970290184021,
"logits/rejected": -1.802970290184021,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.2643,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 520
},
{
"epoch": 0.42,
"learning_rate": 3.564448228912682e-06,
"logits/chosen": -1.6650207042694092,
"logits/rejected": -1.6650207042694092,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.368,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 530
},
{
"epoch": 0.43,
"learning_rate": 3.5008725813922383e-06,
"logits/chosen": -1.8073577880859375,
"logits/rejected": -1.8073577880859375,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3186,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 540
},
{
"epoch": 0.44,
"learning_rate": 3.436516483539781e-06,
"logits/chosen": -1.7098474502563477,
"logits/rejected": -1.7098474502563477,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3556,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 550
},
{
"epoch": 0.45,
"learning_rate": 3.3714301183045382e-06,
"logits/chosen": -1.8214161396026611,
"logits/rejected": -1.8214161396026611,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3606,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 560
},
{
"epoch": 0.46,
"learning_rate": 3.3056642380762783e-06,
"logits/chosen": -1.8090301752090454,
"logits/rejected": -1.8090301752090454,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.4001,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 570
},
{
"epoch": 0.46,
"learning_rate": 3.2392701251101172e-06,
"logits/chosen": -1.6492881774902344,
"logits/rejected": -1.6492881774902344,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3704,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 580
},
{
"epoch": 0.47,
"learning_rate": 3.1722995515381644e-06,
"logits/chosen": -1.7389822006225586,
"logits/rejected": -1.7389822006225586,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3334,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 590
},
{
"epoch": 0.48,
"learning_rate": 3.1048047389991693e-06,
"logits/chosen": -1.6927127838134766,
"logits/rejected": -1.6927127838134766,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3408,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 600
},
{
"epoch": 0.48,
"eval_logits/chosen": -1.8917298316955566,
"eval_logits/rejected": -1.7360574007034302,
"eval_logps/chosen": -305.4412841796875,
"eval_logps/rejected": -285.37774658203125,
"eval_loss": 0.058740295469760895,
"eval_rewards/accuracies": 0.31200000643730164,
"eval_rewards/chosen": -0.038743674755096436,
"eval_rewards/margins": -0.008305751718580723,
"eval_rewards/rejected": -0.030437923967838287,
"eval_runtime": 730.0666,
"eval_samples_per_second": 2.739,
"eval_steps_per_second": 1.37,
"step": 600
},
{
"epoch": 0.49,
"learning_rate": 3.0368383179176584e-06,
"logits/chosen": -1.760236382484436,
"logits/rejected": -1.760236382484436,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3581,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 610
},
{
"epoch": 0.5,
"learning_rate": 2.9684532864643123e-06,
"logits/chosen": -1.7868738174438477,
"logits/rejected": -1.7868738174438477,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3112,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 620
},
{
"epoch": 0.5,
"learning_rate": 2.8997029692295875e-06,
"logits/chosen": -1.7506424188613892,
"logits/rejected": -1.7506424188613892,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3507,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 630
},
{
"epoch": 0.51,
"learning_rate": 2.8306409756428067e-06,
"logits/chosen": -1.6889375448226929,
"logits/rejected": -1.6889375448226929,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3334,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 640
},
{
"epoch": 0.52,
"learning_rate": 2.761321158169134e-06,
"logits/chosen": -1.7860199213027954,
"logits/rejected": -1.7860199213027954,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.368,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 650
},
{
"epoch": 0.53,
"learning_rate": 2.6917975703170466e-06,
"logits/chosen": -1.8066167831420898,
"logits/rejected": -1.8066167831420898,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.368,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 660
},
{
"epoch": 0.54,
"learning_rate": 2.6221244244890336e-06,
"logits/chosen": -1.998725175857544,
"logits/rejected": -1.998725175857544,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.2939,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 670
},
{
"epoch": 0.54,
"learning_rate": 2.5523560497083927e-06,
"logits/chosen": -1.7736581563949585,
"logits/rejected": -1.7736581563949585,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.4025,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 680
},
{
"epoch": 0.55,
"learning_rate": 2.482546849255096e-06,
"logits/chosen": -1.7235050201416016,
"logits/rejected": -1.7235050201416016,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.2865,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 690
},
{
"epoch": 0.56,
"learning_rate": 2.4127512582437486e-06,
"logits/chosen": -1.740098237991333,
"logits/rejected": -1.740098237991333,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3359,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 700
},
{
"epoch": 0.56,
"eval_logits/chosen": -1.8920072317123413,
"eval_logits/rejected": -1.7363018989562988,
"eval_logps/chosen": -305.3719787597656,
"eval_logps/rejected": -285.3293762207031,
"eval_loss": 0.05872537940740585,
"eval_rewards/accuracies": 0.3095000088214874,
"eval_rewards/chosen": -0.038674384355545044,
"eval_rewards/margins": -0.008284823969006538,
"eval_rewards/rejected": -0.030389558523893356,
"eval_runtime": 729.5557,
"eval_samples_per_second": 2.741,
"eval_steps_per_second": 1.371,
"step": 700
},
{
"epoch": 0.57,
"learning_rate": 2.3430237011767166e-06,
"logits/chosen": -1.9098726511001587,
"logits/rejected": -1.9098726511001587,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3556,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 710
},
{
"epoch": 0.58,
"learning_rate": 2.2734185495055503e-06,
"logits/chosen": -1.7153629064559937,
"logits/rejected": -1.7153629064559937,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3828,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 720
},
{
"epoch": 0.58,
"learning_rate": 2.2039900792337477e-06,
"logits/chosen": -1.611203908920288,
"logits/rejected": -1.611203908920288,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3482,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 730
},
{
"epoch": 0.59,
"learning_rate": 2.134792428593971e-06,
"logits/chosen": -1.7782968282699585,
"logits/rejected": -1.7782968282699585,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3112,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 740
},
{
"epoch": 0.6,
"learning_rate": 2.0658795558326745e-06,
"logits/chosen": -1.8602126836776733,
"logits/rejected": -1.8602126836776733,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3877,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 750
},
{
"epoch": 0.61,
"learning_rate": 1.997305197135089e-06,
"logits/chosen": -1.7727556228637695,
"logits/rejected": -1.7727556228637695,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3309,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 760
},
{
"epoch": 0.62,
"learning_rate": 1.9291228247233607e-06,
"logits/chosen": -1.8260834217071533,
"logits/rejected": -1.8260834217071533,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3556,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 770
},
{
"epoch": 0.62,
"learning_rate": 1.8613856051605242e-06,
"logits/chosen": -1.7438442707061768,
"logits/rejected": -1.7438442707061768,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.2988,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 780
},
{
"epoch": 0.63,
"learning_rate": 1.7941463578928088e-06,
"logits/chosen": -1.9674087762832642,
"logits/rejected": -1.9656404256820679,
"logps/chosen": -3.2285637855529785,
"logps/rejected": -2.4141287803649902,
"loss": 0.3516,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.001752424635924399,
"rewards/margins": -0.0005892428453080356,
"rewards/rejected": -0.0011631817324087024,
"step": 790
},
{
"epoch": 0.64,
"learning_rate": 1.7274575140626318e-06,
"logits/chosen": -1.8979122638702393,
"logits/rejected": -1.8979122638702393,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3507,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 800
},
{
"epoch": 0.64,
"eval_logits/chosen": -1.9784096479415894,
"eval_logits/rejected": -1.8172101974487305,
"eval_logps/chosen": -291.8357238769531,
"eval_logps/rejected": -274.8357238769531,
"eval_loss": 0.05687940865755081,
"eval_rewards/accuracies": 0.3215000033378601,
"eval_rewards/chosen": -0.02513808384537697,
"eval_rewards/margins": -0.005242162849754095,
"eval_rewards/rejected": -0.019895924255251884,
"eval_runtime": 728.993,
"eval_samples_per_second": 2.744,
"eval_steps_per_second": 1.372,
"step": 800
},
{
"epoch": 0.65,
"learning_rate": 1.661371075624363e-06,
"logits/chosen": -1.7325031757354736,
"logits/rejected": -1.7325031757354736,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3803,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 810
},
{
"epoch": 0.66,
"learning_rate": 1.5959385747947697e-06,
"logits/chosen": -1.9031715393066406,
"logits/rejected": -1.9031715393066406,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3606,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 820
},
{
"epoch": 0.66,
"learning_rate": 1.5312110338697427e-06,
"logits/chosen": -1.878941297531128,
"logits/rejected": -1.8776565790176392,
"logps/chosen": -4.952629089355469,
"logps/rejected": -4.525970458984375,
"loss": 0.3682,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.000667310378048569,
"rewards/margins": -0.0001743949978845194,
"rewards/rejected": -0.0004929153947159648,
"step": 830
},
{
"epoch": 0.67,
"learning_rate": 1.467238925438646e-06,
"logits/chosen": -1.8778276443481445,
"logits/rejected": -1.8778276443481445,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3507,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 840
},
{
"epoch": 0.68,
"learning_rate": 1.4040721330273063e-06,
"logits/chosen": -1.8605072498321533,
"logits/rejected": -1.8605072498321533,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.289,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 850
},
{
"epoch": 0.69,
"learning_rate": 1.3417599122003464e-06,
"logits/chosen": -1.521748423576355,
"logits/rejected": -1.521748423576355,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3211,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 860
},
{
"epoch": 0.7,
"learning_rate": 1.280350852153168e-06,
"logits/chosen": -1.753326416015625,
"logits/rejected": -1.7436802387237549,
"logps/chosen": -11.578987121582031,
"logps/rejected": -9.436164855957031,
"loss": 0.3037,
"rewards/accuracies": 0.02500000037252903,
"rewards/chosen": -0.0006309890886768699,
"rewards/margins": 4.580840322887525e-05,
"rewards/rejected": -0.0006767975282855332,
"step": 870
},
{
"epoch": 0.7,
"learning_rate": 1.2198928378235717e-06,
"logits/chosen": -1.9987977743148804,
"logits/rejected": -1.9987977743148804,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3507,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 880
},
{
"epoch": 0.71,
"learning_rate": 1.160433012552508e-06,
"logits/chosen": -1.9310076236724854,
"logits/rejected": -1.9310076236724854,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3531,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 890
},
{
"epoch": 0.72,
"learning_rate": 1.1020177413231334e-06,
"logits/chosen": -1.7657368183135986,
"logits/rejected": -1.7657368183135986,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3926,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 900
},
{
"epoch": 0.72,
"eval_logits/chosen": -1.95650053024292,
"eval_logits/rejected": -1.7981593608856201,
"eval_logps/chosen": -291.2066650390625,
"eval_logps/rejected": -277.3841552734375,
"eval_loss": 0.05500446632504463,
"eval_rewards/accuracies": 0.3840000033378601,
"eval_rewards/chosen": -0.024509064853191376,
"eval_rewards/margins": -0.002064692322164774,
"eval_rewards/rejected": -0.02244437113404274,
"eval_runtime": 726.2446,
"eval_samples_per_second": 2.754,
"eval_steps_per_second": 1.377,
"step": 900
},
{
"epoch": 0.73,
"learning_rate": 1.0446925746067768e-06,
"logits/chosen": -1.853788137435913,
"logits/rejected": -1.853788137435913,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3926,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 910
},
{
"epoch": 0.74,
"learning_rate": 9.88502212844063e-07,
"logits/chosen": -1.7569034099578857,
"logits/rejected": -1.7553638219833374,
"logps/chosen": -3.4119091033935547,
"logps/rejected": -3.395718812942505,
"loss": 0.335,
"rewards/accuracies": 0.02500000037252903,
"rewards/chosen": -0.0012354147620499134,
"rewards/margins": 0.0007618310628458858,
"rewards/rejected": -0.001997245941311121,
"step": 920
},
{
"epoch": 0.74,
"learning_rate": 9.334904715888496e-07,
"logits/chosen": -1.8945032358169556,
"logits/rejected": -1.8945032358169556,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3507,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 930
},
{
"epoch": 0.75,
"learning_rate": 8.797002473421729e-07,
"logits/chosen": -1.8704814910888672,
"logits/rejected": -1.8704814910888672,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3581,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 940
},
{
"epoch": 0.76,
"learning_rate": 8.271734841028553e-07,
"logits/chosen": -1.9085773229599,
"logits/rejected": -1.9085773229599,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3457,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 950
},
{
"epoch": 0.77,
"learning_rate": 7.759511406608255e-07,
"logits/chosen": -1.8419487476348877,
"logits/rejected": -1.8419487476348877,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3482,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 960
},
{
"epoch": 0.78,
"learning_rate": 7.260731586586983e-07,
"logits/chosen": -1.9136149883270264,
"logits/rejected": -1.9136149883270264,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3433,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 970
},
{
"epoch": 0.78,
"learning_rate": 6.775784314464717e-07,
"logits/chosen": -1.899609923362732,
"logits/rejected": -1.899609923362732,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3507,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 980
},
{
"epoch": 0.79,
"learning_rate": 6.305047737536707e-07,
"logits/chosen": -1.7761380672454834,
"logits/rejected": -1.7761380672454834,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3062,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 990
},
{
"epoch": 0.8,
"learning_rate": 5.848888922025553e-07,
"logits/chosen": -1.7907581329345703,
"logits/rejected": -1.7907581329345703,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3655,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1000
},
{
"epoch": 0.8,
"eval_logits/chosen": -1.948217511177063,
"eval_logits/rejected": -1.7905058860778809,
"eval_logps/chosen": -292.09368896484375,
"eval_logps/rejected": -278.4593505859375,
"eval_loss": 0.05491425469517708,
"eval_rewards/accuracies": 0.38600000739097595,
"eval_rewards/chosen": -0.025396063923835754,
"eval_rewards/margins": -0.001876543858088553,
"eval_rewards/rejected": -0.023519521579146385,
"eval_runtime": 727.6638,
"eval_samples_per_second": 2.749,
"eval_steps_per_second": 1.374,
"step": 1000
},
{
"epoch": 0.81,
"learning_rate": 5.407663566854008e-07,
"logits/chosen": -1.9497054815292358,
"logits/rejected": -1.9497054815292358,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3408,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1010
},
{
"epoch": 0.82,
"learning_rate": 4.981715726281666e-07,
"logits/chosen": -1.6699402332305908,
"logits/rejected": -1.6699402332305908,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3186,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1020
},
{
"epoch": 0.82,
"learning_rate": 4.5713775416217884e-07,
"logits/chosen": -1.782236099243164,
"logits/rejected": -1.782236099243164,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.368,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1030
},
{
"epoch": 0.83,
"learning_rate": 4.1769689822475147e-07,
"logits/chosen": -1.8598238229751587,
"logits/rejected": -1.8598238229751587,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3136,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1040
},
{
"epoch": 0.84,
"learning_rate": 3.798797596089351e-07,
"logits/chosen": -2.01692795753479,
"logits/rejected": -2.01692795753479,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3383,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1050
},
{
"epoch": 0.85,
"learning_rate": 3.4371582698185636e-07,
"logits/chosen": -1.8278007507324219,
"logits/rejected": -1.8278007507324219,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.326,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1060
},
{
"epoch": 0.86,
"learning_rate": 3.092332998903416e-07,
"logits/chosen": -2.0067131519317627,
"logits/rejected": -2.0067131519317627,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3408,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1070
},
{
"epoch": 0.86,
"learning_rate": 2.764590667717562e-07,
"logits/chosen": -1.8435783386230469,
"logits/rejected": -1.8435783386230469,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3433,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1080
},
{
"epoch": 0.87,
"learning_rate": 2.454186839872158e-07,
"logits/chosen": -1.865335464477539,
"logits/rejected": -1.865335464477539,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3803,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1090
},
{
"epoch": 0.88,
"learning_rate": 2.1613635589349756e-07,
"logits/chosen": -1.829878807067871,
"logits/rejected": -1.8288015127182007,
"logps/chosen": -3.383352756500244,
"logps/rejected": -2.7885169982910156,
"loss": 0.3682,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.0015257485210895538,
"rewards/margins": -0.00039975493564270437,
"rewards/rejected": -0.001125993556343019,
"step": 1100
},
{
"epoch": 0.88,
"eval_logits/chosen": -1.9496779441833496,
"eval_logits/rejected": -1.791935920715332,
"eval_logps/chosen": -292.0441589355469,
"eval_logps/rejected": -278.3316955566406,
"eval_loss": 0.054946158081293106,
"eval_rewards/accuracies": 0.38499999046325684,
"eval_rewards/chosen": -0.025346575304865837,
"eval_rewards/margins": -0.0019547012634575367,
"eval_rewards/rejected": -0.02339187264442444,
"eval_runtime": 727.7574,
"eval_samples_per_second": 2.748,
"eval_steps_per_second": 1.374,
"step": 1100
},
{
"epoch": 0.89,
"learning_rate": 1.8863491596921745e-07,
"logits/chosen": -1.9723230600357056,
"logits/rejected": -1.9723230600357056,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3828,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1110
},
{
"epoch": 0.9,
"learning_rate": 1.629358090099639e-07,
"logits/chosen": -1.8381221294403076,
"logits/rejected": -1.8381221294403076,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3877,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1120
},
{
"epoch": 0.9,
"learning_rate": 1.3905907440629752e-07,
"logits/chosen": -1.6961113214492798,
"logits/rejected": -1.6961113214492798,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.363,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1130
},
{
"epoch": 0.91,
"learning_rate": 1.1702333051763271e-07,
"logits/chosen": -1.796628713607788,
"logits/rejected": -1.796628713607788,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3778,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1140
},
{
"epoch": 0.92,
"learning_rate": 9.684576015420277e-08,
"logits/chosen": -1.8420875072479248,
"logits/rejected": -1.8420875072479248,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3556,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1150
},
{
"epoch": 0.93,
"learning_rate": 7.854209717842231e-08,
"logits/chosen": -1.8557026386260986,
"logits/rejected": -1.8557026386260986,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3482,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1160
},
{
"epoch": 0.94,
"learning_rate": 6.212661423609184e-08,
"logits/chosen": -1.9385162591934204,
"logits/rejected": -1.9385162591934204,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3655,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1170
},
{
"epoch": 0.94,
"learning_rate": 4.761211162702117e-08,
"logits/chosen": -1.9021384716033936,
"logits/rejected": -1.9021384716033936,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.363,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1180
},
{
"epoch": 0.95,
"learning_rate": 3.5009907323737826e-08,
"logits/chosen": -1.8466148376464844,
"logits/rejected": -1.8466148376464844,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3087,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1190
},
{
"epoch": 0.96,
"learning_rate": 2.4329828146074096e-08,
"logits/chosen": -1.7545801401138306,
"logits/rejected": -1.7545801401138306,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3531,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1200
},
{
"epoch": 0.96,
"eval_logits/chosen": -1.949344515800476,
"eval_logits/rejected": -1.7915234565734863,
"eval_logps/chosen": -291.8377990722656,
"eval_logps/rejected": -278.0787048339844,
"eval_loss": 0.0550062395632267,
"eval_rewards/accuracies": 0.39100000262260437,
"eval_rewards/chosen": -0.02514021284878254,
"eval_rewards/margins": -0.0020012950990349054,
"eval_rewards/rejected": -0.02313891611993313,
"eval_runtime": 729.194,
"eval_samples_per_second": 2.743,
"eval_steps_per_second": 1.371,
"step": 1200
},
{
"epoch": 0.97,
"learning_rate": 1.5580202098509078e-08,
"logits/chosen": -1.6881771087646484,
"logits/rejected": -1.6881771087646484,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3087,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1210
},
{
"epoch": 0.98,
"learning_rate": 8.767851876239075e-09,
"logits/chosen": -1.9288533926010132,
"logits/rejected": -1.9288533926010132,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.368,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1220
},
{
"epoch": 0.98,
"learning_rate": 3.8980895450474455e-09,
"logits/chosen": -1.8713302612304688,
"logits/rejected": -1.8713302612304688,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.3606,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1230
},
{
"epoch": 0.99,
"learning_rate": 9.747123991141193e-10,
"logits/chosen": -1.8912433385849,
"logits/rejected": -1.8912433385849,
"logps/chosen": 0.0,
"logps/rejected": 0.0,
"loss": 0.2791,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1240
},
{
"epoch": 1.0,
"learning_rate": 0.0,
"logits/chosen": -1.8090438842773438,
"logits/rejected": -1.800258994102478,
"logps/chosen": -5.058285713195801,
"logps/rejected": -3.845151901245117,
"loss": 0.3094,
"rewards/accuracies": 0.0,
"rewards/chosen": -0.002232617698609829,
"rewards/margins": -0.0004155985952820629,
"rewards/rejected": -0.0018170190742239356,
"step": 1250
},
{
"epoch": 1.0,
"step": 1250,
"total_flos": 0.0,
"train_loss": 0.3462253446340561,
"train_runtime": 13267.124,
"train_samples_per_second": 0.377,
"train_steps_per_second": 0.094
}
],
"logging_steps": 10,
"max_steps": 1250,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 100,
"total_flos": 0.0,
"train_batch_size": 2,
"trial_name": null,
"trial_params": null
}