{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 3000,
  "global_step": 7642,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 6.535947712418301e-09,
      "logits/chosen": -3.15731143951416,
      "logits/rejected": -2.798032760620117,
      "logps/chosen": -362.5858154296875,
      "logps/rejected": -239.5453338623047,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.0,
      "learning_rate": 6.535947712418302e-08,
      "logits/chosen": -2.847041130065918,
      "logits/rejected": -2.83606219291687,
      "logps/chosen": -298.7374267578125,
      "logps/rejected": -247.803955078125,
      "loss": 0.6925,
      "rewards/accuracies": 0.5277777910232544,
      "rewards/chosen": 0.0011123637668788433,
      "rewards/margins": 0.0014208466745913029,
      "rewards/rejected": -0.00030848290771245956,
      "step": 10
    },
    {
      "epoch": 0.0,
      "learning_rate": 1.3071895424836603e-07,
      "logits/chosen": -2.7611355781555176,
      "logits/rejected": -2.752760648727417,
      "logps/chosen": -262.10382080078125,
      "logps/rejected": -281.1115417480469,
      "loss": 0.6929,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.0011931858025491238,
      "rewards/margins": 9.312294650953845e-07,
      "rewards/rejected": 0.0011922543635591865,
      "step": 20
    },
    {
      "epoch": 0.0,
      "learning_rate": 1.9607843137254904e-07,
      "logits/chosen": -2.932368516921997,
      "logits/rejected": -2.9066896438598633,
      "logps/chosen": -252.71548461914062,
      "logps/rejected": -236.00338745117188,
      "loss": 0.6932,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.00011899097444256768,
      "rewards/margins": 0.0004326296038925648,
      "rewards/rejected": -0.00031363870948553085,
      "step": 30
    },
    {
      "epoch": 0.01,
      "learning_rate": 2.6143790849673207e-07,
      "logits/chosen": -2.873145341873169,
      "logits/rejected": -2.880809783935547,
      "logps/chosen": -259.3544921875,
      "logps/rejected": -247.77493286132812,
      "loss": 0.6929,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.0008003931725397706,
      "rewards/margins": 0.0007967362180352211,
      "rewards/rejected": 3.6569545045495033e-06,
      "step": 40
    },
    {
      "epoch": 0.01,
      "learning_rate": 3.267973856209151e-07,
      "logits/chosen": -2.883204460144043,
      "logits/rejected": -2.9383983612060547,
      "logps/chosen": -307.0992431640625,
      "logps/rejected": -261.85723876953125,
      "loss": 0.6925,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0009268176509067416,
      "rewards/margins": 0.0016272146021947265,
      "rewards/rejected": -0.0007003970677033067,
      "step": 50
    },
    {
      "epoch": 0.01,
      "learning_rate": 3.921568627450981e-07,
      "logits/chosen": -2.952158212661743,
      "logits/rejected": -2.9316773414611816,
      "logps/chosen": -337.2663269042969,
      "logps/rejected": -268.79132080078125,
      "loss": 0.6934,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.0011328494874760509,
      "rewards/margins": 0.000162464493769221,
      "rewards/rejected": 0.0009703851537778974,
      "step": 60
    },
    {
      "epoch": 0.01,
      "learning_rate": 4.5751633986928105e-07,
      "logits/chosen": -2.8809123039245605,
      "logits/rejected": -2.8493149280548096,
      "logps/chosen": -296.692138671875,
      "logps/rejected": -298.32220458984375,
      "loss": 0.6932,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": -0.00022940686903893948,
      "rewards/margins": -0.00013572932220995426,
      "rewards/rejected": -9.367751044919714e-05,
      "step": 70
    },
    {
      "epoch": 0.01,
      "learning_rate": 5.228758169934641e-07,
      "logits/chosen": -2.8788790702819824,
      "logits/rejected": -2.8450794219970703,
      "logps/chosen": -297.30792236328125,
      "logps/rejected": -261.411376953125,
      "loss": 0.6932,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.0009753646445460618,
      "rewards/margins": 0.0006360862171277404,
      "rewards/rejected": 0.00033927842741832137,
      "step": 80
    },
    {
      "epoch": 0.01,
      "learning_rate": 5.882352941176471e-07,
      "logits/chosen": -2.877856731414795,
      "logits/rejected": -2.8819994926452637,
      "logps/chosen": -253.0044708251953,
      "logps/rejected": -222.84390258789062,
      "loss": 0.6929,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.0003014683607034385,
      "rewards/margins": 0.0006676745833829045,
      "rewards/rejected": -0.0003662061644718051,
      "step": 90
    },
    {
      "epoch": 0.01,
      "learning_rate": 6.535947712418302e-07,
      "logits/chosen": -2.8481335639953613,
      "logits/rejected": -2.7708733081817627,
      "logps/chosen": -276.61749267578125,
      "logps/rejected": -276.0560607910156,
      "loss": 0.6936,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 6.637955084443092e-05,
      "rewards/margins": -0.0015732726315036416,
      "rewards/rejected": 0.0016396520659327507,
      "step": 100
    },
    {
      "epoch": 0.01,
      "learning_rate": 7.189542483660131e-07,
      "logits/chosen": -2.812410831451416,
      "logits/rejected": -2.8500609397888184,
      "logps/chosen": -302.9320373535156,
      "logps/rejected": -320.2331237792969,
      "loss": 0.6933,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.00024477095576003194,
      "rewards/margins": -0.0008945292793214321,
      "rewards/rejected": 0.0011393001768738031,
      "step": 110
    },
    {
      "epoch": 0.02,
      "learning_rate": 7.843137254901962e-07,
      "logits/chosen": -2.7782273292541504,
      "logits/rejected": -2.786151170730591,
      "logps/chosen": -235.53518676757812,
      "logps/rejected": -234.79306030273438,
      "loss": 0.6934,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.00024773445329628885,
      "rewards/margins": -0.001447710208594799,
      "rewards/rejected": 0.0016954450402408838,
      "step": 120
    },
    {
      "epoch": 0.02,
      "learning_rate": 8.496732026143792e-07,
      "logits/chosen": -2.6703851222991943,
      "logits/rejected": -2.723660469055176,
      "logps/chosen": -279.0316467285156,
      "logps/rejected": -230.14675903320312,
      "loss": 0.693,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0005290083936415613,
      "rewards/margins": 0.0002718138857744634,
      "rewards/rejected": 0.0002571945369709283,
      "step": 130
    },
    {
      "epoch": 0.02,
      "learning_rate": 9.150326797385621e-07,
      "logits/chosen": -2.8054141998291016,
      "logits/rejected": -2.8203861713409424,
      "logps/chosen": -239.762939453125,
      "logps/rejected": -234.76779174804688,
      "loss": 0.6931,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -9.767948358785361e-05,
      "rewards/margins": -0.000681340170558542,
      "rewards/rejected": 0.0005836606142111123,
      "step": 140
    },
    {
      "epoch": 0.02,
      "learning_rate": 9.80392156862745e-07,
      "logits/chosen": -2.7740328311920166,
      "logits/rejected": -2.7924256324768066,
      "logps/chosen": -246.41683959960938,
      "logps/rejected": -292.7257385253906,
      "loss": 0.6934,
      "rewards/accuracies": 0.32499998807907104,
      "rewards/chosen": 9.760662214830518e-05,
      "rewards/margins": -0.0012625637464225292,
      "rewards/rejected": 0.0013601703103631735,
      "step": 150
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.0457516339869283e-06,
      "logits/chosen": -2.7917070388793945,
      "logits/rejected": -2.72552490234375,
      "logps/chosen": -247.941650390625,
      "logps/rejected": -256.9376220703125,
      "loss": 0.6931,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0016826872015371919,
      "rewards/margins": 0.0002282693749293685,
      "rewards/rejected": 0.0014544179430231452,
      "step": 160
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.111111111111111e-06,
      "logits/chosen": -2.9668118953704834,
      "logits/rejected": -2.855332612991333,
      "logps/chosen": -272.6416320800781,
      "logps/rejected": -224.99267578125,
      "loss": 0.6932,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.0012751845642924309,
      "rewards/margins": 0.0004584183043334633,
      "rewards/rejected": 0.0008167661726474762,
      "step": 170
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.1764705882352942e-06,
      "logits/chosen": -2.8704068660736084,
      "logits/rejected": -2.86428165435791,
      "logps/chosen": -198.56057739257812,
      "logps/rejected": -208.8024444580078,
      "loss": 0.6931,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0017385890241712332,
      "rewards/margins": 0.0009926703060045838,
      "rewards/rejected": 0.0007459186599589884,
      "step": 180
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.2418300653594772e-06,
      "logits/chosen": -2.9300076961517334,
      "logits/rejected": -2.964507818222046,
      "logps/chosen": -298.9310302734375,
      "logps/rejected": -239.85885620117188,
      "loss": 0.6929,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.0012464747997000813,
      "rewards/margins": -0.000528771779499948,
      "rewards/rejected": 0.0017752466956153512,
      "step": 190
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.3071895424836604e-06,
      "logits/chosen": -2.8765249252319336,
      "logits/rejected": -2.7931816577911377,
      "logps/chosen": -252.77047729492188,
      "logps/rejected": -220.84866333007812,
      "loss": 0.6928,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.0029885726980865,
      "rewards/margins": 0.0007496483740396798,
      "rewards/rejected": 0.002238924615085125,
      "step": 200
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.3725490196078434e-06,
      "logits/chosen": -2.8990020751953125,
      "logits/rejected": -2.867999792098999,
      "logps/chosen": -246.72994995117188,
      "logps/rejected": -271.1702880859375,
      "loss": 0.6928,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.0023519580718129873,
      "rewards/margins": 0.001143458066508174,
      "rewards/rejected": 0.0012085003545507789,
      "step": 210
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.4379084967320261e-06,
      "logits/chosen": -2.86352276802063,
      "logits/rejected": -2.8969032764434814,
      "logps/chosen": -241.1385040283203,
      "logps/rejected": -232.1473388671875,
      "loss": 0.6929,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.001888754777610302,
      "rewards/margins": -0.0003588128020055592,
      "rewards/rejected": 0.0022475675214082003,
      "step": 220
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.5032679738562091e-06,
      "logits/chosen": -2.8990299701690674,
      "logits/rejected": -2.8520991802215576,
      "logps/chosen": -261.7444152832031,
      "logps/rejected": -233.3668670654297,
      "loss": 0.6925,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.002989122411236167,
      "rewards/margins": 0.0005400863010436296,
      "rewards/rejected": 0.002449036343023181,
      "step": 230
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.5686274509803923e-06,
      "logits/chosen": -2.907646656036377,
      "logits/rejected": -2.817707061767578,
      "logps/chosen": -282.6989440917969,
      "logps/rejected": -309.9423828125,
      "loss": 0.6925,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0033154035918414593,
      "rewards/margins": 0.0005889445310458541,
      "rewards/rejected": 0.002726459177210927,
      "step": 240
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.6339869281045753e-06,
      "logits/chosen": -2.7770705223083496,
      "logits/rejected": -2.824580669403076,
      "logps/chosen": -272.4206848144531,
      "logps/rejected": -261.48199462890625,
      "loss": 0.6925,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.004111588001251221,
      "rewards/margins": 0.001057318178936839,
      "rewards/rejected": 0.0030542700551450253,
      "step": 250
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.6993464052287585e-06,
      "logits/chosen": -2.868993043899536,
      "logits/rejected": -2.860084056854248,
      "logps/chosen": -339.67071533203125,
      "logps/rejected": -264.09552001953125,
      "loss": 0.6927,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.0046674818731844425,
      "rewards/margins": 0.0013502954971045256,
      "rewards/rejected": 0.0033171861432492733,
      "step": 260
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.7647058823529414e-06,
      "logits/chosen": -2.97316575050354,
      "logits/rejected": -2.9198946952819824,
      "logps/chosen": -300.5587463378906,
      "logps/rejected": -270.83416748046875,
      "loss": 0.6927,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.004219466354697943,
      "rewards/margins": 0.0005742977955378592,
      "rewards/rejected": 0.0036451686173677444,
      "step": 270
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.8300653594771242e-06,
      "logits/chosen": -2.8331806659698486,
      "logits/rejected": -2.8653924465179443,
      "logps/chosen": -303.3408508300781,
      "logps/rejected": -245.0085906982422,
      "loss": 0.6915,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.004968521185219288,
      "rewards/margins": 0.0015199267072603106,
      "rewards/rejected": 0.0034485943615436554,
      "step": 280
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.8954248366013072e-06,
      "logits/chosen": -2.8719141483306885,
      "logits/rejected": -2.8208324909210205,
      "logps/chosen": -257.5692443847656,
      "logps/rejected": -274.0968017578125,
      "loss": 0.6931,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.004160556010901928,
      "rewards/margins": -0.0008697215234860778,
      "rewards/rejected": 0.0050302776508033276,
      "step": 290
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.96078431372549e-06,
      "logits/chosen": -2.9372448921203613,
      "logits/rejected": -2.9619555473327637,
      "logps/chosen": -271.332275390625,
      "logps/rejected": -278.2612609863281,
      "loss": 0.6916,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.008250665850937366,
      "rewards/margins": 0.0011537367245182395,
      "rewards/rejected": 0.007096928544342518,
      "step": 300
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.0261437908496734e-06,
      "logits/chosen": -2.7562594413757324,
      "logits/rejected": -2.7253847122192383,
      "logps/chosen": -296.3819274902344,
      "logps/rejected": -246.0597381591797,
      "loss": 0.6918,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.008744063787162304,
      "rewards/margins": 0.0032774447463452816,
      "rewards/rejected": 0.005466618575155735,
      "step": 310
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.0915032679738565e-06,
      "logits/chosen": -2.858609676361084,
      "logits/rejected": -2.918300151824951,
      "logps/chosen": -207.58529663085938,
      "logps/rejected": -209.2336883544922,
      "loss": 0.6913,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.011316259391605854,
      "rewards/margins": 0.00452456995844841,
      "rewards/rejected": 0.006791689433157444,
      "step": 320
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.1568627450980393e-06,
      "logits/chosen": -3.0018887519836426,
      "logits/rejected": -2.9340403079986572,
      "logps/chosen": -324.8250427246094,
      "logps/rejected": -265.0435485839844,
      "loss": 0.6918,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.012073887512087822,
      "rewards/margins": 0.004768506623804569,
      "rewards/rejected": 0.007305382285267115,
      "step": 330
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.222222222222222e-06,
      "logits/chosen": -2.821833372116089,
      "logits/rejected": -2.827418088912964,
      "logps/chosen": -325.23211669921875,
      "logps/rejected": -223.10629272460938,
      "loss": 0.6914,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.011198493652045727,
      "rewards/margins": 0.004123987164348364,
      "rewards/rejected": 0.007074506487697363,
      "step": 340
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.2875816993464053e-06,
      "logits/chosen": -2.8357648849487305,
      "logits/rejected": -2.8311212062835693,
      "logps/chosen": -237.04891967773438,
      "logps/rejected": -237.57809448242188,
      "loss": 0.6907,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.012098469771444798,
      "rewards/margins": 0.0031062206253409386,
      "rewards/rejected": 0.008992249146103859,
      "step": 350
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.3529411764705885e-06,
      "logits/chosen": -2.8127646446228027,
      "logits/rejected": -2.7550415992736816,
      "logps/chosen": -274.84478759765625,
      "logps/rejected": -223.2805938720703,
      "loss": 0.6911,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.016145002096891403,
      "rewards/margins": 0.0037769395858049393,
      "rewards/rejected": 0.012368063442409039,
      "step": 360
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.4183006535947716e-06,
      "logits/chosen": -2.92099666595459,
      "logits/rejected": -2.9377002716064453,
      "logps/chosen": -241.8701629638672,
      "logps/rejected": -393.9468078613281,
      "loss": 0.6906,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.013012359850108624,
      "rewards/margins": 0.0014860105002298951,
      "rewards/rejected": 0.011526349931955338,
      "step": 370
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.4836601307189544e-06,
      "logits/chosen": -2.8547887802124023,
      "logits/rejected": -2.816089391708374,
      "logps/chosen": -226.1658172607422,
      "logps/rejected": -205.92953491210938,
      "loss": 0.6902,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.01577352173626423,
      "rewards/margins": 0.006564581301063299,
      "rewards/rejected": 0.009208940900862217,
      "step": 380
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.549019607843137e-06,
      "logits/chosen": -2.8543856143951416,
      "logits/rejected": -2.809668779373169,
      "logps/chosen": -304.81158447265625,
      "logps/rejected": -267.87872314453125,
      "loss": 0.6893,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.01894640363752842,
      "rewards/margins": 0.008028329350054264,
      "rewards/rejected": 0.01091807335615158,
      "step": 390
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.6143790849673208e-06,
      "logits/chosen": -2.906557083129883,
      "logits/rejected": -2.9965012073516846,
      "logps/chosen": -292.5071105957031,
      "logps/rejected": -282.63116455078125,
      "loss": 0.6903,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.022404711693525314,
      "rewards/margins": 0.005580331198871136,
      "rewards/rejected": 0.016824383288621902,
      "step": 400
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.6797385620915036e-06,
      "logits/chosen": -2.950514793395996,
      "logits/rejected": -2.924778461456299,
      "logps/chosen": -254.015869140625,
      "logps/rejected": -235.6728973388672,
      "loss": 0.6914,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.019588075578212738,
      "rewards/margins": 0.006429559551179409,
      "rewards/rejected": 0.013158516958355904,
      "step": 410
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.7450980392156867e-06,
      "logits/chosen": -2.8651256561279297,
      "logits/rejected": -2.859278917312622,
      "logps/chosen": -397.34173583984375,
      "logps/rejected": -366.5126953125,
      "loss": 0.691,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.023986633867025375,
      "rewards/margins": 0.005491149611771107,
      "rewards/rejected": 0.018495485186576843,
      "step": 420
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.8104575163398695e-06,
      "logits/chosen": -2.8140807151794434,
      "logits/rejected": -2.8027613162994385,
      "logps/chosen": -284.19488525390625,
      "logps/rejected": -286.6159362792969,
      "loss": 0.6877,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.024940496310591698,
      "rewards/margins": 0.010390857234597206,
      "rewards/rejected": 0.014549640007317066,
      "step": 430
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.8758169934640523e-06,
      "logits/chosen": -2.91127872467041,
      "logits/rejected": -2.9656119346618652,
      "logps/chosen": -289.0911865234375,
      "logps/rejected": -265.87554931640625,
      "loss": 0.6887,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02747970260679722,
      "rewards/margins": 0.00899188220500946,
      "rewards/rejected": 0.018487820401787758,
      "step": 440
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.9411764705882355e-06,
      "logits/chosen": -2.8094940185546875,
      "logits/rejected": -2.7788679599761963,
      "logps/chosen": -234.64382934570312,
      "logps/rejected": -245.06405639648438,
      "loss": 0.6896,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.02517036534845829,
      "rewards/margins": 0.008376935496926308,
      "rewards/rejected": 0.01679343171417713,
      "step": 450
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.0065359477124182e-06,
      "logits/chosen": -2.8675389289855957,
      "logits/rejected": -2.8082258701324463,
      "logps/chosen": -291.63427734375,
      "logps/rejected": -189.21340942382812,
      "loss": 0.6859,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.03575144335627556,
      "rewards/margins": 0.018049785867333412,
      "rewards/rejected": 0.017701663076877594,
      "step": 460
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.071895424836602e-06,
      "logits/chosen": -2.8911495208740234,
      "logits/rejected": -2.9287312030792236,
      "logps/chosen": -260.6916198730469,
      "logps/rejected": -240.414794921875,
      "loss": 0.6881,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.02840578183531761,
      "rewards/margins": 0.010620644316077232,
      "rewards/rejected": 0.01778513565659523,
      "step": 470
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.1372549019607846e-06,
      "logits/chosen": -2.850501537322998,
      "logits/rejected": -2.8640151023864746,
      "logps/chosen": -284.7303161621094,
      "logps/rejected": -307.1053161621094,
      "loss": 0.6903,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.03151124715805054,
      "rewards/margins": 0.004283557645976543,
      "rewards/rejected": 0.02722769044339657,
      "step": 480
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.2026143790849674e-06,
      "logits/chosen": -2.9750914573669434,
      "logits/rejected": -2.8569235801696777,
      "logps/chosen": -270.89971923828125,
      "logps/rejected": -239.67733764648438,
      "loss": 0.6867,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.031015124171972275,
      "rewards/margins": 0.01439888495951891,
      "rewards/rejected": 0.01661624014377594,
      "step": 490
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.2679738562091506e-06,
      "logits/chosen": -2.913080930709839,
      "logits/rejected": -2.9230728149414062,
      "logps/chosen": -231.50753784179688,
      "logps/rejected": -235.9259490966797,
      "loss": 0.6861,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.03188391774892807,
      "rewards/margins": 0.019386431202292442,
      "rewards/rejected": 0.012497490271925926,
      "step": 500
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.3333333333333333e-06,
      "logits/chosen": -2.949521541595459,
      "logits/rejected": -2.937497615814209,
      "logps/chosen": -352.7322082519531,
      "logps/rejected": -253.5258331298828,
      "loss": 0.6828,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.04447092488408089,
      "rewards/margins": 0.024645883589982986,
      "rewards/rejected": 0.0198250412940979,
      "step": 510
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.398692810457517e-06,
      "logits/chosen": -2.9339003562927246,
      "logits/rejected": -2.9172234535217285,
      "logps/chosen": -302.03924560546875,
      "logps/rejected": -246.05117797851562,
      "loss": 0.6859,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.0391179658472538,
      "rewards/margins": 0.015613268129527569,
      "rewards/rejected": 0.023504700511693954,
      "step": 520
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.4640522875816997e-06,
      "logits/chosen": -2.9032986164093018,
      "logits/rejected": -2.9032576084136963,
      "logps/chosen": -329.2999267578125,
      "logps/rejected": -275.84149169921875,
      "loss": 0.6858,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.04176343232393265,
      "rewards/margins": 0.017148349434137344,
      "rewards/rejected": 0.024615082889795303,
      "step": 530
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.529411764705883e-06,
      "logits/chosen": -2.925100803375244,
      "logits/rejected": -2.940062999725342,
      "logps/chosen": -306.13970947265625,
      "logps/rejected": -286.7759704589844,
      "loss": 0.6865,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.04364129155874252,
      "rewards/margins": 0.011506977491080761,
      "rewards/rejected": 0.03213431313633919,
      "step": 540
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.5947712418300657e-06,
      "logits/chosen": -2.7900989055633545,
      "logits/rejected": -2.7149672508239746,
      "logps/chosen": -275.6036682128906,
      "logps/rejected": -221.5449981689453,
      "loss": 0.6817,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.04563576728105545,
      "rewards/margins": 0.026486817747354507,
      "rewards/rejected": 0.019148947671055794,
      "step": 550
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.6601307189542484e-06,
      "logits/chosen": -2.8575167655944824,
      "logits/rejected": -2.919316053390503,
      "logps/chosen": -268.87335205078125,
      "logps/rejected": -253.87344360351562,
      "loss": 0.6828,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.041931238025426865,
      "rewards/margins": 0.02648743987083435,
      "rewards/rejected": 0.015443804673850536,
      "step": 560
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.7254901960784316e-06,
      "logits/chosen": -2.7716431617736816,
      "logits/rejected": -2.86437726020813,
      "logps/chosen": -313.5101013183594,
      "logps/rejected": -239.9435577392578,
      "loss": 0.6814,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.049281515181064606,
      "rewards/margins": 0.032593242824077606,
      "rewards/rejected": 0.016688276082277298,
      "step": 570
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.7908496732026144e-06,
      "logits/chosen": -2.8355050086975098,
      "logits/rejected": -2.7985503673553467,
      "logps/chosen": -324.6849670410156,
      "logps/rejected": -237.7136688232422,
      "loss": 0.68,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04397376626729965,
      "rewards/margins": 0.024682357907295227,
      "rewards/rejected": 0.019291408360004425,
      "step": 580
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.856209150326798e-06,
      "logits/chosen": -2.890514850616455,
      "logits/rejected": -2.913870334625244,
      "logps/chosen": -237.35205078125,
      "logps/rejected": -230.11428833007812,
      "loss": 0.6826,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.044465962797403336,
      "rewards/margins": 0.021796438843011856,
      "rewards/rejected": 0.02266952209174633,
      "step": 590
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.92156862745098e-06,
      "logits/chosen": -2.872098207473755,
      "logits/rejected": -2.8931877613067627,
      "logps/chosen": -275.00848388671875,
      "logps/rejected": -267.65802001953125,
      "loss": 0.6812,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04082503542304039,
      "rewards/margins": 0.026575183495879173,
      "rewards/rejected": 0.014249855652451515,
      "step": 600
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.986928104575164e-06,
      "logits/chosen": -2.755969285964966,
      "logits/rejected": -2.7946555614471436,
      "logps/chosen": -327.8434143066406,
      "logps/rejected": -279.28509521484375,
      "loss": 0.6817,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.04074610769748688,
      "rewards/margins": 0.032059285789728165,
      "rewards/rejected": 0.008686823770403862,
      "step": 610
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.052287581699347e-06,
      "logits/chosen": -2.881762981414795,
      "logits/rejected": -2.8726096153259277,
      "logps/chosen": -274.4825744628906,
      "logps/rejected": -254.6781768798828,
      "loss": 0.6782,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.04438147693872452,
      "rewards/margins": 0.020229998975992203,
      "rewards/rejected": 0.024151481688022614,
      "step": 620
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.11764705882353e-06,
      "logits/chosen": -2.784241199493408,
      "logits/rejected": -2.8636996746063232,
      "logps/chosen": -220.9033203125,
      "logps/rejected": -205.02206420898438,
      "loss": 0.681,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.029765665531158447,
      "rewards/margins": 0.020760158076882362,
      "rewards/rejected": 0.009005507454276085,
      "step": 630
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.183006535947713e-06,
      "logits/chosen": -2.87015962600708,
      "logits/rejected": -2.8887946605682373,
      "logps/chosen": -290.60382080078125,
      "logps/rejected": -308.30462646484375,
      "loss": 0.6756,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.0495864637196064,
      "rewards/margins": 0.036216553300619125,
      "rewards/rejected": 0.013369908556342125,
      "step": 640
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.2483660130718954e-06,
      "logits/chosen": -2.8120949268341064,
      "logits/rejected": -2.882768154144287,
      "logps/chosen": -310.6004333496094,
      "logps/rejected": -279.83148193359375,
      "loss": 0.6831,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.04262063279747963,
      "rewards/margins": 0.016967860981822014,
      "rewards/rejected": 0.025652769953012466,
      "step": 650
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.313725490196079e-06,
      "logits/chosen": -2.878135919570923,
      "logits/rejected": -2.896784543991089,
      "logps/chosen": -235.76217651367188,
      "logps/rejected": -198.4434814453125,
      "loss": 0.6812,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.036157600581645966,
      "rewards/margins": 0.02879006788134575,
      "rewards/rejected": 0.007367536425590515,
      "step": 660
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.379084967320262e-06,
      "logits/chosen": -2.8781301975250244,
      "logits/rejected": -2.846781015396118,
      "logps/chosen": -255.5959014892578,
      "logps/rejected": -259.38726806640625,
      "loss": 0.6795,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.04209792613983154,
      "rewards/margins": 0.035201676189899445,
      "rewards/rejected": 0.006896245293319225,
      "step": 670
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.444444444444444e-06,
      "logits/chosen": -2.701145648956299,
      "logits/rejected": -2.797853946685791,
      "logps/chosen": -217.748046875,
      "logps/rejected": -192.89178466796875,
      "loss": 0.6766,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.03686373680830002,
      "rewards/margins": 0.03340522199869156,
      "rewards/rejected": 0.0034585162065923214,
      "step": 680
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.509803921568628e-06,
      "logits/chosen": -2.799405097961426,
      "logits/rejected": -2.800239324569702,
      "logps/chosen": -224.0169219970703,
      "logps/rejected": -208.5043487548828,
      "loss": 0.6764,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.020510878413915634,
      "rewards/margins": 0.03946324437856674,
      "rewards/rejected": -0.01895236223936081,
      "step": 690
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.5751633986928105e-06,
      "logits/chosen": -2.8377106189727783,
      "logits/rejected": -2.7427613735198975,
      "logps/chosen": -308.39202880859375,
      "logps/rejected": -265.7181701660156,
      "loss": 0.6783,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.01599673368036747,
      "rewards/margins": 0.02599218487739563,
      "rewards/rejected": -0.009995453990995884,
      "step": 700
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.640522875816994e-06,
      "logits/chosen": -2.791973352432251,
      "logits/rejected": -2.7882332801818848,
      "logps/chosen": -321.95928955078125,
      "logps/rejected": -268.7115783691406,
      "loss": 0.6702,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.038942694664001465,
      "rewards/margins": 0.06790758669376373,
      "rewards/rejected": -0.02896488830447197,
      "step": 710
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.705882352941177e-06,
      "logits/chosen": -2.8617916107177734,
      "logits/rejected": -2.833081007003784,
      "logps/chosen": -240.85415649414062,
      "logps/rejected": -262.15325927734375,
      "loss": 0.6694,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.03635946661233902,
      "rewards/margins": 0.04408254846930504,
      "rewards/rejected": -0.007723079528659582,
      "step": 720
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.77124183006536e-06,
      "logits/chosen": -2.954502582550049,
      "logits/rejected": -2.8966224193573,
      "logps/chosen": -286.0347595214844,
      "logps/rejected": -258.43487548828125,
      "loss": 0.6733,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.03311768174171448,
      "rewards/margins": 0.04485425353050232,
      "rewards/rejected": -0.011736570857465267,
      "step": 730
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.836601307189543e-06,
      "logits/chosen": -2.7532668113708496,
      "logits/rejected": -2.789754867553711,
      "logps/chosen": -287.6087341308594,
      "logps/rejected": -268.8944396972656,
      "loss": 0.6721,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.02871871553361416,
      "rewards/margins": 0.06711134314537048,
      "rewards/rejected": -0.03839262202382088,
      "step": 740
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.901960784313726e-06,
      "logits/chosen": -2.722290277481079,
      "logits/rejected": -2.757640838623047,
      "logps/chosen": -299.148681640625,
      "logps/rejected": -257.52972412109375,
      "loss": 0.6725,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.006840469781309366,
      "rewards/margins": 0.05222883075475693,
      "rewards/rejected": -0.045388363301754,
      "step": 750
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.967320261437909e-06,
      "logits/chosen": -2.7702317237854004,
      "logits/rejected": -2.718785285949707,
      "logps/chosen": -237.90707397460938,
      "logps/rejected": -182.05593872070312,
      "loss": 0.6783,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.016088716685771942,
      "rewards/margins": 0.03843966871500015,
      "rewards/rejected": -0.05452839285135269,
      "step": 760
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.99999347843947e-06,
      "logits/chosen": -2.893463611602783,
      "logits/rejected": -2.838412046432495,
      "logps/chosen": -257.88311767578125,
      "logps/rejected": -229.4723358154297,
      "loss": 0.6723,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.025889281183481216,
      "rewards/margins": 0.03446760028600693,
      "rewards/rejected": -0.06035688519477844,
      "step": 770
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999941306159375e-06,
      "logits/chosen": -2.819159984588623,
      "logits/rejected": -2.859947919845581,
      "logps/chosen": -302.0553283691406,
      "logps/rejected": -292.2879638671875,
      "loss": 0.6758,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.051253754645586014,
      "rewards/margins": 0.03177189081907272,
      "rewards/rejected": -0.08302563428878784,
      "step": 780
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999836962687967e-06,
      "logits/chosen": -2.6668179035186768,
      "logits/rejected": -2.667832851409912,
      "logps/chosen": -262.34027099609375,
      "logps/rejected": -295.4078674316406,
      "loss": 0.6779,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.02748146653175354,
      "rewards/margins": 0.035053275525569916,
      "rewards/rejected": -0.06253473460674286,
      "step": 790
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999680450202786e-06,
      "logits/chosen": -2.7049105167388916,
      "logits/rejected": -2.6771888732910156,
      "logps/chosen": -341.10137939453125,
      "logps/rejected": -301.953857421875,
      "loss": 0.6637,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.009321318939328194,
      "rewards/margins": 0.09051088988780975,
      "rewards/rejected": -0.08118956536054611,
      "step": 800
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.999471771970087e-06,
      "logits/chosen": -2.892444372177124,
      "logits/rejected": -2.9383559226989746,
      "logps/chosen": -229.4305877685547,
      "logps/rejected": -203.98361206054688,
      "loss": 0.6734,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0560198538005352,
      "rewards/margins": 0.05636056512594223,
      "rewards/rejected": -0.11238042265176773,
      "step": 810
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.999210932344767e-06,
      "logits/chosen": -2.910004138946533,
      "logits/rejected": -2.824584484100342,
      "logps/chosen": -305.6726989746094,
      "logps/rejected": -284.2978210449219,
      "loss": 0.6624,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.042909882962703705,
      "rewards/margins": 0.06963402032852173,
      "rewards/rejected": -0.11254390329122543,
      "step": 820
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.998897936770281e-06,
      "logits/chosen": -2.7205851078033447,
      "logits/rejected": -2.6165709495544434,
      "logps/chosen": -245.6926727294922,
      "logps/rejected": -230.71853637695312,
      "loss": 0.659,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07828576862812042,
      "rewards/margins": 0.0336625911295414,
      "rewards/rejected": -0.11194835603237152,
      "step": 830
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.998532791778521e-06,
      "logits/chosen": -2.8109004497528076,
      "logits/rejected": -2.7994141578674316,
      "logps/chosen": -290.1483459472656,
      "logps/rejected": -229.80569458007812,
      "loss": 0.6652,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.052752308547496796,
      "rewards/margins": 0.05021792650222778,
      "rewards/rejected": -0.10297022759914398,
      "step": 840
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.9981155049896885e-06,
      "logits/chosen": -2.822997570037842,
      "logits/rejected": -2.8100199699401855,
      "logps/chosen": -293.0641174316406,
      "logps/rejected": -258.8555603027344,
      "loss": 0.6549,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.032249774783849716,
      "rewards/margins": 0.08926355838775635,
      "rewards/rejected": -0.12151332944631577,
      "step": 850
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.997646085112126e-06,
      "logits/chosen": -2.8823812007904053,
      "logits/rejected": -2.889356851577759,
      "logps/chosen": -285.6360778808594,
      "logps/rejected": -243.662109375,
      "loss": 0.669,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.032262224704027176,
      "rewards/margins": 0.05970774218440056,
      "rewards/rejected": -0.09196996688842773,
      "step": 860
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.997124541942141e-06,
      "logits/chosen": -2.683950185775757,
      "logits/rejected": -2.7463009357452393,
      "logps/chosen": -272.642333984375,
      "logps/rejected": -301.0301513671875,
      "loss": 0.657,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.0004665583255700767,
      "rewards/margins": 0.060696106404066086,
      "rewards/rejected": -0.06116267293691635,
      "step": 870
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.996550886363801e-06,
      "logits/chosen": -2.870196580886841,
      "logits/rejected": -2.90971040725708,
      "logps/chosen": -323.8336181640625,
      "logps/rejected": -310.396728515625,
      "loss": 0.6698,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.007984615862369537,
      "rewards/margins": 0.04810230806469917,
      "rewards/rejected": -0.040117692202329636,
      "step": 880
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.995925130348706e-06,
      "logits/chosen": -2.7594635486602783,
      "logits/rejected": -2.7156622409820557,
      "logps/chosen": -339.4968566894531,
      "logps/rejected": -281.2803649902344,
      "loss": 0.6711,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.042974114418029785,
      "rewards/margins": 0.05631721019744873,
      "rewards/rejected": -0.09929133951663971,
      "step": 890
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.995247286955734e-06,
      "logits/chosen": -2.8020763397216797,
      "logits/rejected": -2.7794690132141113,
      "logps/chosen": -290.42620849609375,
      "logps/rejected": -286.94012451171875,
      "loss": 0.6586,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.045134831219911575,
      "rewards/margins": 0.07758808135986328,
      "rewards/rejected": -0.12272290885448456,
      "step": 900
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.994517370330779e-06,
      "logits/chosen": -2.7767722606658936,
      "logits/rejected": -2.7736542224884033,
      "logps/chosen": -271.8722229003906,
      "logps/rejected": -282.7121276855469,
      "loss": 0.6505,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.031097183004021645,
      "rewards/margins": 0.10152481496334076,
      "rewards/rejected": -0.13262200355529785,
      "step": 910
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.993735395706446e-06,
      "logits/chosen": -2.7642810344696045,
      "logits/rejected": -2.728177070617676,
      "logps/chosen": -405.43310546875,
      "logps/rejected": -429.303955078125,
      "loss": 0.6646,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.11826081573963165,
      "rewards/margins": 0.06041279435157776,
      "rewards/rejected": -0.1786736249923706,
      "step": 920
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.992901379401737e-06,
      "logits/chosen": -2.7699973583221436,
      "logits/rejected": -2.6845641136169434,
      "logps/chosen": -228.12570190429688,
      "logps/rejected": -257.15802001953125,
      "loss": 0.6511,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0507289357483387,
      "rewards/margins": 0.11279155313968658,
      "rewards/rejected": -0.16352050006389618,
      "step": 930
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.992015338821711e-06,
      "logits/chosen": -2.811319351196289,
      "logits/rejected": -2.7655045986175537,
      "logps/chosen": -276.06842041015625,
      "logps/rejected": -219.69436645507812,
      "loss": 0.6588,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.087889164686203,
      "rewards/margins": 0.07118016481399536,
      "rewards/rejected": -0.15906934440135956,
      "step": 940
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.991077292457117e-06,
      "logits/chosen": -2.8166513442993164,
      "logits/rejected": -2.821939706802368,
      "logps/chosen": -197.7022705078125,
      "logps/rejected": -187.2440185546875,
      "loss": 0.6406,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.017550457268953323,
      "rewards/margins": 0.13717380166053772,
      "rewards/rejected": -0.15472427010536194,
      "step": 950
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.990087259884016e-06,
      "logits/chosen": -2.847860813140869,
      "logits/rejected": -2.8105127811431885,
      "logps/chosen": -293.509765625,
      "logps/rejected": -290.2424621582031,
      "loss": 0.665,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.05264557525515556,
      "rewards/margins": 0.09494204819202423,
      "rewards/rejected": -0.1475876271724701,
      "step": 960
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.989045261763362e-06,
      "logits/chosen": -2.8128838539123535,
      "logits/rejected": -2.7957558631896973,
      "logps/chosen": -269.32257080078125,
      "logps/rejected": -269.24151611328125,
      "loss": 0.6493,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.04630335047841072,
      "rewards/margins": 0.08781363815069199,
      "rewards/rejected": -0.1341169774532318,
      "step": 970
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.98795131984058e-06,
      "logits/chosen": -2.638960838317871,
      "logits/rejected": -2.679980754852295,
      "logps/chosen": -304.7935485839844,
      "logps/rejected": -252.098388671875,
      "loss": 0.6546,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.04581091180443764,
      "rewards/margins": 0.11138496547937393,
      "rewards/rejected": -0.15719588100910187,
      "step": 980
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.986805456945107e-06,
      "logits/chosen": -2.7171177864074707,
      "logits/rejected": -2.7990949153900146,
      "logps/chosen": -306.76385498046875,
      "logps/rejected": -291.9147033691406,
      "loss": 0.6411,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.05044771358370781,
      "rewards/margins": 0.18383342027664185,
      "rewards/rejected": -0.23428115248680115,
      "step": 990
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.985607696989919e-06,
      "logits/chosen": -2.778848648071289,
      "logits/rejected": -2.735416889190674,
      "logps/chosen": -302.1616516113281,
      "logps/rejected": -293.0306091308594,
      "loss": 0.668,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.013386363163590431,
      "rewards/margins": 0.08964868634939194,
      "rewards/rejected": -0.10303504765033722,
      "step": 1000
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.984358064971026e-06,
      "logits/chosen": -2.837163209915161,
      "logits/rejected": -2.8279595375061035,
      "logps/chosen": -298.20404052734375,
      "logps/rejected": -283.8133239746094,
      "loss": 0.6451,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.039313506335020065,
      "rewards/margins": 0.12387903034687042,
      "rewards/rejected": -0.08456553518772125,
      "step": 1010
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.983056586966958e-06,
      "logits/chosen": -2.7270495891571045,
      "logits/rejected": -2.7269086837768555,
      "logps/chosen": -296.46673583984375,
      "logps/rejected": -290.07720947265625,
      "loss": 0.6367,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.004742083139717579,
      "rewards/margins": 0.09548323601484299,
      "rewards/rejected": -0.09074117243289948,
      "step": 1020
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.981703290138215e-06,
      "logits/chosen": -2.824183940887451,
      "logits/rejected": -2.818276882171631,
      "logps/chosen": -328.6833190917969,
      "logps/rejected": -306.18365478515625,
      "loss": 0.6431,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.08621495962142944,
      "rewards/margins": 0.07550667226314545,
      "rewards/rejected": -0.1617216318845749,
      "step": 1030
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.980298202726706e-06,
      "logits/chosen": -2.8186137676239014,
      "logits/rejected": -2.7979001998901367,
      "logps/chosen": -311.40106201171875,
      "logps/rejected": -313.767822265625,
      "loss": 0.6632,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06439457088708878,
      "rewards/margins": 0.05475534871220589,
      "rewards/rejected": -0.11914992332458496,
      "step": 1040
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.978841354055148e-06,
      "logits/chosen": -2.7829668521881104,
      "logits/rejected": -2.7322540283203125,
      "logps/chosen": -236.6449432373047,
      "logps/rejected": -241.97909545898438,
      "loss": 0.6235,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.09969043731689453,
      "rewards/margins": 0.14499905705451965,
      "rewards/rejected": -0.24468949437141418,
      "step": 1050
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.977332774526471e-06,
      "logits/chosen": -2.672950029373169,
      "logits/rejected": -2.7831132411956787,
      "logps/chosen": -245.9104461669922,
      "logps/rejected": -272.1865234375,
      "loss": 0.6499,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.062217772006988525,
      "rewards/margins": 0.10472693294286728,
      "rewards/rejected": -0.1669446974992752,
      "step": 1060
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.97577249562317e-06,
      "logits/chosen": -2.8210768699645996,
      "logits/rejected": -2.7688510417938232,
      "logps/chosen": -260.833740234375,
      "logps/rejected": -300.2538146972656,
      "loss": 0.6339,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09711422771215439,
      "rewards/margins": 0.12160750478506088,
      "rewards/rejected": -0.21872171759605408,
      "step": 1070
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.974160549906652e-06,
      "logits/chosen": -2.8435871601104736,
      "logits/rejected": -2.8830039501190186,
      "logps/chosen": -341.4996643066406,
      "logps/rejected": -307.61663818359375,
      "loss": 0.628,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.010590804740786552,
      "rewards/margins": 0.13791541755199432,
      "rewards/rejected": -0.12732459604740143,
      "step": 1080
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.972496971016559e-06,
      "logits/chosen": -2.7579638957977295,
      "logits/rejected": -2.735511064529419,
      "logps/chosen": -264.26220703125,
      "logps/rejected": -244.8957977294922,
      "loss": 0.6322,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.029954945668578148,
      "rewards/margins": 0.12742123007774353,
      "rewards/rejected": -0.09746628999710083,
      "step": 1090
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.9707817936700635e-06,
      "logits/chosen": -2.823887586593628,
      "logits/rejected": -2.794739246368408,
      "logps/chosen": -280.4059753417969,
      "logps/rejected": -250.8085479736328,
      "loss": 0.6697,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1311468780040741,
      "rewards/margins": 0.04369507357478142,
      "rewards/rejected": -0.174841970205307,
      "step": 1100
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.969015053661142e-06,
      "logits/chosen": -2.7738354206085205,
      "logits/rejected": -2.8087518215179443,
      "logps/chosen": -307.9255065917969,
      "logps/rejected": -286.568359375,
      "loss": 0.6431,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.09180223196744919,
      "rewards/margins": 0.14357972145080566,
      "rewards/rejected": -0.23538196086883545,
      "step": 1110
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.967196787859835e-06,
      "logits/chosen": -2.6887173652648926,
      "logits/rejected": -2.7288146018981934,
      "logps/chosen": -350.5284423828125,
      "logps/rejected": -327.24774169921875,
      "loss": 0.665,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0915946513414383,
      "rewards/margins": 0.013863353058695793,
      "rewards/rejected": -0.10545799881219864,
      "step": 1120
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.965327034211469e-06,
      "logits/chosen": -2.7602741718292236,
      "logits/rejected": -2.723498821258545,
      "logps/chosen": -272.59857177734375,
      "logps/rejected": -213.2363739013672,
      "loss": 0.6471,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.015425595454871655,
      "rewards/margins": 0.12277853488922119,
      "rewards/rejected": -0.13820412755012512,
      "step": 1130
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.96340583173587e-06,
      "logits/chosen": -2.871131420135498,
      "logits/rejected": -2.8382210731506348,
      "logps/chosen": -331.5573425292969,
      "logps/rejected": -303.3537292480469,
      "loss": 0.6341,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.02129245176911354,
      "rewards/margins": 0.15463818609714508,
      "rewards/rejected": -0.13334573805332184,
      "step": 1140
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.96143322052655e-06,
      "logits/chosen": -2.7604260444641113,
      "logits/rejected": -2.766071319580078,
      "logps/chosen": -253.5708465576172,
      "logps/rejected": -287.50201416015625,
      "loss": 0.6475,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.012628789059817791,
      "rewards/margins": 0.10162287950515747,
      "rewards/rejected": -0.11425165832042694,
      "step": 1150
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.959409241749864e-06,
      "logits/chosen": -2.6584596633911133,
      "logits/rejected": -2.6555018424987793,
      "logps/chosen": -284.01031494140625,
      "logps/rejected": -294.9400939941406,
      "loss": 0.6496,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.13796189427375793,
      "rewards/margins": 0.13216018676757812,
      "rewards/rejected": -0.27012208104133606,
      "step": 1160
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.957333937644159e-06,
      "logits/chosen": -2.805464029312134,
      "logits/rejected": -2.775489091873169,
      "logps/chosen": -231.558837890625,
      "logps/rejected": -246.1699981689453,
      "loss": 0.626,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1663055121898651,
      "rewards/margins": 0.1382933408021927,
      "rewards/rejected": -0.304598867893219,
      "step": 1170
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.955207351518885e-06,
      "logits/chosen": -2.8096632957458496,
      "logits/rejected": -2.8571386337280273,
      "logps/chosen": -310.173583984375,
      "logps/rejected": -275.7320861816406,
      "loss": 0.6637,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.04798074811697006,
      "rewards/margins": 0.12220487743616104,
      "rewards/rejected": -0.1701856255531311,
      "step": 1180
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.953029527753699e-06,
      "logits/chosen": -2.6950483322143555,
      "logits/rejected": -2.6458487510681152,
      "logps/chosen": -303.374755859375,
      "logps/rejected": -294.6081237792969,
      "loss": 0.6576,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.0030715905595570803,
      "rewards/margins": 0.1294359564781189,
      "rewards/rejected": -0.1263643503189087,
      "step": 1190
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.95080051179753e-06,
      "logits/chosen": -2.8290605545043945,
      "logits/rejected": -2.814423084259033,
      "logps/chosen": -240.0433807373047,
      "logps/rejected": -255.37765502929688,
      "loss": 0.6607,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.001875871792435646,
      "rewards/margins": 0.14930638670921326,
      "rewards/rejected": -0.14743052423000336,
      "step": 1200
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.948520350167637e-06,
      "logits/chosen": -2.730070114135742,
      "logits/rejected": -2.6461236476898193,
      "logps/chosen": -343.88818359375,
      "logps/rejected": -319.3534240722656,
      "loss": 0.631,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.03022381290793419,
      "rewards/margins": 0.14663995802402496,
      "rewards/rejected": -0.11641611903905869,
      "step": 1210
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.946189090448639e-06,
      "logits/chosen": -2.880946397781372,
      "logits/rejected": -2.7842841148376465,
      "logps/chosen": -335.5487060546875,
      "logps/rejected": -312.669189453125,
      "loss": 0.6076,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.03348718211054802,
      "rewards/margins": 0.18006981909275055,
      "rewards/rejected": -0.14658261835575104,
      "step": 1220
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.943806781291515e-06,
      "logits/chosen": -2.687602996826172,
      "logits/rejected": -2.674149751663208,
      "logps/chosen": -300.13128662109375,
      "logps/rejected": -349.22174072265625,
      "loss": 0.6257,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06928616017103195,
      "rewards/margins": 0.12933610379695892,
      "rewards/rejected": -0.19862230122089386,
      "step": 1230
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.941373472412595e-06,
      "logits/chosen": -2.758183479309082,
      "logits/rejected": -2.7490592002868652,
      "logps/chosen": -320.9246520996094,
      "logps/rejected": -261.5167541503906,
      "loss": 0.6353,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15161526203155518,
      "rewards/margins": 0.18470586836338043,
      "rewards/rejected": -0.336321085691452,
      "step": 1240
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.938889214592521e-06,
      "logits/chosen": -2.6537938117980957,
      "logits/rejected": -2.7111458778381348,
      "logps/chosen": -225.9049072265625,
      "logps/rejected": -289.18975830078125,
      "loss": 0.6225,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.21186769008636475,
      "rewards/margins": 0.1365642547607422,
      "rewards/rejected": -0.34843194484710693,
      "step": 1250
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.936354059675186e-06,
      "logits/chosen": -2.8012726306915283,
      "logits/rejected": -2.750591516494751,
      "logps/chosen": -284.7136535644531,
      "logps/rejected": -292.6429138183594,
      "loss": 0.5986,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.13322897255420685,
      "rewards/margins": 0.18906542658805847,
      "rewards/rejected": -0.3222944140434265,
      "step": 1260
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.933768060566654e-06,
      "logits/chosen": -2.752645969390869,
      "logits/rejected": -2.6327085494995117,
      "logps/chosen": -294.1012268066406,
      "logps/rejected": -268.6238708496094,
      "loss": 0.6216,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11605393886566162,
      "rewards/margins": 0.19025316834449768,
      "rewards/rejected": -0.3063071370124817,
      "step": 1270
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.931131271234052e-06,
      "logits/chosen": -2.746007204055786,
      "logits/rejected": -2.756093978881836,
      "logps/chosen": -302.9208679199219,
      "logps/rejected": -326.42120361328125,
      "loss": 0.6451,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07623710483312607,
      "rewards/margins": 0.17326778173446655,
      "rewards/rejected": -0.2495049238204956,
      "step": 1280
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.928443746704448e-06,
      "logits/chosen": -2.76210355758667,
      "logits/rejected": -2.666907548904419,
      "logps/chosen": -274.781494140625,
      "logps/rejected": -263.9569396972656,
      "loss": 0.6144,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.009095259010791779,
      "rewards/margins": 0.23592600226402283,
      "rewards/rejected": -0.2450212687253952,
      "step": 1290
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.925705543063703e-06,
      "logits/chosen": -2.745572090148926,
      "logits/rejected": -2.7342967987060547,
      "logps/chosen": -282.6916809082031,
      "logps/rejected": -343.0442199707031,
      "loss": 0.6367,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0001681298017501831,
      "rewards/margins": 0.11077561229467392,
      "rewards/rejected": -0.1109437495470047,
      "step": 1300
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.922916717455297e-06,
      "logits/chosen": -2.647264003753662,
      "logits/rejected": -2.682744264602661,
      "logps/chosen": -249.7043914794922,
      "logps/rejected": -263.6383056640625,
      "loss": 0.6649,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.024844978004693985,
      "rewards/margins": 0.07102416455745697,
      "rewards/rejected": -0.04617919400334358,
      "step": 1310
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.920077328079136e-06,
      "logits/chosen": -2.755326986312866,
      "logits/rejected": -2.681166410446167,
      "logps/chosen": -297.91131591796875,
      "logps/rejected": -278.9104919433594,
      "loss": 0.6113,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.014719530940055847,
      "rewards/margins": 0.24292895197868347,
      "rewards/rejected": -0.2282094657421112,
      "step": 1320
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.9171874341903445e-06,
      "logits/chosen": -2.677175521850586,
      "logits/rejected": -2.686843156814575,
      "logps/chosen": -269.16278076171875,
      "logps/rejected": -256.2588806152344,
      "loss": 0.6117,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.18205681443214417,
      "rewards/margins": 0.17718163132667542,
      "rewards/rejected": -0.3592384457588196,
      "step": 1330
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.914247096098019e-06,
      "logits/chosen": -2.647059679031372,
      "logits/rejected": -2.6469664573669434,
      "logps/chosen": -268.0980224609375,
      "logps/rejected": -263.8585510253906,
      "loss": 0.6107,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15819644927978516,
      "rewards/margins": 0.19166433811187744,
      "rewards/rejected": -0.3498607575893402,
      "step": 1340
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.911256375163977e-06,
      "logits/chosen": -2.7415127754211426,
      "logits/rejected": -2.729755163192749,
      "logps/chosen": -325.60430908203125,
      "logps/rejected": -279.28582763671875,
      "loss": 0.6013,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.10827972739934921,
      "rewards/margins": 0.21758142113685608,
      "rewards/rejected": -0.3258611261844635,
      "step": 1350
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.908215333801474e-06,
      "logits/chosen": -2.917020797729492,
      "logits/rejected": -2.875368118286133,
      "logps/chosen": -341.59417724609375,
      "logps/rejected": -284.4775695800781,
      "loss": 0.6644,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.2299484759569168,
      "rewards/margins": 0.12131299078464508,
      "rewards/rejected": -0.3512614667415619,
      "step": 1360
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.9051240354739004e-06,
      "logits/chosen": -2.7348055839538574,
      "logits/rejected": -2.7037601470947266,
      "logps/chosen": -320.5626525878906,
      "logps/rejected": -323.69329833984375,
      "loss": 0.6265,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.21234694123268127,
      "rewards/margins": 0.1680133193731308,
      "rewards/rejected": -0.3803602457046509,
      "step": 1370
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.901982544693457e-06,
      "logits/chosen": -2.636758327484131,
      "logits/rejected": -2.5937962532043457,
      "logps/chosen": -183.79354858398438,
      "logps/rejected": -258.00732421875,
      "loss": 0.6105,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.16744934022426605,
      "rewards/margins": 0.1875566989183426,
      "rewards/rejected": -0.35500603914260864,
      "step": 1380
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.898790927019809e-06,
      "logits/chosen": -2.757171869277954,
      "logits/rejected": -2.754363775253296,
      "logps/chosen": -278.60546875,
      "logps/rejected": -283.8389892578125,
      "loss": 0.6109,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07067025452852249,
      "rewards/margins": 0.10649889707565308,
      "rewards/rejected": -0.17716915905475616,
      "step": 1390
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.895549249058718e-06,
      "logits/chosen": -2.720224618911743,
      "logits/rejected": -2.675530433654785,
      "logps/chosen": -321.7203063964844,
      "logps/rejected": -277.67974853515625,
      "loss": 0.6092,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.016543136909604073,
      "rewards/margins": 0.2174593210220337,
      "rewards/rejected": -0.2340024709701538,
      "step": 1400
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.892257578460656e-06,
      "logits/chosen": -2.677842617034912,
      "logits/rejected": -2.664290189743042,
      "logps/chosen": -218.2035675048828,
      "logps/rejected": -224.66384887695312,
      "loss": 0.6588,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.15112267434597015,
      "rewards/margins": 0.13598957657814026,
      "rewards/rejected": -0.2871122658252716,
      "step": 1410
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.888915983919383e-06,
      "logits/chosen": -2.6758079528808594,
      "logits/rejected": -2.7004618644714355,
      "logps/chosen": -228.3094940185547,
      "logps/rejected": -265.48486328125,
      "loss": 0.5952,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08630862087011337,
      "rewards/margins": 0.2088243067264557,
      "rewards/rejected": -0.29513293504714966,
      "step": 1420
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.885524535170525e-06,
      "logits/chosen": -2.6005783081054688,
      "logits/rejected": -2.4832072257995605,
      "logps/chosen": -231.92025756835938,
      "logps/rejected": -235.0711212158203,
      "loss": 0.6017,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20524215698242188,
      "rewards/margins": 0.1942833662033081,
      "rewards/rejected": -0.39952555298805237,
      "step": 1430
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.882083302990113e-06,
      "logits/chosen": -2.6818032264709473,
      "logits/rejected": -2.6769537925720215,
      "logps/chosen": -285.8664245605469,
      "logps/rejected": -284.16156005859375,
      "loss": 0.6581,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.28984886407852173,
      "rewards/margins": 0.030770767480134964,
      "rewards/rejected": -0.3206196427345276,
      "step": 1440
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.878592359193104e-06,
      "logits/chosen": -2.692941665649414,
      "logits/rejected": -2.6352081298828125,
      "logps/chosen": -276.248046875,
      "logps/rejected": -248.68612670898438,
      "loss": 0.5998,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.29445046186447144,
      "rewards/margins": 0.2336672991514206,
      "rewards/rejected": -0.5281177759170532,
      "step": 1450
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.875051776631888e-06,
      "logits/chosen": -2.64967679977417,
      "logits/rejected": -2.7691729068756104,
      "logps/chosen": -294.6110534667969,
      "logps/rejected": -364.65570068359375,
      "loss": 0.6243,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2757050096988678,
      "rewards/margins": 0.1370132863521576,
      "rewards/rejected": -0.4127182960510254,
      "step": 1460
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.871461629194764e-06,
      "logits/chosen": -2.757596492767334,
      "logits/rejected": -2.6786651611328125,
      "logps/chosen": -336.01593017578125,
      "logps/rejected": -315.3603515625,
      "loss": 0.626,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08129008114337921,
      "rewards/margins": 0.2590528726577759,
      "rewards/rejected": -0.3403429388999939,
      "step": 1470
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.8678219918043984e-06,
      "logits/chosen": -2.7759063243865967,
      "logits/rejected": -2.7906689643859863,
      "logps/chosen": -283.36651611328125,
      "logps/rejected": -330.7609558105469,
      "loss": 0.6293,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1509023904800415,
      "rewards/margins": 0.17229530215263367,
      "rewards/rejected": -0.3231976628303528,
      "step": 1480
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.864132940416262e-06,
      "logits/chosen": -2.7591805458068848,
      "logits/rejected": -2.7330336570739746,
      "logps/chosen": -234.86569213867188,
      "logps/rejected": -237.08261108398438,
      "loss": 0.6472,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.22255483269691467,
      "rewards/margins": 0.09442047029733658,
      "rewards/rejected": -0.31697529554367065,
      "step": 1490
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.860394552017044e-06,
      "logits/chosen": -2.828935146331787,
      "logits/rejected": -2.750209331512451,
      "logps/chosen": -312.0937194824219,
      "logps/rejected": -307.0955505371094,
      "loss": 0.5986,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.03175521641969681,
      "rewards/margins": 0.20605985820293427,
      "rewards/rejected": -0.23781506717205048,
      "step": 1500
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.856606904623047e-06,
      "logits/chosen": -2.8377044200897217,
      "logits/rejected": -2.7731568813323975,
      "logps/chosen": -297.10894775390625,
      "logps/rejected": -285.8016662597656,
      "loss": 0.5955,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07155664265155792,
      "rewards/margins": 0.2173517495393753,
      "rewards/rejected": -0.2889083921909332,
      "step": 1510
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.852770077278557e-06,
      "logits/chosen": -2.7048516273498535,
      "logits/rejected": -2.6029093265533447,
      "logps/chosen": -305.9083557128906,
      "logps/rejected": -282.14971923828125,
      "loss": 0.6173,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.17138591408729553,
      "rewards/margins": 0.20368929207324982,
      "rewards/rejected": -0.37507519125938416,
      "step": 1520
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.848884150054196e-06,
      "logits/chosen": -2.572361946105957,
      "logits/rejected": -2.5174009799957275,
      "logps/chosen": -314.22760009765625,
      "logps/rejected": -353.42315673828125,
      "loss": 0.655,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.25344517827033997,
      "rewards/margins": 0.11698782444000244,
      "rewards/rejected": -0.3704330027103424,
      "step": 1530
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.8449492040452495e-06,
      "logits/chosen": -2.7566611766815186,
      "logits/rejected": -2.650944232940674,
      "logps/chosen": -315.4299011230469,
      "logps/rejected": -298.1441345214844,
      "loss": 0.6212,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.25965338945388794,
      "rewards/margins": 0.16350455582141876,
      "rewards/rejected": -0.4231579303741455,
      "step": 1540
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.840965321369973e-06,
      "logits/chosen": -2.717017650604248,
      "logits/rejected": -2.8227248191833496,
      "logps/chosen": -335.89678955078125,
      "logps/rejected": -369.3492126464844,
      "loss": 0.5908,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18783393502235413,
      "rewards/margins": 0.28056249022483826,
      "rewards/rejected": -0.4683963656425476,
      "step": 1550
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.8369325851678795e-06,
      "logits/chosen": -2.669900417327881,
      "logits/rejected": -2.695427417755127,
      "logps/chosen": -303.61322021484375,
      "logps/rejected": -267.2237243652344,
      "loss": 0.584,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.08962138742208481,
      "rewards/margins": 0.3239901661872864,
      "rewards/rejected": -0.41361159086227417,
      "step": 1560
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.832851079598007e-06,
      "logits/chosen": -2.6860480308532715,
      "logits/rejected": -2.7314352989196777,
      "logps/chosen": -292.6394348144531,
      "logps/rejected": -296.482666015625,
      "loss": 0.6553,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.23995356261730194,
      "rewards/margins": 0.13955575227737427,
      "rewards/rejected": -0.3795093297958374,
      "step": 1570
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.828720889837158e-06,
      "logits/chosen": -2.8179385662078857,
      "logits/rejected": -2.793445110321045,
      "logps/chosen": -338.71771240234375,
      "logps/rejected": -317.84600830078125,
      "loss": 0.6828,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.1939702033996582,
      "rewards/margins": 0.043380144983530045,
      "rewards/rejected": -0.23735034465789795,
      "step": 1580
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.824542102078125e-06,
      "logits/chosen": -2.684544801712036,
      "logits/rejected": -2.69701886177063,
      "logps/chosen": -304.6727600097656,
      "logps/rejected": -278.4373474121094,
      "loss": 0.5815,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.011012783274054527,
      "rewards/margins": 0.2999088168144226,
      "rewards/rejected": -0.3109216094017029,
      "step": 1590
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.820314803527888e-06,
      "logits/chosen": -2.664644718170166,
      "logits/rejected": -2.612112522125244,
      "logps/chosen": -264.01153564453125,
      "logps/rejected": -292.8648681640625,
      "loss": 0.6086,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.22297194600105286,
      "rewards/margins": 0.20747871696949005,
      "rewards/rejected": -0.4304506182670593,
      "step": 1600
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.816039082405799e-06,
      "logits/chosen": -2.5841476917266846,
      "logits/rejected": -2.6593730449676514,
      "logps/chosen": -277.6162414550781,
      "logps/rejected": -287.6755676269531,
      "loss": 0.6558,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.205702543258667,
      "rewards/margins": 0.06671354919672012,
      "rewards/rejected": -0.27241605520248413,
      "step": 1610
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.81171502794174e-06,
      "logits/chosen": -2.6567516326904297,
      "logits/rejected": -2.631348133087158,
      "logps/chosen": -271.10296630859375,
      "logps/rejected": -277.9122009277344,
      "loss": 0.6361,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1482880860567093,
      "rewards/margins": 0.19127169251441956,
      "rewards/rejected": -0.33955979347229004,
      "step": 1620
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.8073427303742584e-06,
      "logits/chosen": -2.6917014122009277,
      "logits/rejected": -2.6560750007629395,
      "logps/chosen": -287.2611389160156,
      "logps/rejected": -231.60037231445312,
      "loss": 0.5887,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.09727681428194046,
      "rewards/margins": 0.2694247364997864,
      "rewards/rejected": -0.36670148372650146,
      "step": 1630
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.802922280948685e-06,
      "logits/chosen": -2.655128240585327,
      "logits/rejected": -2.6118204593658447,
      "logps/chosen": -275.65118408203125,
      "logps/rejected": -313.3224792480469,
      "loss": 0.622,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09502645581960678,
      "rewards/margins": 0.17091654241085052,
      "rewards/rejected": -0.2659429907798767,
      "step": 1640
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.798453771915231e-06,
      "logits/chosen": -2.7059059143066406,
      "logits/rejected": -2.646460771560669,
      "logps/chosen": -287.76092529296875,
      "logps/rejected": -277.286865234375,
      "loss": 0.5896,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.05740933492779732,
      "rewards/margins": 0.2564863860607147,
      "rewards/rejected": -0.31389567255973816,
      "step": 1650
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.793937296527062e-06,
      "logits/chosen": -2.775752544403076,
      "logits/rejected": -2.674415111541748,
      "logps/chosen": -327.850830078125,
      "logps/rejected": -313.1639099121094,
      "loss": 0.6179,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.10228496789932251,
      "rewards/margins": 0.2658602297306061,
      "rewards/rejected": -0.3681451976299286,
      "step": 1660
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.78937294903835e-06,
      "logits/chosen": -2.779148578643799,
      "logits/rejected": -2.675994396209717,
      "logps/chosen": -351.1578369140625,
      "logps/rejected": -290.1517639160156,
      "loss": 0.6293,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.09466046094894409,
      "rewards/margins": 0.10398276150226593,
      "rewards/rejected": -0.19864322245121002,
      "step": 1670
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.78476082470231e-06,
      "logits/chosen": -2.767909526824951,
      "logits/rejected": -2.756925106048584,
      "logps/chosen": -302.81573486328125,
      "logps/rejected": -312.35565185546875,
      "loss": 0.6076,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.048796527087688446,
      "rewards/margins": 0.21524441242218018,
      "rewards/rejected": -0.2640409767627716,
      "step": 1680
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.780101019769212e-06,
      "logits/chosen": -2.6890337467193604,
      "logits/rejected": -2.6910290718078613,
      "logps/chosen": -293.86968994140625,
      "logps/rejected": -321.2923278808594,
      "loss": 0.6275,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06831858307123184,
      "rewards/margins": 0.1262729912996292,
      "rewards/rejected": -0.19459158182144165,
      "step": 1690
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.775393631484368e-06,
      "logits/chosen": -2.777174472808838,
      "logits/rejected": -2.7287497520446777,
      "logps/chosen": -377.688720703125,
      "logps/rejected": -403.62530517578125,
      "loss": 0.5685,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06738211214542389,
      "rewards/margins": 0.33521461486816406,
      "rewards/rejected": -0.40259671211242676,
      "step": 1700
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.770638758086105e-06,
      "logits/chosen": -2.7955563068389893,
      "logits/rejected": -2.7565627098083496,
      "logps/chosen": -322.578857421875,
      "logps/rejected": -353.1636962890625,
      "loss": 0.6419,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.16695697605609894,
      "rewards/margins": 0.25171276926994324,
      "rewards/rejected": -0.41866976022720337,
      "step": 1710
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.7658364988037184e-06,
      "logits/chosen": -2.547750949859619,
      "logits/rejected": -2.5859413146972656,
      "logps/chosen": -260.44635009765625,
      "logps/rejected": -299.1117858886719,
      "loss": 0.5923,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.2033233344554901,
      "rewards/margins": 0.2312351018190384,
      "rewards/rejected": -0.4345583915710449,
      "step": 1720
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.760986953855395e-06,
      "logits/chosen": -2.6293139457702637,
      "logits/rejected": -2.5827693939208984,
      "logps/chosen": -345.98992919921875,
      "logps/rejected": -302.2580261230469,
      "loss": 0.6219,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1974361389875412,
      "rewards/margins": 0.1821739375591278,
      "rewards/rejected": -0.3796100616455078,
      "step": 1730
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.756090224446127e-06,
      "logits/chosen": -2.6410300731658936,
      "logits/rejected": -2.5768771171569824,
      "logps/chosen": -334.6339111328125,
      "logps/rejected": -314.4638671875,
      "loss": 0.6083,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17295120656490326,
      "rewards/margins": 0.2973530888557434,
      "rewards/rejected": -0.47030431032180786,
      "step": 1740
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.7511464127655945e-06,
      "logits/chosen": -2.531867742538452,
      "logits/rejected": -2.5611560344696045,
      "logps/chosen": -261.2344970703125,
      "logps/rejected": -305.4712219238281,
      "loss": 0.6304,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.3530547320842743,
      "rewards/margins": 0.15563586354255676,
      "rewards/rejected": -0.508690595626831,
      "step": 1750
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.74615562198604e-06,
      "logits/chosen": -2.623383045196533,
      "logits/rejected": -2.608071804046631,
      "logps/chosen": -271.5995788574219,
      "logps/rejected": -261.22711181640625,
      "loss": 0.6452,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.26331251859664917,
      "rewards/margins": 0.21371789276599884,
      "rewards/rejected": -0.4770304262638092,
      "step": 1760
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.741117956260107e-06,
      "logits/chosen": -2.74521541595459,
      "logits/rejected": -2.7192955017089844,
      "logps/chosen": -363.3270263671875,
      "logps/rejected": -349.2979431152344,
      "loss": 0.6285,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.2583506107330322,
      "rewards/margins": 0.19173289835453033,
      "rewards/rejected": -0.45008349418640137,
      "step": 1770
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.736033520718672e-06,
      "logits/chosen": -2.708975315093994,
      "logits/rejected": -2.62796950340271,
      "logps/chosen": -256.0467224121094,
      "logps/rejected": -234.60635375976562,
      "loss": 0.613,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.29684996604919434,
      "rewards/margins": 0.2133146971464157,
      "rewards/rejected": -0.5101646780967712,
      "step": 1780
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.730902421468652e-06,
      "logits/chosen": -2.72163724899292,
      "logits/rejected": -2.676257610321045,
      "logps/chosen": -292.6100769042969,
      "logps/rejected": -270.1462707519531,
      "loss": 0.5963,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.11978425830602646,
      "rewards/margins": 0.30899566411972046,
      "rewards/rejected": -0.4287799000740051,
      "step": 1790
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.7257247655907854e-06,
      "logits/chosen": -2.6853718757629395,
      "logits/rejected": -2.5619893074035645,
      "logps/chosen": -307.2440185546875,
      "logps/rejected": -281.82159423828125,
      "loss": 0.6117,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.14979851245880127,
      "rewards/margins": 0.23921413719654083,
      "rewards/rejected": -0.3890126645565033,
      "step": 1800
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.720500661137397e-06,
      "logits/chosen": -2.618361711502075,
      "logits/rejected": -2.6298348903656006,
      "logps/chosen": -216.2972412109375,
      "logps/rejected": -278.35882568359375,
      "loss": 0.5912,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.19139596819877625,
      "rewards/margins": 0.2812741696834564,
      "rewards/rejected": -0.47267013788223267,
      "step": 1810
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.71523021713015e-06,
      "logits/chosen": -2.756312608718872,
      "logits/rejected": -2.6926093101501465,
      "logps/chosen": -319.30950927734375,
      "logps/rejected": -278.87603759765625,
      "loss": 0.597,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06740576773881912,
      "rewards/margins": 0.1675378978252411,
      "rewards/rejected": -0.2349436730146408,
      "step": 1820
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.709913543557761e-06,
      "logits/chosen": -2.689087390899658,
      "logits/rejected": -2.7317633628845215,
      "logps/chosen": -337.2334899902344,
      "logps/rejected": -316.35626220703125,
      "loss": 0.6185,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1094675213098526,
      "rewards/margins": 0.18793943524360657,
      "rewards/rejected": -0.297406941652298,
      "step": 1830
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.704550751373715e-06,
      "logits/chosen": -2.6714820861816406,
      "logits/rejected": -2.6737353801727295,
      "logps/chosen": -289.0609436035156,
      "logps/rejected": -312.31292724609375,
      "loss": 0.6554,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.12423040717840195,
      "rewards/margins": 0.023334268480539322,
      "rewards/rejected": -0.14756467938423157,
      "step": 1840
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.699141952493941e-06,
      "logits/chosen": -2.6949803829193115,
      "logits/rejected": -2.6727728843688965,
      "logps/chosen": -294.4242248535156,
      "logps/rejected": -266.7899169921875,
      "loss": 0.5744,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06736332923173904,
      "rewards/margins": 0.38571593165397644,
      "rewards/rejected": -0.4530792832374573,
      "step": 1850
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.6936872597944814e-06,
      "logits/chosen": -2.6007885932922363,
      "logits/rejected": -2.542579174041748,
      "logps/chosen": -281.5158996582031,
      "logps/rejected": -312.28460693359375,
      "loss": 0.6074,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1378696709871292,
      "rewards/margins": 0.3318738341331482,
      "rewards/rejected": -0.469743549823761,
      "step": 1860
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.688186787109136e-06,
      "logits/chosen": -2.7046103477478027,
      "logits/rejected": -2.680391311645508,
      "logps/chosen": -259.2096252441406,
      "logps/rejected": -260.27227783203125,
      "loss": 0.6248,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1114644855260849,
      "rewards/margins": 0.25269684195518494,
      "rewards/rejected": -0.36416134238243103,
      "step": 1870
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.682640649227085e-06,
      "logits/chosen": -2.5605578422546387,
      "logits/rejected": -2.5646719932556152,
      "logps/chosen": -298.17041015625,
      "logps/rejected": -292.0390319824219,
      "loss": 0.6108,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.11922509968280792,
      "rewards/margins": 0.2266741245985031,
      "rewards/rejected": -0.34589922428131104,
      "step": 1880
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.677048961890492e-06,
      "logits/chosen": -2.531548023223877,
      "logits/rejected": -2.5735363960266113,
      "logps/chosen": -294.02911376953125,
      "logps/rejected": -286.54901123046875,
      "loss": 0.6073,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.08507753908634186,
      "rewards/margins": 0.2579110264778137,
      "rewards/rejected": -0.34298861026763916,
      "step": 1890
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.671411841792096e-06,
      "logits/chosen": -2.741948366165161,
      "logits/rejected": -2.741699695587158,
      "logps/chosen": -361.36614990234375,
      "logps/rejected": -290.7783203125,
      "loss": 0.6137,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.11754343658685684,
      "rewards/margins": 0.2452048510313034,
      "rewards/rejected": -0.36274832487106323,
      "step": 1900
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.665729406572764e-06,
      "logits/chosen": -2.6238443851470947,
      "logits/rejected": -2.5798392295837402,
      "logps/chosen": -221.949951171875,
      "logps/rejected": -245.45327758789062,
      "loss": 0.6165,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06824745237827301,
      "rewards/margins": 0.2026282548904419,
      "rewards/rejected": -0.2708757519721985,
      "step": 1910
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.660001774819048e-06,
      "logits/chosen": -2.5808560848236084,
      "logits/rejected": -2.5920627117156982,
      "logps/chosen": -192.7483367919922,
      "logps/rejected": -254.2974090576172,
      "loss": 0.6229,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.11057275533676147,
      "rewards/margins": 0.13108907639980316,
      "rewards/rejected": -0.24166183173656464,
      "step": 1920
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.654229066060702e-06,
      "logits/chosen": -2.71026349067688,
      "logits/rejected": -2.7135469913482666,
      "logps/chosen": -290.25140380859375,
      "logps/rejected": -421.7354431152344,
      "loss": 0.6206,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.18719585239887238,
      "rewards/margins": 0.24483227729797363,
      "rewards/rejected": -0.4320281147956848,
      "step": 1930
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.648411400768193e-06,
      "logits/chosen": -2.6288414001464844,
      "logits/rejected": -2.67307710647583,
      "logps/chosen": -275.16961669921875,
      "logps/rejected": -317.13226318359375,
      "loss": 0.6248,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.09002985060214996,
      "rewards/margins": 0.2158748209476471,
      "rewards/rejected": -0.30590468645095825,
      "step": 1940
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.642548900350182e-06,
      "logits/chosen": -2.5653369426727295,
      "logits/rejected": -2.5764999389648438,
      "logps/chosen": -307.288330078125,
      "logps/rejected": -318.7103271484375,
      "loss": 0.5834,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.07187150418758392,
      "rewards/margins": 0.3741157650947571,
      "rewards/rejected": -0.4459873139858246,
      "step": 1950
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.636641687150994e-06,
      "logits/chosen": -2.672543525695801,
      "logits/rejected": -2.6475882530212402,
      "logps/chosen": -273.38629150390625,
      "logps/rejected": -268.9444885253906,
      "loss": 0.6402,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11344961076974869,
      "rewards/margins": 0.15282130241394043,
      "rewards/rejected": -0.2662709057331085,
      "step": 1960
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.6306898844480615e-06,
      "logits/chosen": -2.6119863986968994,
      "logits/rejected": -2.5836076736450195,
      "logps/chosen": -250.29446411132812,
      "logps/rejected": -299.84393310546875,
      "loss": 0.6036,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.09432916343212128,
      "rewards/margins": 0.20420034229755402,
      "rewards/rejected": -0.2985295057296753,
      "step": 1970
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.624693616449358e-06,
      "logits/chosen": -2.6079952716827393,
      "logits/rejected": -2.5498125553131104,
      "logps/chosen": -316.224365234375,
      "logps/rejected": -280.0063171386719,
      "loss": 0.6177,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.09396733343601227,
      "rewards/margins": 0.21322092413902283,
      "rewards/rejected": -0.3071882426738739,
      "step": 1980
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.6186530082908e-06,
      "logits/chosen": -2.5833072662353516,
      "logits/rejected": -2.627884864807129,
      "logps/chosen": -283.31219482421875,
      "logps/rejected": -310.50531005859375,
      "loss": 0.6102,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07193948328495026,
      "rewards/margins": 0.12251697480678558,
      "rewards/rejected": -0.19445645809173584,
      "step": 1990
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.612568186033633e-06,
      "logits/chosen": -2.4248900413513184,
      "logits/rejected": -2.510383129119873,
      "logps/chosen": -252.7774658203125,
      "logps/rejected": -208.8021240234375,
      "loss": 0.6096,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.007582767400890589,
      "rewards/margins": 0.2787870764732361,
      "rewards/rejected": -0.28636986017227173,
      "step": 2000
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.6064392766618125e-06,
      "logits/chosen": -2.569242238998413,
      "logits/rejected": -2.593503952026367,
      "logps/chosen": -278.3699645996094,
      "logps/rejected": -289.50384521484375,
      "loss": 0.6226,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.2043885886669159,
      "rewards/margins": 0.2912368178367615,
      "rewards/rejected": -0.49562540650367737,
      "step": 2010
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.60026640807934e-06,
      "logits/chosen": -2.65549373626709,
      "logits/rejected": -2.6767234802246094,
      "logps/chosen": -328.12628173828125,
      "logps/rejected": -380.0748596191406,
      "loss": 0.5954,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.09818612784147263,
      "rewards/margins": 0.28552350401878357,
      "rewards/rejected": -0.3837096393108368,
      "step": 2020
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.594049709107604e-06,
      "logits/chosen": -2.631639003753662,
      "logits/rejected": -2.656987428665161,
      "logps/chosen": -294.75909423828125,
      "logps/rejected": -306.7351989746094,
      "loss": 0.6244,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.2435932159423828,
      "rewards/margins": 0.16294609010219574,
      "rewards/rejected": -0.40653929114341736,
      "step": 2030
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.587789309482687e-06,
      "logits/chosen": -2.6387524604797363,
      "logits/rejected": -2.6367218494415283,
      "logps/chosen": -262.7574462890625,
      "logps/rejected": -306.42279052734375,
      "loss": 0.6419,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.4058169722557068,
      "rewards/margins": 0.1947314441204071,
      "rewards/rejected": -0.6005484461784363,
      "step": 2040
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.581485339852659e-06,
      "logits/chosen": -2.6451351642608643,
      "logits/rejected": -2.525254726409912,
      "logps/chosen": -294.7330627441406,
      "logps/rejected": -257.17169189453125,
      "loss": 0.63,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.4744682312011719,
      "rewards/margins": 0.11925798654556274,
      "rewards/rejected": -0.5937261581420898,
      "step": 2050
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.5751379317748514e-06,
      "logits/chosen": -2.61735463142395,
      "logits/rejected": -2.6056129932403564,
      "logps/chosen": -380.99456787109375,
      "logps/rejected": -324.75494384765625,
      "loss": 0.617,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3881939947605133,
      "rewards/margins": 0.17417709529399872,
      "rewards/rejected": -0.5623710751533508,
      "step": 2060
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.56874721771311e-06,
      "logits/chosen": -2.672650098800659,
      "logits/rejected": -2.6057205200195312,
      "logps/chosen": -382.2373962402344,
      "logps/rejected": -356.7444763183594,
      "loss": 0.5966,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.29305747151374817,
      "rewards/margins": 0.2830229103565216,
      "rewards/rejected": -0.5760804414749146,
      "step": 2070
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.562313331035032e-06,
      "logits/chosen": -2.511033058166504,
      "logits/rejected": -2.5323522090911865,
      "logps/chosen": -255.25381469726562,
      "logps/rejected": -270.14080810546875,
      "loss": 0.6579,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.31634312868118286,
      "rewards/margins": 0.1493835151195526,
      "rewards/rejected": -0.46572667360305786,
      "step": 2080
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.555836406009183e-06,
      "logits/chosen": -2.562288761138916,
      "logits/rejected": -2.595261573791504,
      "logps/chosen": -307.12518310546875,
      "logps/rejected": -275.62664794921875,
      "loss": 0.5593,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.10846608877182007,
      "rewards/margins": 0.2859782874584198,
      "rewards/rejected": -0.39444440603256226,
      "step": 2090
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.5493165778022945e-06,
      "logits/chosen": -2.5555379390716553,
      "logits/rejected": -2.5553812980651855,
      "logps/chosen": -262.2703857421875,
      "logps/rejected": -292.42962646484375,
      "loss": 0.6283,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.05403865501284599,
      "rewards/margins": 0.2357197254896164,
      "rewards/rejected": -0.2897583544254303,
      "step": 2100
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.542753982476443e-06,
      "logits/chosen": -2.5372321605682373,
      "logits/rejected": -2.5293571949005127,
      "logps/chosen": -190.13748168945312,
      "logps/rejected": -252.0640411376953,
      "loss": 0.5592,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.03468228876590729,
      "rewards/margins": 0.34676939249038696,
      "rewards/rejected": -0.31208711862564087,
      "step": 2110
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.53614875698621e-06,
      "logits/chosen": -2.6718530654907227,
      "logits/rejected": -2.624863386154175,
      "logps/chosen": -252.5389862060547,
      "logps/rejected": -296.339111328125,
      "loss": 0.6101,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.028590941801667213,
      "rewards/margins": 0.25347763299942017,
      "rewards/rejected": -0.28206855058670044,
      "step": 2120
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.529501039175824e-06,
      "logits/chosen": -2.7555930614471436,
      "logits/rejected": -2.6663403511047363,
      "logps/chosen": -334.1688537597656,
      "logps/rejected": -282.73583984375,
      "loss": 0.6187,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.043031539767980576,
      "rewards/margins": 0.25927868485450745,
      "rewards/rejected": -0.3023102581501007,
      "step": 2130
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.522810967776287e-06,
      "logits/chosen": -2.5664029121398926,
      "logits/rejected": -2.607612133026123,
      "logps/chosen": -274.611328125,
      "logps/rejected": -273.4698791503906,
      "loss": 0.6162,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.16074858605861664,
      "rewards/margins": 0.19277243316173553,
      "rewards/rejected": -0.35352104902267456,
      "step": 2140
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.516078682402473e-06,
      "logits/chosen": -2.594787359237671,
      "logits/rejected": -2.6934938430786133,
      "logps/chosen": -347.00531005859375,
      "logps/rejected": -346.56829833984375,
      "loss": 0.6192,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.15159906446933746,
      "rewards/margins": 0.2665919363498688,
      "rewards/rejected": -0.41819101572036743,
      "step": 2150
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.509304323550221e-06,
      "logits/chosen": -2.7246994972229004,
      "logits/rejected": -2.6845176219940186,
      "logps/chosen": -282.2721252441406,
      "logps/rejected": -293.69183349609375,
      "loss": 0.5828,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.15215139091014862,
      "rewards/margins": 0.19692817330360413,
      "rewards/rejected": -0.34907954931259155,
      "step": 2160
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.502488032593398e-06,
      "logits/chosen": -2.599320888519287,
      "logits/rejected": -2.6243388652801514,
      "logps/chosen": -285.3999938964844,
      "logps/rejected": -285.7099609375,
      "loss": 0.6159,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.2641464173793793,
      "rewards/margins": 0.25711730122566223,
      "rewards/rejected": -0.5212637186050415,
      "step": 2170
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.495629951780951e-06,
      "logits/chosen": -2.5960028171539307,
      "logits/rejected": -2.637044906616211,
      "logps/chosen": -265.2070007324219,
      "logps/rejected": -312.5008239746094,
      "loss": 0.6113,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.3445412516593933,
      "rewards/margins": 0.24259276688098907,
      "rewards/rejected": -0.587134063243866,
      "step": 2180
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.488730224233941e-06,
      "logits/chosen": -2.498960018157959,
      "logits/rejected": -2.470959186553955,
      "logps/chosen": -261.1866455078125,
      "logps/rejected": -286.18438720703125,
      "loss": 0.6261,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.3730993866920471,
      "rewards/margins": 0.17336183786392212,
      "rewards/rejected": -0.5464612245559692,
      "step": 2190
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.481788993942547e-06,
      "logits/chosen": -2.605865955352783,
      "logits/rejected": -2.5414657592773438,
      "logps/chosen": -273.25665283203125,
      "logps/rejected": -297.3575134277344,
      "loss": 0.5895,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.17746077477931976,
      "rewards/margins": 0.26385658979415894,
      "rewards/rejected": -0.4413173198699951,
      "step": 2200
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.474806405763076e-06,
      "logits/chosen": -2.6408517360687256,
      "logits/rejected": -2.654766082763672,
      "logps/chosen": -335.73382568359375,
      "logps/rejected": -394.8053894042969,
      "loss": 0.6099,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18956801295280457,
      "rewards/margins": 0.32702407240867615,
      "rewards/rejected": -0.5165921449661255,
      "step": 2210
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.4677826054149235e-06,
      "logits/chosen": -2.714592695236206,
      "logits/rejected": -2.6655542850494385,
      "logps/chosen": -307.36798095703125,
      "logps/rejected": -319.69329833984375,
      "loss": 0.6278,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.14881154894828796,
      "rewards/margins": 0.2081369161605835,
      "rewards/rejected": -0.35694849491119385,
      "step": 2220
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.460717739477543e-06,
      "logits/chosen": -2.722327470779419,
      "logits/rejected": -2.643770694732666,
      "logps/chosen": -292.2183837890625,
      "logps/rejected": -276.9352111816406,
      "loss": 0.608,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12699641287326813,
      "rewards/margins": 0.22917278110980988,
      "rewards/rejected": -0.3561691641807556,
      "step": 2230
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.4536119553873866e-06,
      "logits/chosen": -2.6525509357452393,
      "logits/rejected": -2.5999083518981934,
      "logps/chosen": -264.67681884765625,
      "logps/rejected": -323.0718994140625,
      "loss": 0.6038,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.13629117608070374,
      "rewards/margins": 0.2666645348072052,
      "rewards/rejected": -0.40295568108558655,
      "step": 2240
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.446465401434824e-06,
      "logits/chosen": -2.734409809112549,
      "logits/rejected": -2.800912380218506,
      "logps/chosen": -304.42913818359375,
      "logps/rejected": -317.63055419921875,
      "loss": 0.6475,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.014048044569790363,
      "rewards/margins": 0.23579058051109314,
      "rewards/rejected": -0.22174255549907684,
      "step": 2250
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.43927822676105e-06,
      "logits/chosen": -2.636152982711792,
      "logits/rejected": -2.699537992477417,
      "logps/chosen": -292.0948181152344,
      "logps/rejected": -280.37127685546875,
      "loss": 0.5936,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0466705784201622,
      "rewards/margins": 0.27235180139541626,
      "rewards/rejected": -0.31902235746383667,
      "step": 2260
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.432050581354972e-06,
      "logits/chosen": -2.638198137283325,
      "logits/rejected": -2.615269184112549,
      "logps/chosen": -233.3173828125,
      "logps/rejected": -236.16293334960938,
      "loss": 0.5884,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.0751354768872261,
      "rewards/margins": 0.2804257869720459,
      "rewards/rejected": -0.3555612564086914,
      "step": 2270
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.424782616050078e-06,
      "logits/chosen": -2.502224922180176,
      "logits/rejected": -2.532297372817993,
      "logps/chosen": -256.5329895019531,
      "logps/rejected": -264.8097229003906,
      "loss": 0.6088,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.0532631054520607,
      "rewards/margins": 0.24914205074310303,
      "rewards/rejected": -0.30240514874458313,
      "step": 2280
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.4174744825212954e-06,
      "logits/chosen": -2.6726107597351074,
      "logits/rejected": -2.677053451538086,
      "logps/chosen": -314.2052307128906,
      "logps/rejected": -336.2852783203125,
      "loss": 0.5755,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.051678866147994995,
      "rewards/margins": 0.33552059531211853,
      "rewards/rejected": -0.3871994912624359,
      "step": 2290
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.410126333281815e-06,
      "logits/chosen": -2.5050625801086426,
      "logits/rejected": -2.568044900894165,
      "logps/chosen": -312.2085876464844,
      "logps/rejected": -248.065185546875,
      "loss": 0.6049,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.148899644613266,
      "rewards/margins": 0.2276383936405182,
      "rewards/rejected": -0.3765380382537842,
      "step": 2300
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.402738321679918e-06,
      "logits/chosen": -2.5214810371398926,
      "logits/rejected": -2.5547168254852295,
      "logps/chosen": -225.326904296875,
      "logps/rejected": -262.37127685546875,
      "loss": 0.6211,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07122506946325302,
      "rewards/margins": 0.2433677613735199,
      "rewards/rejected": -0.3145928680896759,
      "step": 2310
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.395310601895772e-06,
      "logits/chosen": -2.680088520050049,
      "logits/rejected": -2.694715976715088,
      "logps/chosen": -347.5968017578125,
      "logps/rejected": -317.98797607421875,
      "loss": 0.5905,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.058080125600099564,
      "rewards/margins": 0.2747972011566162,
      "rewards/rejected": -0.3328772783279419,
      "step": 2320
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.38784332893821e-06,
      "logits/chosen": -2.61822509765625,
      "logits/rejected": -2.62508225440979,
      "logps/chosen": -309.4066467285156,
      "logps/rejected": -281.98089599609375,
      "loss": 0.6112,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.058225117623806,
      "rewards/margins": 0.18333223462104797,
      "rewards/rejected": -0.24155735969543457,
      "step": 2330
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.380336658641503e-06,
      "logits/chosen": -2.738903284072876,
      "logits/rejected": -2.714930772781372,
      "logps/chosen": -298.86505126953125,
      "logps/rejected": -351.3769836425781,
      "loss": 0.5585,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.27263695001602173,
      "rewards/margins": 0.23865346610546112,
      "rewards/rejected": -0.5112903714179993,
      "step": 2340
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.372790747662101e-06,
      "logits/chosen": -2.6900925636291504,
      "logits/rejected": -2.6017818450927734,
      "logps/chosen": -295.0585021972656,
      "logps/rejected": -327.07720947265625,
      "loss": 0.6036,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.25599241256713867,
      "rewards/margins": 0.16895782947540283,
      "rewards/rejected": -0.4249502122402191,
      "step": 2350
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.365205753475367e-06,
      "logits/chosen": -2.600548505783081,
      "logits/rejected": -2.595583200454712,
      "logps/chosen": -321.8951110839844,
      "logps/rejected": -297.0264587402344,
      "loss": 0.5745,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.20295660197734833,
      "rewards/margins": 0.32308974862098694,
      "rewards/rejected": -0.5260463356971741,
      "step": 2360
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.35758183437229e-06,
      "logits/chosen": -2.76790189743042,
      "logits/rejected": -2.672685384750366,
      "logps/chosen": -351.3072509765625,
      "logps/rejected": -269.2875061035156,
      "loss": 0.5585,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07739736884832382,
      "rewards/margins": 0.2926849126815796,
      "rewards/rejected": -0.3700822591781616,
      "step": 2370
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.3499191494561835e-06,
      "logits/chosen": -2.5067715644836426,
      "logits/rejected": -2.4817240238189697,
      "logps/chosen": -318.60321044921875,
      "logps/rejected": -330.294677734375,
      "loss": 0.6151,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.14703303575515747,
      "rewards/margins": 0.286705881357193,
      "rewards/rejected": -0.4337388873100281,
      "step": 2380
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.3422178586393615e-06,
      "logits/chosen": -2.65042781829834,
      "logits/rejected": -2.702307939529419,
      "logps/chosen": -345.66900634765625,
      "logps/rejected": -303.6436462402344,
      "loss": 0.5587,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10787608474493027,
      "rewards/margins": 0.35162174701690674,
      "rewards/rejected": -0.4594978392124176,
      "step": 2390
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.334478122639804e-06,
      "logits/chosen": -2.618354320526123,
      "logits/rejected": -2.6288890838623047,
      "logps/chosen": -348.8436584472656,
      "logps/rejected": -257.6031799316406,
      "loss": 0.6252,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.1821097582578659,
      "rewards/margins": 0.15081281960010529,
      "rewards/rejected": -0.3329225182533264,
      "step": 2400
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.3267001029778015e-06,
      "logits/chosen": -2.5049855709075928,
      "logits/rejected": -2.5256404876708984,
      "logps/chosen": -303.19287109375,
      "logps/rejected": -276.8233642578125,
      "loss": 0.5768,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.05462324619293213,
      "rewards/margins": 0.33293142914772034,
      "rewards/rejected": -0.38755467534065247,
      "step": 2410
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.318883961972585e-06,
      "logits/chosen": -2.7734265327453613,
      "logits/rejected": -2.732790231704712,
      "logps/chosen": -262.537109375,
      "logps/rejected": -261.60992431640625,
      "loss": 0.5923,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.18073582649230957,
      "rewards/margins": 0.18429867923259735,
      "rewards/rejected": -0.3650345206260681,
      "step": 2420
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.311029862738942e-06,
      "logits/chosen": -2.5392842292785645,
      "logits/rejected": -2.447587251663208,
      "logps/chosen": -234.7491455078125,
      "logps/rejected": -296.4173278808594,
      "loss": 0.6278,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.15868954360485077,
      "rewards/margins": 0.24600186944007874,
      "rewards/rejected": -0.4046913981437683,
      "step": 2430
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.303137969183804e-06,
      "logits/chosen": -2.666558265686035,
      "logits/rejected": -2.6016013622283936,
      "logps/chosen": -304.29144287109375,
      "logps/rejected": -345.33074951171875,
      "loss": 0.5998,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.15894465148448944,
      "rewards/margins": 0.28240758180618286,
      "rewards/rejected": -0.4413522779941559,
      "step": 2440
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.295208446002832e-06,
      "logits/chosen": -2.5865111351013184,
      "logits/rejected": -2.5046756267547607,
      "logps/chosen": -230.9037628173828,
      "logps/rejected": -231.08023071289062,
      "loss": 0.5849,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.17778857052326202,
      "rewards/margins": 0.2345859706401825,
      "rewards/rejected": -0.4123745560646057,
      "step": 2450
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.287241458676981e-06,
      "logits/chosen": -2.5890355110168457,
      "logits/rejected": -2.5796329975128174,
      "logps/chosen": -300.4346923828125,
      "logps/rejected": -302.90716552734375,
      "loss": 0.5272,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08139663934707642,
      "rewards/margins": 0.45793309807777405,
      "rewards/rejected": -0.5393297076225281,
      "step": 2460
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.279237173469043e-06,
      "logits/chosen": -2.575840473175049,
      "logits/rejected": -2.5465903282165527,
      "logps/chosen": -327.3525085449219,
      "logps/rejected": -335.3431701660156,
      "loss": 0.6049,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.21273550391197205,
      "rewards/margins": 0.35986778140068054,
      "rewards/rejected": -0.5726032853126526,
      "step": 2470
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.271195757420177e-06,
      "logits/chosen": -2.612546443939209,
      "logits/rejected": -2.5679924488067627,
      "logps/chosen": -294.57562255859375,
      "logps/rejected": -294.7464599609375,
      "loss": 0.6151,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.3233681917190552,
      "rewards/margins": 0.2548545002937317,
      "rewards/rejected": -0.5782226920127869,
      "step": 2480
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.263117378346425e-06,
      "logits/chosen": -2.7228329181671143,
      "logits/rejected": -2.685037136077881,
      "logps/chosen": -280.5471496582031,
      "logps/rejected": -290.08734130859375,
      "loss": 0.5829,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.17137329280376434,
      "rewards/margins": 0.3288649618625641,
      "rewards/rejected": -0.500238299369812,
      "step": 2490
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.255002204835208e-06,
      "logits/chosen": -2.5566020011901855,
      "logits/rejected": -2.4897701740264893,
      "logps/chosen": -294.4997253417969,
      "logps/rejected": -278.2536315917969,
      "loss": 0.5891,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.26133105158805847,
      "rewards/margins": 0.29285332560539246,
      "rewards/rejected": -0.5541844367980957,
      "step": 2500
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.246850406241812e-06,
      "logits/chosen": -2.6599020957946777,
      "logits/rejected": -2.5850393772125244,
      "logps/chosen": -371.8307800292969,
      "logps/rejected": -401.79144287109375,
      "loss": 0.5856,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.17885318398475647,
      "rewards/margins": 0.34780776500701904,
      "rewards/rejected": -0.5266609787940979,
      "step": 2510
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.2386621526858465e-06,
      "logits/chosen": -2.6022260189056396,
      "logits/rejected": -2.662214756011963,
      "logps/chosen": -286.67987060546875,
      "logps/rejected": -300.2646789550781,
      "loss": 0.6027,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1267765760421753,
      "rewards/margins": 0.29710814356803894,
      "rewards/rejected": -0.423884779214859,
      "step": 2520
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.2304376150477015e-06,
      "logits/chosen": -2.750474452972412,
      "logits/rejected": -2.586371660232544,
      "logps/chosen": -297.4253234863281,
      "logps/rejected": -304.0201416015625,
      "loss": 0.6329,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.15587542951107025,
      "rewards/margins": 0.16625931859016418,
      "rewards/rejected": -0.32213476300239563,
      "step": 2530
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.222176964964977e-06,
      "logits/chosen": -2.681471109390259,
      "logits/rejected": -2.567335367202759,
      "logps/chosen": -292.96337890625,
      "logps/rejected": -307.2474670410156,
      "loss": 0.5617,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08317027986049652,
      "rewards/margins": 0.4039889872074127,
      "rewards/rejected": -0.4871593117713928,
      "step": 2540
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.213880374828903e-06,
      "logits/chosen": -2.5859360694885254,
      "logits/rejected": -2.645756244659424,
      "logps/chosen": -301.61871337890625,
      "logps/rejected": -295.96185302734375,
      "loss": 0.575,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.14113113284111023,
      "rewards/margins": 0.3010835647583008,
      "rewards/rejected": -0.4422147274017334,
      "step": 2550
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.2055480177807406e-06,
      "logits/chosen": -2.6160552501678467,
      "logits/rejected": -2.603322744369507,
      "logps/chosen": -305.6348571777344,
      "logps/rejected": -302.64752197265625,
      "loss": 0.5837,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.11300075054168701,
      "rewards/margins": 0.35259196162223816,
      "rewards/rejected": -0.46559271216392517,
      "step": 2560
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.1971800677081696e-06,
      "logits/chosen": -2.694335460662842,
      "logits/rejected": -2.601956844329834,
      "logps/chosen": -246.8837890625,
      "logps/rejected": -275.73382568359375,
      "loss": 0.5944,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.12262852489948273,
      "rewards/margins": 0.37996551394462585,
      "rewards/rejected": -0.5025940537452698,
      "step": 2570
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.188776699241661e-06,
      "logits/chosen": -2.563056230545044,
      "logits/rejected": -2.551748275756836,
      "logps/chosen": -245.7527618408203,
      "logps/rejected": -295.61627197265625,
      "loss": 0.6244,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.27490201592445374,
      "rewards/margins": 0.20333334803581238,
      "rewards/rejected": -0.4782353341579437,
      "step": 2580
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.180338087750827e-06,
      "logits/chosen": -2.5578365325927734,
      "logits/rejected": -2.445997714996338,
      "logps/chosen": -328.2390441894531,
      "logps/rejected": -315.45623779296875,
      "loss": 0.627,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3033788502216339,
      "rewards/margins": 0.23796923458576202,
      "rewards/rejected": -0.5413480997085571,
      "step": 2590
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.1718644093407704e-06,
      "logits/chosen": -2.6223864555358887,
      "logits/rejected": -2.6003425121307373,
      "logps/chosen": -280.17181396484375,
      "logps/rejected": -307.8580627441406,
      "loss": 0.6391,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.15243183076381683,
      "rewards/margins": 0.2037380486726761,
      "rewards/rejected": -0.35616984963417053,
      "step": 2600
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.163355840848401e-06,
      "logits/chosen": -2.6480376720428467,
      "logits/rejected": -2.6332411766052246,
      "logps/chosen": -287.12371826171875,
      "logps/rejected": -306.5587158203125,
      "loss": 0.598,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.10371170192956924,
      "rewards/margins": 0.2772793769836426,
      "rewards/rejected": -0.3809911012649536,
      "step": 2610
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.154812559838748e-06,
      "logits/chosen": -2.655802011489868,
      "logits/rejected": -2.6213440895080566,
      "logps/chosen": -295.09576416015625,
      "logps/rejected": -291.01861572265625,
      "loss": 0.5772,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.10176579654216766,
      "rewards/margins": 0.3795744776725769,
      "rewards/rejected": -0.48134031891822815,
      "step": 2620
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.146234744601259e-06,
      "logits/chosen": -2.516303300857544,
      "logits/rejected": -2.4881467819213867,
      "logps/chosen": -328.66119384765625,
      "logps/rejected": -295.0292663574219,
      "loss": 0.5791,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.20398323237895966,
      "rewards/margins": 0.32930490374565125,
      "rewards/rejected": -0.5332881212234497,
      "step": 2630
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.137622574146071e-06,
      "logits/chosen": -2.7313828468322754,
      "logits/rejected": -2.6179592609405518,
      "logps/chosen": -272.50152587890625,
      "logps/rejected": -266.53436279296875,
      "loss": 0.643,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.2861572206020355,
      "rewards/margins": 0.20379790663719177,
      "rewards/rejected": -0.4899550974369049,
      "step": 2640
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.12897622820028e-06,
      "logits/chosen": -2.56074857711792,
      "logits/rejected": -2.6304028034210205,
      "logps/chosen": -348.1752624511719,
      "logps/rejected": -356.3681335449219,
      "loss": 0.6095,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2683704197406769,
      "rewards/margins": 0.28429487347602844,
      "rewards/rejected": -0.5526652932167053,
      "step": 2650
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.120295887204191e-06,
      "logits/chosen": -2.4651379585266113,
      "logits/rejected": -2.4907755851745605,
      "logps/chosen": -326.99737548828125,
      "logps/rejected": -277.8379821777344,
      "loss": 0.6253,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.17758725583553314,
      "rewards/margins": 0.28153958916664124,
      "rewards/rejected": -0.4591267704963684,
      "step": 2660
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.111581732307548e-06,
      "logits/chosen": -2.5480923652648926,
      "logits/rejected": -2.5650529861450195,
      "logps/chosen": -389.673583984375,
      "logps/rejected": -335.89691162109375,
      "loss": 0.6038,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2898654341697693,
      "rewards/margins": 0.297260046005249,
      "rewards/rejected": -0.5871254205703735,
      "step": 2670
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.1028339453657595e-06,
      "logits/chosen": -2.5542445182800293,
      "logits/rejected": -2.5450551509857178,
      "logps/chosen": -300.4084777832031,
      "logps/rejected": -293.3271789550781,
      "loss": 0.5868,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1817891150712967,
      "rewards/margins": 0.2439722716808319,
      "rewards/rejected": -0.4257614016532898,
      "step": 2680
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.094052708936096e-06,
      "logits/chosen": -2.6329503059387207,
      "logits/rejected": -2.6375489234924316,
      "logps/chosen": -302.6545104980469,
      "logps/rejected": -266.0619201660156,
      "loss": 0.5953,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.22319269180297852,
      "rewards/margins": 0.2468569576740265,
      "rewards/rejected": -0.4700496792793274,
      "step": 2690
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.0852382062738874e-06,
      "logits/chosen": -2.5434086322784424,
      "logits/rejected": -2.616213083267212,
      "logps/chosen": -265.0691833496094,
      "logps/rejected": -288.4176330566406,
      "loss": 0.6006,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.295448362827301,
      "rewards/margins": 0.2674034833908081,
      "rewards/rejected": -0.5628517866134644,
      "step": 2700
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.076390621328693e-06,
      "logits/chosen": -2.605640411376953,
      "logits/rejected": -2.5978264808654785,
      "logps/chosen": -278.6802062988281,
      "logps/rejected": -245.44406127929688,
      "loss": 0.6441,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.22599074244499207,
      "rewards/margins": 0.18402789533138275,
      "rewards/rejected": -0.4100186228752136,
      "step": 2710
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.067510138740467e-06,
      "logits/chosen": -2.5916764736175537,
      "logits/rejected": -2.516136407852173,
      "logps/chosen": -295.1086120605469,
      "logps/rejected": -310.83648681640625,
      "loss": 0.6608,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.31419694423675537,
      "rewards/margins": 0.19866469502449036,
      "rewards/rejected": -0.5128616094589233,
      "step": 2720
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.058596943835703e-06,
      "logits/chosen": -2.6483025550842285,
      "logits/rejected": -2.579637289047241,
      "logps/chosen": -307.8289794921875,
      "logps/rejected": -359.41021728515625,
      "loss": 0.5861,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.23769135773181915,
      "rewards/margins": 0.35159462690353394,
      "rewards/rejected": -0.5892860293388367,
      "step": 2730
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.049651222623568e-06,
      "logits/chosen": -2.4489874839782715,
      "logits/rejected": -2.464409112930298,
      "logps/chosen": -321.61859130859375,
      "logps/rejected": -280.51934814453125,
      "loss": 0.6288,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.17855051159858704,
      "rewards/margins": 0.26512032747268677,
      "rewards/rejected": -0.4436708390712738,
      "step": 2740
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.040673161792014e-06,
      "logits/chosen": -2.571610927581787,
      "logits/rejected": -2.623656749725342,
      "logps/chosen": -244.89987182617188,
      "logps/rejected": -275.18072509765625,
      "loss": 0.5884,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1748734712600708,
      "rewards/margins": 0.3799809217453003,
      "rewards/rejected": -0.5548543930053711,
      "step": 2750
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.031662948703896e-06,
      "logits/chosen": -2.6747171878814697,
      "logits/rejected": -2.6468615531921387,
      "logps/chosen": -337.708251953125,
      "logps/rejected": -333.896240234375,
      "loss": 0.5863,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.120061494410038,
      "rewards/margins": 0.35260099172592163,
      "rewards/rejected": -0.4726625084877014,
      "step": 2760
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.022620771393047e-06,
      "logits/chosen": -2.544447660446167,
      "logits/rejected": -2.5561606884002686,
      "logps/chosen": -316.19488525390625,
      "logps/rejected": -294.08282470703125,
      "loss": 0.5838,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.252451092004776,
      "rewards/margins": 0.18410982191562653,
      "rewards/rejected": -0.43656086921691895,
      "step": 2770
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.013546818560362e-06,
      "logits/chosen": -2.668536901473999,
      "logits/rejected": -2.6385653018951416,
      "logps/chosen": -361.6271667480469,
      "logps/rejected": -322.5511169433594,
      "loss": 0.5876,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.25951510667800903,
      "rewards/margins": 0.21376793086528778,
      "rewards/rejected": -0.473283052444458,
      "step": 2780
    },
    {
      "epoch": 0.37,
      "learning_rate": 4.00444127956986e-06,
      "logits/chosen": -2.5495948791503906,
      "logits/rejected": -2.532189130783081,
      "logps/chosen": -326.3888244628906,
      "logps/rejected": -347.3705139160156,
      "loss": 0.5931,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21870854496955872,
      "rewards/margins": 0.30374085903167725,
      "rewards/rejected": -0.5224493741989136,
      "step": 2790
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.9953043444447255e-06,
      "logits/chosen": -2.64678955078125,
      "logits/rejected": -2.6592841148376465,
      "logps/chosen": -333.729736328125,
      "logps/rejected": -371.07586669921875,
      "loss": 0.6334,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.24805398285388947,
      "rewards/margins": 0.23774032294750214,
      "rewards/rejected": -0.4857942461967468,
      "step": 2800
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.986136203863355e-06,
      "logits/chosen": -2.6806514263153076,
      "logits/rejected": -2.665834903717041,
      "logps/chosen": -304.5228576660156,
      "logps/rejected": -318.13751220703125,
      "loss": 0.5897,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.2563796043395996,
      "rewards/margins": 0.24397775530815125,
      "rewards/rejected": -0.5003573298454285,
      "step": 2810
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.976937049155365e-06,
      "logits/chosen": -2.6606526374816895,
      "logits/rejected": -2.564753293991089,
      "logps/chosen": -314.8397521972656,
      "logps/rejected": -387.33428955078125,
      "loss": 0.6089,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.16455021500587463,
      "rewards/margins": 0.2586231231689453,
      "rewards/rejected": -0.42317327857017517,
      "step": 2820
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.967707072297608e-06,
      "logits/chosen": -2.628833532333374,
      "logits/rejected": -2.5874598026275635,
      "logps/chosen": -237.16250610351562,
      "logps/rejected": -254.37435913085938,
      "loss": 0.5682,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.02314874157309532,
      "rewards/margins": 0.287275105714798,
      "rewards/rejected": -0.3104238510131836,
      "step": 2830
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.958446465910159e-06,
      "logits/chosen": -2.5451273918151855,
      "logits/rejected": -2.622796058654785,
      "logps/chosen": -285.39422607421875,
      "logps/rejected": -320.67919921875,
      "loss": 0.6052,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.12590385973453522,
      "rewards/margins": 0.18081295490264893,
      "rewards/rejected": -0.30671682953834534,
      "step": 2840
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.9491554232523066e-06,
      "logits/chosen": -2.461094856262207,
      "logits/rejected": -2.485332489013672,
      "logps/chosen": -262.26190185546875,
      "logps/rejected": -277.8798828125,
      "loss": 0.619,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09473375976085663,
      "rewards/margins": 0.26800814270973206,
      "rewards/rejected": -0.3627418875694275,
      "step": 2850
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.939834138218505e-06,
      "logits/chosen": -2.543159008026123,
      "logits/rejected": -2.5069081783294678,
      "logps/chosen": -239.6870574951172,
      "logps/rejected": -274.2434387207031,
      "loss": 0.5939,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07561665028333664,
      "rewards/margins": 0.23530197143554688,
      "rewards/rejected": -0.3109186291694641,
      "step": 2860
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.930482805334339e-06,
      "logits/chosen": -2.613980293273926,
      "logits/rejected": -2.5922513008117676,
      "logps/chosen": -265.4554748535156,
      "logps/rejected": -252.3996124267578,
      "loss": 0.6082,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.189830020070076,
      "rewards/margins": 0.1698548048734665,
      "rewards/rejected": -0.35968488454818726,
      "step": 2870
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.921101619752464e-06,
      "logits/chosen": -2.606279134750366,
      "logits/rejected": -2.6501641273498535,
      "logps/chosen": -312.08782958984375,
      "logps/rejected": -354.805419921875,
      "loss": 0.5963,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.15292449295520782,
      "rewards/margins": 0.2811005413532257,
      "rewards/rejected": -0.43402498960494995,
      "step": 2880
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.911690777248525e-06,
      "logits/chosen": -2.5766007900238037,
      "logits/rejected": -2.56482195854187,
      "logps/chosen": -257.36468505859375,
      "logps/rejected": -266.5788879394531,
      "loss": 0.6007,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1831916868686676,
      "rewards/margins": 0.14657798409461975,
      "rewards/rejected": -0.32976967096328735,
      "step": 2890
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.902250474217079e-06,
      "logits/chosen": -2.509827136993408,
      "logits/rejected": -2.5269761085510254,
      "logps/chosen": -199.52395629882812,
      "logps/rejected": -279.4906921386719,
      "loss": 0.5741,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.13312488794326782,
      "rewards/margins": 0.4061674177646637,
      "rewards/rejected": -0.5392922759056091,
      "step": 2900
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.892780907667495e-06,
      "logits/chosen": -2.6970276832580566,
      "logits/rejected": -2.577418565750122,
      "logps/chosen": -323.4083557128906,
      "logps/rejected": -338.27716064453125,
      "loss": 0.5847,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06340484321117401,
      "rewards/margins": 0.26021626591682434,
      "rewards/rejected": -0.32362112402915955,
      "step": 2910
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.883282275219837e-06,
      "logits/chosen": -2.563586711883545,
      "logits/rejected": -2.5951290130615234,
      "logps/chosen": -277.5760192871094,
      "logps/rejected": -300.0455017089844,
      "loss": 0.5965,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.168487086892128,
      "rewards/margins": 0.26566335558891296,
      "rewards/rejected": -0.4341503977775574,
      "step": 2920
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.873754775100751e-06,
      "logits/chosen": -2.4823224544525146,
      "logits/rejected": -2.4202988147735596,
      "logps/chosen": -268.9900207519531,
      "logps/rejected": -318.63873291015625,
      "loss": 0.5996,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.21859273314476013,
      "rewards/margins": 0.2954866886138916,
      "rewards/rejected": -0.5140794515609741,
      "step": 2930
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.8641986061393145e-06,
      "logits/chosen": -2.676154613494873,
      "logits/rejected": -2.6082122325897217,
      "logps/chosen": -307.00872802734375,
      "logps/rejected": -281.6736755371094,
      "loss": 0.5508,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08748184889554977,
      "rewards/margins": 0.4146133363246918,
      "rewards/rejected": -0.5020952224731445,
      "step": 2940
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.854613967762898e-06,
      "logits/chosen": -2.4909651279449463,
      "logits/rejected": -2.5078840255737305,
      "logps/chosen": -279.0023498535156,
      "logps/rejected": -332.1703186035156,
      "loss": 0.5596,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.09293212741613388,
      "rewards/margins": 0.3573867678642273,
      "rewards/rejected": -0.4503189027309418,
      "step": 2950
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.845001059992999e-06,
      "logits/chosen": -2.661405086517334,
      "logits/rejected": -2.599020004272461,
      "logps/chosen": -331.3071594238281,
      "logps/rejected": -333.71746826171875,
      "loss": 0.6092,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.19790147244930267,
      "rewards/margins": 0.24655024707317352,
      "rewards/rejected": -0.4444517195224762,
      "step": 2960
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.835360083441067e-06,
      "logits/chosen": -2.714463472366333,
      "logits/rejected": -2.6763317584991455,
      "logps/chosen": -273.2596740722656,
      "logps/rejected": -295.5934143066406,
      "loss": 0.6252,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.3185594081878662,
      "rewards/margins": 0.18595674633979797,
      "rewards/rejected": -0.5045161247253418,
      "step": 2970
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.825691239304318e-06,
      "logits/chosen": -2.5877997875213623,
      "logits/rejected": -2.5963594913482666,
      "logps/chosen": -289.68316650390625,
      "logps/rejected": -308.5525817871094,
      "loss": 0.6279,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.20810465514659882,
      "rewards/margins": 0.3502613306045532,
      "rewards/rejected": -0.5583660006523132,
      "step": 2980
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.8159947293615385e-06,
      "logits/chosen": -2.6402833461761475,
      "logits/rejected": -2.645634412765503,
      "logps/chosen": -328.13360595703125,
      "logps/rejected": -304.3066101074219,
      "loss": 0.6314,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.23268917202949524,
      "rewards/margins": 0.17129898071289062,
      "rewards/rejected": -0.40398818254470825,
      "step": 2990
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.806270755968866e-06,
      "logits/chosen": -2.6010279655456543,
      "logits/rejected": -2.635829448699951,
      "logps/chosen": -193.94699096679688,
      "logps/rejected": -273.6504211425781,
      "loss": 0.5526,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.020387057214975357,
      "rewards/margins": 0.34069180488586426,
      "rewards/rejected": -0.3610789179801941,
      "step": 3000
    },
    {
      "epoch": 0.39,
      "eval_logits/chosen": -2.592276096343994,
      "eval_logits/rejected": -2.565138101577759,
      "eval_logps/chosen": -299.8006591796875,
      "eval_logps/rejected": -291.83782958984375,
      "eval_loss": 0.6012948751449585,
      "eval_rewards/accuracies": 0.7039999961853027,
      "eval_rewards/chosen": -0.15081314742565155,
      "eval_rewards/margins": 0.2546119689941406,
      "eval_rewards/rejected": -0.40542513132095337,
      "eval_runtime": 469.9385,
      "eval_samples_per_second": 4.256,
      "eval_steps_per_second": 1.064,
      "step": 3000
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.7965195220555784e-06,
      "logits/chosen": -2.6464450359344482,
      "logits/rejected": -2.64005446434021,
      "logps/chosen": -304.0558166503906,
      "logps/rejected": -334.44268798828125,
      "loss": 0.5749,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.06986355036497116,
      "rewards/margins": 0.3272497355937958,
      "rewards/rejected": -0.39711323380470276,
      "step": 3010
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.786741231119847e-06,
      "logits/chosen": -2.5847103595733643,
      "logits/rejected": -2.610323667526245,
      "logps/chosen": -271.672607421875,
      "logps/rejected": -317.8426208496094,
      "loss": 0.5225,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07842519134283066,
      "rewards/margins": 0.4202388823032379,
      "rewards/rejected": -0.4986640512943268,
      "step": 3020
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7769360872244992e-06,
      "logits/chosen": -2.639026641845703,
      "logits/rejected": -2.666348934173584,
      "logps/chosen": -333.561767578125,
      "logps/rejected": -347.01654052734375,
      "loss": 0.6074,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.2456272393465042,
      "rewards/margins": 0.20590810477733612,
      "rewards/rejected": -0.4515353739261627,
      "step": 3030
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.767104294992754e-06,
      "logits/chosen": -2.6955556869506836,
      "logits/rejected": -2.588688373565674,
      "logps/chosen": -276.519775390625,
      "logps/rejected": -271.98175048828125,
      "loss": 0.6202,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1983802318572998,
      "rewards/margins": 0.18987175822257996,
      "rewards/rejected": -0.38825201988220215,
      "step": 3040
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7572460596039524e-06,
      "logits/chosen": -2.574049949645996,
      "logits/rejected": -2.558225631713867,
      "logps/chosen": -259.74078369140625,
      "logps/rejected": -285.2066955566406,
      "loss": 0.639,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.2721092998981476,
      "rewards/margins": 0.23243455588817596,
      "rewards/rejected": -0.5045438408851624,
      "step": 3050
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.74736158678928e-06,
      "logits/chosen": -2.3812010288238525,
      "logits/rejected": -2.39877986907959,
      "logps/chosen": -238.91720581054688,
      "logps/rejected": -265.1916198730469,
      "loss": 0.5699,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.2072840929031372,
      "rewards/margins": 0.3541075587272644,
      "rewards/rejected": -0.5613916516304016,
      "step": 3060
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7374510828274673e-06,
      "logits/chosen": -2.516413688659668,
      "logits/rejected": -2.531301498413086,
      "logps/chosen": -277.7062072753906,
      "logps/rejected": -348.3893127441406,
      "loss": 0.6018,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19995872676372528,
      "rewards/margins": 0.2855435311794281,
      "rewards/rejected": -0.4855022430419922,
      "step": 3070
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.72751475454049e-06,
      "logits/chosen": -2.5162839889526367,
      "logits/rejected": -2.4353349208831787,
      "logps/chosen": -319.59063720703125,
      "logps/rejected": -318.60955810546875,
      "loss": 0.5743,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.20623521506786346,
      "rewards/margins": 0.3293301463127136,
      "rewards/rejected": -0.5355653166770935,
      "step": 3080
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7175528092892503e-06,
      "logits/chosen": -2.6033437252044678,
      "logits/rejected": -2.5976102352142334,
      "logps/chosen": -297.6019592285156,
      "logps/rejected": -282.84210205078125,
      "loss": 0.6204,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.22910642623901367,
      "rewards/margins": 0.24689094722270966,
      "rewards/rejected": -0.47599735856056213,
      "step": 3090
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.7075654549692498e-06,
      "logits/chosen": -2.5105113983154297,
      "logits/rejected": -2.4831717014312744,
      "logps/chosen": -291.5171203613281,
      "logps/rejected": -290.1797790527344,
      "loss": 0.614,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2578617036342621,
      "rewards/margins": 0.2936268448829651,
      "rewards/rejected": -0.5514885187149048,
      "step": 3100
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.697552900006249e-06,
      "logits/chosen": -2.6630959510803223,
      "logits/rejected": -2.620387315750122,
      "logps/chosen": -306.1521301269531,
      "logps/rejected": -358.87200927734375,
      "loss": 0.5936,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3080160915851593,
      "rewards/margins": 0.1846139132976532,
      "rewards/rejected": -0.4926300048828125,
      "step": 3110
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6875153533519244e-06,
      "logits/chosen": -2.624591827392578,
      "logits/rejected": -2.493398666381836,
      "logps/chosen": -308.59588623046875,
      "logps/rejected": -290.4993591308594,
      "loss": 0.5787,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1921892762184143,
      "rewards/margins": 0.33598488569259644,
      "rewards/rejected": -0.5281742215156555,
      "step": 3120
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6774530244794992e-06,
      "logits/chosen": -2.6168274879455566,
      "logits/rejected": -2.5857300758361816,
      "logps/chosen": -343.65496826171875,
      "logps/rejected": -316.0412292480469,
      "loss": 0.588,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.18571509420871735,
      "rewards/margins": 0.2884679436683655,
      "rewards/rejected": -0.47418302297592163,
      "step": 3130
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.667366123379378e-06,
      "logits/chosen": -2.5256686210632324,
      "logits/rejected": -2.486649751663208,
      "logps/chosen": -301.6999206542969,
      "logps/rejected": -353.1916198730469,
      "loss": 0.566,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.22445659339427948,
      "rewards/margins": 0.341928631067276,
      "rewards/rejected": -0.5663852691650391,
      "step": 3140
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6572548605547607e-06,
      "logits/chosen": -2.6104562282562256,
      "logits/rejected": -2.4722983837127686,
      "logps/chosen": -324.6009216308594,
      "logps/rejected": -305.4631042480469,
      "loss": 0.5586,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2462780922651291,
      "rewards/margins": 0.3060448169708252,
      "rewards/rejected": -0.5523229241371155,
      "step": 3150
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6471194470172538e-06,
      "logits/chosen": -2.751009702682495,
      "logits/rejected": -2.7248828411102295,
      "logps/chosen": -379.75750732421875,
      "logps/rejected": -380.437744140625,
      "loss": 0.5762,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.25567176938056946,
      "rewards/margins": 0.29870104789733887,
      "rewards/rejected": -0.5543727874755859,
      "step": 3160
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.636960094282461e-06,
      "logits/chosen": -2.548859119415283,
      "logits/rejected": -2.6039133071899414,
      "logps/chosen": -244.54806518554688,
      "logps/rejected": -294.4767150878906,
      "loss": 0.5389,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.14730247855186462,
      "rewards/margins": 0.3509523272514343,
      "rewards/rejected": -0.49825483560562134,
      "step": 3170
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.6267770143655743e-06,
      "logits/chosen": -2.550873279571533,
      "logits/rejected": -2.5480148792266846,
      "logps/chosen": -288.40411376953125,
      "logps/rejected": -284.4498291015625,
      "loss": 0.6286,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.15646964311599731,
      "rewards/margins": 0.17518159747123718,
      "rewards/rejected": -0.3316512405872345,
      "step": 3180
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.6165704197769484e-06,
      "logits/chosen": -2.5557799339294434,
      "logits/rejected": -2.531587600708008,
      "logps/chosen": -258.42315673828125,
      "logps/rejected": -285.3637390136719,
      "loss": 0.5297,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.12565067410469055,
      "rewards/margins": 0.4425191879272461,
      "rewards/rejected": -0.568169891834259,
      "step": 3190
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.606340523517663e-06,
      "logits/chosen": -2.667398452758789,
      "logits/rejected": -2.635216474533081,
      "logps/chosen": -324.2204895019531,
      "logps/rejected": -351.78875732421875,
      "loss": 0.5878,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.22811134159564972,
      "rewards/margins": 0.3804187774658203,
      "rewards/rejected": -0.6085301637649536,
      "step": 3200
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.5960875390750793e-06,
      "logits/chosen": -2.570162057876587,
      "logits/rejected": -2.485708236694336,
      "logps/chosen": -303.61395263671875,
      "logps/rejected": -358.18255615234375,
      "loss": 0.6511,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3610957860946655,
      "rewards/margins": 0.16257894039154053,
      "rewards/rejected": -0.523674726486206,
      "step": 3210
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.585811680418386e-06,
      "logits/chosen": -2.5814812183380127,
      "logits/rejected": -2.510042667388916,
      "logps/chosen": -278.59783935546875,
      "logps/rejected": -262.82220458984375,
      "loss": 0.5444,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.20521113276481628,
      "rewards/margins": 0.4517434537410736,
      "rewards/rejected": -0.6569545269012451,
      "step": 3220
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.5755131619941347e-06,
      "logits/chosen": -2.6885128021240234,
      "logits/rejected": -2.6287333965301514,
      "logps/chosen": -350.37750244140625,
      "logps/rejected": -355.7803039550781,
      "loss": 0.5734,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.30031758546829224,
      "rewards/margins": 0.32573914527893066,
      "rewards/rejected": -0.6260567903518677,
      "step": 3230
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.565192198721759e-06,
      "logits/chosen": -2.6210732460021973,
      "logits/rejected": -2.5164902210235596,
      "logps/chosen": -295.5282287597656,
      "logps/rejected": -249.3524169921875,
      "loss": 0.6068,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1660500019788742,
      "rewards/margins": 0.30626028776168823,
      "rewards/rejected": -0.47231030464172363,
      "step": 3240
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5548490059890965e-06,
      "logits/chosen": -2.649322509765625,
      "logits/rejected": -2.6184802055358887,
      "logps/chosen": -332.575927734375,
      "logps/rejected": -330.59796142578125,
      "loss": 0.6232,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.217446967959404,
      "rewards/margins": 0.2006688416004181,
      "rewards/rejected": -0.4181157946586609,
      "step": 3250
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5444837996478903e-06,
      "logits/chosen": -2.4342164993286133,
      "logits/rejected": -2.4442899227142334,
      "logps/chosen": -268.0245056152344,
      "logps/rejected": -336.77752685546875,
      "loss": 0.5982,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.19876603782176971,
      "rewards/margins": 0.32545793056488037,
      "rewards/rejected": -0.5242239236831665,
      "step": 3260
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.534096796009282e-06,
      "logits/chosen": -2.589726686477661,
      "logits/rejected": -2.63083815574646,
      "logps/chosen": -257.81732177734375,
      "logps/rejected": -292.96624755859375,
      "loss": 0.6304,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1578315794467926,
      "rewards/margins": 0.19295494258403778,
      "rewards/rejected": -0.3507865369319916,
      "step": 3270
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5236882118393046e-06,
      "logits/chosen": -2.65148663520813,
      "logits/rejected": -2.5955045223236084,
      "logps/chosen": -292.2409973144531,
      "logps/rejected": -288.8282470703125,
      "loss": 0.5885,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.17893002927303314,
      "rewards/margins": 0.21064350008964539,
      "rewards/rejected": -0.38957351446151733,
      "step": 3280
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5132582643543513e-06,
      "logits/chosen": -2.5080788135528564,
      "logits/rejected": -2.4526593685150146,
      "logps/chosen": -284.8882751464844,
      "logps/rejected": -280.62945556640625,
      "loss": 0.5817,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.10132601112127304,
      "rewards/margins": 0.3045029044151306,
      "rewards/rejected": -0.40582889318466187,
      "step": 3290
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5028071712166456e-06,
      "logits/chosen": -2.4202044010162354,
      "logits/rejected": -2.429652452468872,
      "logps/chosen": -287.6400451660156,
      "logps/rejected": -284.93267822265625,
      "loss": 0.5631,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.011086313053965569,
      "rewards/margins": 0.41601577401161194,
      "rewards/rejected": -0.4271020293235779,
      "step": 3300
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.4923351505297008e-06,
      "logits/chosen": -2.477534294128418,
      "logits/rejected": -2.572798252105713,
      "logps/chosen": -309.67822265625,
      "logps/rejected": -251.0945281982422,
      "loss": 0.6126,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.16956202685832977,
      "rewards/margins": 0.2933441996574402,
      "rewards/rejected": -0.46290621161460876,
      "step": 3310
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.481842420833766e-06,
      "logits/chosen": -2.6691091060638428,
      "logits/rejected": -2.6291580200195312,
      "logps/chosen": -295.6717529296875,
      "logps/rejected": -315.63372802734375,
      "loss": 0.5969,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.10933154821395874,
      "rewards/margins": 0.3607695698738098,
      "rewards/rejected": -0.4701010584831238,
      "step": 3320
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4713292011012645e-06,
      "logits/chosen": -2.5209145545959473,
      "logits/rejected": -2.5216760635375977,
      "logps/chosen": -263.0899658203125,
      "logps/rejected": -292.06268310546875,
      "loss": 0.5841,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.037089962512254715,
      "rewards/margins": 0.273418128490448,
      "rewards/rejected": -0.3105081021785736,
      "step": 3330
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4607957107322277e-06,
      "logits/chosen": -2.421794891357422,
      "logits/rejected": -2.4947047233581543,
      "logps/chosen": -231.6426239013672,
      "logps/rejected": -287.62005615234375,
      "loss": 0.5952,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.14515933394432068,
      "rewards/margins": 0.34159886837005615,
      "rewards/rejected": -0.48675817251205444,
      "step": 3340
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4502421695497112e-06,
      "logits/chosen": -2.638671398162842,
      "logits/rejected": -2.59818172454834,
      "logps/chosen": -315.3717956542969,
      "logps/rejected": -306.5726013183594,
      "loss": 0.6529,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1656460464000702,
      "rewards/margins": 0.34280601143836975,
      "rewards/rejected": -0.5084519982337952,
      "step": 3350
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4396687977952137e-06,
      "logits/chosen": -2.6577351093292236,
      "logits/rejected": -2.5850212574005127,
      "logps/chosen": -248.22463989257812,
      "logps/rejected": -258.7904357910156,
      "loss": 0.6046,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.17443545162677765,
      "rewards/margins": 0.25829359889030457,
      "rewards/rejected": -0.4327290654182434,
      "step": 3360
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.429075816124075e-06,
      "logits/chosen": -2.681647300720215,
      "logits/rejected": -2.596585750579834,
      "logps/chosen": -400.94036865234375,
      "logps/rejected": -363.10113525390625,
      "loss": 0.5466,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.19438858330249786,
      "rewards/margins": 0.32669034600257874,
      "rewards/rejected": -0.521078884601593,
      "step": 3370
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.418463445600874e-06,
      "logits/chosen": -2.681594133377075,
      "logits/rejected": -2.652360200881958,
      "logps/chosen": -337.20330810546875,
      "logps/rejected": -276.9524841308594,
      "loss": 0.6515,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.22456975281238556,
      "rewards/margins": 0.1682281792163849,
      "rewards/rejected": -0.39279794692993164,
      "step": 3380
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4078319076948173e-06,
      "logits/chosen": -2.5730137825012207,
      "logits/rejected": -2.4797158241271973,
      "logps/chosen": -296.2660827636719,
      "logps/rejected": -280.090576171875,
      "loss": 0.6146,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.24738606810569763,
      "rewards/margins": 0.13821235299110413,
      "rewards/rejected": -0.38559848070144653,
      "step": 3390
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.3971814242751123e-06,
      "logits/chosen": -2.5072569847106934,
      "logits/rejected": -2.531062364578247,
      "logps/chosen": -335.22747802734375,
      "logps/rejected": -343.5042419433594,
      "loss": 0.625,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.21663913130760193,
      "rewards/margins": 0.30990713834762573,
      "rewards/rejected": -0.5265463590621948,
      "step": 3400
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.386512217606339e-06,
      "logits/chosen": -2.6378777027130127,
      "logits/rejected": -2.516758441925049,
      "logps/chosen": -294.718994140625,
      "logps/rejected": -314.53192138671875,
      "loss": 0.5579,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.16852737963199615,
      "rewards/margins": 0.29910632967948914,
      "rewards/rejected": -0.4676336646080017,
      "step": 3410
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.375824510343816e-06,
      "logits/chosen": -2.488013505935669,
      "logits/rejected": -2.472822666168213,
      "logps/chosen": -248.4076690673828,
      "logps/rejected": -239.4960479736328,
      "loss": 0.6288,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2241649329662323,
      "rewards/margins": 0.27233701944351196,
      "rewards/rejected": -0.4965019226074219,
      "step": 3420
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3651185255289466e-06,
      "logits/chosen": -2.6994948387145996,
      "logits/rejected": -2.635267734527588,
      "logps/chosen": -312.0728454589844,
      "logps/rejected": -306.41021728515625,
      "loss": 0.5684,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1855587512254715,
      "rewards/margins": 0.33502134680747986,
      "rewards/rejected": -0.5205801129341125,
      "step": 3430
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.354394486584568e-06,
      "logits/chosen": -2.4346115589141846,
      "logits/rejected": -2.4193854331970215,
      "logps/chosen": -253.3272247314453,
      "logps/rejected": -299.01416015625,
      "loss": 0.6398,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.20227909088134766,
      "rewards/margins": 0.1328783482313156,
      "rewards/rejected": -0.33515748381614685,
      "step": 3440
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3436526173102913e-06,
      "logits/chosen": -2.5650057792663574,
      "logits/rejected": -2.5391316413879395,
      "logps/chosen": -324.17486572265625,
      "logps/rejected": -323.8554382324219,
      "loss": 0.59,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.16734786331653595,
      "rewards/margins": 0.2571793496608734,
      "rewards/rejected": -0.42452722787857056,
      "step": 3450
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3328931418778254e-06,
      "logits/chosen": -2.5991523265838623,
      "logits/rejected": -2.483994722366333,
      "logps/chosen": -273.0755920410156,
      "logps/rejected": -265.11083984375,
      "loss": 0.5952,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.25192922353744507,
      "rewards/margins": 0.3073795735836029,
      "rewards/rejected": -0.5593088269233704,
      "step": 3460
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3221162848263028e-06,
      "logits/chosen": -2.590855121612549,
      "logits/rejected": -2.6153273582458496,
      "logps/chosen": -305.8322448730469,
      "logps/rejected": -262.98583984375,
      "loss": 0.6158,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.30184227228164673,
      "rewards/margins": 0.24508753418922424,
      "rewards/rejected": -0.5469297170639038,
      "step": 3470
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.3113222710575914e-06,
      "logits/chosen": -2.6684579849243164,
      "logits/rejected": -2.6100895404815674,
      "logps/chosen": -346.12347412109375,
      "logps/rejected": -341.442626953125,
      "loss": 0.6017,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.18102851510047913,
      "rewards/margins": 0.3186485767364502,
      "rewards/rejected": -0.49967703223228455,
      "step": 3480
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.300511325831603e-06,
      "logits/chosen": -2.638643264770508,
      "logits/rejected": -2.5978457927703857,
      "logps/chosen": -328.1517028808594,
      "logps/rejected": -293.4931335449219,
      "loss": 0.5997,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.29977235198020935,
      "rewards/margins": 0.23009338974952698,
      "rewards/rejected": -0.5298658013343811,
      "step": 3490
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.289683674761592e-06,
      "logits/chosen": -2.625931739807129,
      "logits/rejected": -2.6796207427978516,
      "logps/chosen": -332.6492614746094,
      "logps/rejected": -315.2752990722656,
      "loss": 0.5766,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.2368648499250412,
      "rewards/margins": 0.351283460855484,
      "rewards/rejected": -0.588148295879364,
      "step": 3500
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2788395438094444e-06,
      "logits/chosen": -2.543121814727783,
      "logits/rejected": -2.523128032684326,
      "logps/chosen": -298.61126708984375,
      "logps/rejected": -305.8750915527344,
      "loss": 0.6151,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.16757427155971527,
      "rewards/margins": 0.27241280674934387,
      "rewards/rejected": -0.43998709321022034,
      "step": 3510
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2679791592809653e-06,
      "logits/chosen": -2.568057060241699,
      "logits/rejected": -2.665128231048584,
      "logps/chosen": -282.148681640625,
      "logps/rejected": -323.38238525390625,
      "loss": 0.5821,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.2427080124616623,
      "rewards/margins": 0.2534430921077728,
      "rewards/rejected": -0.4961511194705963,
      "step": 3520
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.257102747821157e-06,
      "logits/chosen": -2.4993700981140137,
      "logits/rejected": -2.559767246246338,
      "logps/chosen": -326.0221862792969,
      "logps/rejected": -322.7140197753906,
      "loss": 0.6242,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.23177289962768555,
      "rewards/margins": 0.330489844083786,
      "rewards/rejected": -0.5622627139091492,
      "step": 3530
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.246210536409484e-06,
      "logits/chosen": -2.466169834136963,
      "logits/rejected": -2.5016674995422363,
      "logps/chosen": -224.3962860107422,
      "logps/rejected": -225.11111450195312,
      "loss": 0.594,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.26115483045578003,
      "rewards/margins": 0.2677431106567383,
      "rewards/rejected": -0.5288979411125183,
      "step": 3540
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.235302752355142e-06,
      "logits/chosen": -2.3972480297088623,
      "logits/rejected": -2.4166183471679688,
      "logps/chosen": -303.21630859375,
      "logps/rejected": -310.1206359863281,
      "loss": 0.5783,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.3191254734992981,
      "rewards/margins": 0.32761865854263306,
      "rewards/rejected": -0.6467441320419312,
      "step": 3550
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.2243796232923097e-06,
      "logits/chosen": -2.5244884490966797,
      "logits/rejected": -2.566132068634033,
      "logps/chosen": -226.5001220703125,
      "logps/rejected": -263.5140380859375,
      "loss": 0.6309,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.17714844644069672,
      "rewards/margins": 0.2616042494773865,
      "rewards/rejected": -0.43875280022621155,
      "step": 3560
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.2134413771754037e-06,
      "logits/chosen": -2.4819984436035156,
      "logits/rejected": -2.504448890686035,
      "logps/chosen": -289.71966552734375,
      "logps/rejected": -281.09686279296875,
      "loss": 0.561,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.19265899062156677,
      "rewards/margins": 0.31390810012817383,
      "rewards/rejected": -0.506567120552063,
      "step": 3570
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.2024882422743118e-06,
      "logits/chosen": -2.560467481613159,
      "logits/rejected": -2.511833667755127,
      "logps/chosen": -282.52093505859375,
      "logps/rejected": -281.93939208984375,
      "loss": 0.6001,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.17843423783779144,
      "rewards/margins": 0.31184864044189453,
      "rewards/rejected": -0.4902828633785248,
      "step": 3580
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.1915204471696425e-06,
      "logits/chosen": -2.5695555210113525,
      "logits/rejected": -2.576490640640259,
      "logps/chosen": -296.9548645019531,
      "logps/rejected": -303.47674560546875,
      "loss": 0.5785,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.22846440970897675,
      "rewards/margins": 0.3821622431278229,
      "rewards/rejected": -0.6106266379356384,
      "step": 3590
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.180538220747943e-06,
      "logits/chosen": -2.483707904815674,
      "logits/rejected": -2.5312798023223877,
      "logps/chosen": -276.6914367675781,
      "logps/rejected": -263.9627990722656,
      "loss": 0.6374,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.33055758476257324,
      "rewards/margins": 0.12794001400470734,
      "rewards/rejected": -0.4584975838661194,
      "step": 3600
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.1695417921969287e-06,
      "logits/chosen": -2.5744469165802,
      "logits/rejected": -2.7000794410705566,
      "logps/chosen": -263.73992919921875,
      "logps/rejected": -256.0915832519531,
      "loss": 0.569,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.22559526562690735,
      "rewards/margins": 0.28088870644569397,
      "rewards/rejected": -0.5064839720726013,
      "step": 3610
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.158531391000697e-06,
      "logits/chosen": -2.6358370780944824,
      "logits/rejected": -2.648641347885132,
      "logps/chosen": -355.3272399902344,
      "logps/rejected": -328.2330322265625,
      "loss": 0.5619,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.030888553708791733,
      "rewards/margins": 0.43781572580337524,
      "rewards/rejected": -0.46870431303977966,
      "step": 3620
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.147507246934943e-06,
      "logits/chosen": -2.6370327472686768,
      "logits/rejected": -2.6142680644989014,
      "logps/chosen": -317.4229431152344,
      "logps/rejected": -301.7786560058594,
      "loss": 0.5968,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.14548811316490173,
      "rewards/margins": 0.24573619663715363,
      "rewards/rejected": -0.39122429490089417,
      "step": 3630
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.136469590062158e-06,
      "logits/chosen": -2.6314265727996826,
      "logits/rejected": -2.547762632369995,
      "logps/chosen": -284.0455322265625,
      "logps/rejected": -252.8461151123047,
      "loss": 0.6125,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.1615857183933258,
      "rewards/margins": 0.3351583480834961,
      "rewards/rejected": -0.4967440068721771,
      "step": 3640
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1254186507268354e-06,
      "logits/chosen": -2.647684097290039,
      "logits/rejected": -2.5562212467193604,
      "logps/chosen": -334.295654296875,
      "logps/rejected": -313.12054443359375,
      "loss": 0.5983,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.16967463493347168,
      "rewards/margins": 0.28552836179733276,
      "rewards/rejected": -0.45520296692848206,
      "step": 3650
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.114354659550656e-06,
      "logits/chosen": -2.6611533164978027,
      "logits/rejected": -2.6439900398254395,
      "logps/chosen": -297.71160888671875,
      "logps/rejected": -358.6143493652344,
      "loss": 0.6145,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.2865394949913025,
      "rewards/margins": 0.2464900016784668,
      "rewards/rejected": -0.5330294966697693,
      "step": 3660
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1032778474276816e-06,
      "logits/chosen": -2.620631694793701,
      "logits/rejected": -2.472928524017334,
      "logps/chosen": -323.84246826171875,
      "logps/rejected": -308.86431884765625,
      "loss": 0.6706,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.393810510635376,
      "rewards/margins": 0.13828083872795105,
      "rewards/rejected": -0.5320913791656494,
      "step": 3670
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.092188445519532e-06,
      "logits/chosen": -2.511369228363037,
      "logits/rejected": -2.5086829662323,
      "logps/chosen": -310.9083557128906,
      "logps/rejected": -297.79791259765625,
      "loss": 0.5898,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.1467629224061966,
      "rewards/margins": 0.32766348123550415,
      "rewards/rejected": -0.47442641854286194,
      "step": 3680
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.081086685250565e-06,
      "logits/chosen": -2.709105968475342,
      "logits/rejected": -2.684390068054199,
      "logps/chosen": -345.8997497558594,
      "logps/rejected": -322.6979064941406,
      "loss": 0.5933,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.19092276692390442,
      "rewards/margins": 0.25971782207489014,
      "rewards/rejected": -0.4506405293941498,
      "step": 3690
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.0699727983030434e-06,
      "logits/chosen": -2.693297863006592,
      "logits/rejected": -2.6480326652526855,
      "logps/chosen": -312.6332702636719,
      "logps/rejected": -364.988525390625,
      "loss": 0.6024,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1347484141588211,
      "rewards/margins": 0.36063846945762634,
      "rewards/rejected": -0.4953867793083191,
      "step": 3700
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.058847016612301e-06,
      "logits/chosen": -2.680645704269409,
      "logits/rejected": -2.5878353118896484,
      "logps/chosen": -380.2493896484375,
      "logps/rejected": -363.9061279296875,
      "loss": 0.5614,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.2745390236377716,
      "rewards/margins": 0.3324805796146393,
      "rewards/rejected": -0.6070197224617004,
      "step": 3710
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0477095723619034e-06,
      "logits/chosen": -2.67134952545166,
      "logits/rejected": -2.5982470512390137,
      "logps/chosen": -321.2911376953125,
      "logps/rejected": -349.79022216796875,
      "loss": 0.5855,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3027748763561249,
      "rewards/margins": 0.35173699259757996,
      "rewards/rejected": -0.6545118093490601,
      "step": 3720
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0365606979788003e-06,
      "logits/chosen": -2.4542784690856934,
      "logits/rejected": -2.5599284172058105,
      "logps/chosen": -244.4676513671875,
      "logps/rejected": -282.34149169921875,
      "loss": 0.6112,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.23147158324718475,
      "rewards/margins": 0.22280550003051758,
      "rewards/rejected": -0.4542770981788635,
      "step": 3730
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0254006261284786e-06,
      "logits/chosen": -2.676450729370117,
      "logits/rejected": -2.698291301727295,
      "logps/chosen": -315.92138671875,
      "logps/rejected": -318.750244140625,
      "loss": 0.5708,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.15017244219779968,
      "rewards/margins": 0.34940075874328613,
      "rewards/rejected": -0.4995731711387634,
      "step": 3740
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0142295897101032e-06,
      "logits/chosen": -2.508303165435791,
      "logits/rejected": -2.5160272121429443,
      "logps/chosen": -296.66790771484375,
      "logps/rejected": -296.8723449707031,
      "loss": 0.5772,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1724482625722885,
      "rewards/margins": 0.4398605227470398,
      "rewards/rejected": -0.6123087406158447,
      "step": 3750
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0030478218516578e-06,
      "logits/chosen": -2.6099557876586914,
      "logits/rejected": -2.4607856273651123,
      "logps/chosen": -300.81793212890625,
      "logps/rejected": -226.68038940429688,
      "loss": 0.5685,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1304190754890442,
      "rewards/margins": 0.4726713299751282,
      "rewards/rejected": -0.6030904054641724,
      "step": 3760
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.9918555559050826e-06,
      "logits/chosen": -2.660377264022827,
      "logits/rejected": -2.6089494228363037,
      "logps/chosen": -302.4432678222656,
      "logps/rejected": -351.26971435546875,
      "loss": 0.6301,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2564275562763214,
      "rewards/margins": 0.2971305251121521,
      "rewards/rejected": -0.5535580515861511,
      "step": 3770
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.980653025441399e-06,
      "logits/chosen": -2.7107796669006348,
      "logits/rejected": -2.7100653648376465,
      "logps/chosen": -257.9647521972656,
      "logps/rejected": -319.4724426269531,
      "loss": 0.6101,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2261107861995697,
      "rewards/margins": 0.21425211429595947,
      "rewards/rejected": -0.44036293029785156,
      "step": 3780
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.969440464245841e-06,
      "logits/chosen": -2.531522274017334,
      "logits/rejected": -2.5888233184814453,
      "logps/chosen": -250.8273162841797,
      "logps/rejected": -275.148681640625,
      "loss": 0.6257,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20485302805900574,
      "rewards/margins": 0.13838718831539154,
      "rewards/rejected": -0.34324023127555847,
      "step": 3790
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.95821810631297e-06,
      "logits/chosen": -2.6218361854553223,
      "logits/rejected": -2.605215311050415,
      "logps/chosen": -330.93865966796875,
      "logps/rejected": -379.01898193359375,
      "loss": 0.6228,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.2750341296195984,
      "rewards/margins": 0.23783652484416962,
      "rewards/rejected": -0.5128706693649292,
      "step": 3800
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.946986185841801e-06,
      "logits/chosen": -2.5598433017730713,
      "logits/rejected": -2.4834113121032715,
      "logps/chosen": -294.7371520996094,
      "logps/rejected": -299.9499206542969,
      "loss": 0.6378,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.20022502541542053,
      "rewards/margins": 0.2527645528316498,
      "rewards/rejected": -0.4529895782470703,
      "step": 3810
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.935744937230903e-06,
      "logits/chosen": -2.571002721786499,
      "logits/rejected": -2.62736177444458,
      "logps/chosen": -302.08734130859375,
      "logps/rejected": -268.59735107421875,
      "loss": 0.6057,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20417217910289764,
      "rewards/margins": 0.22127452492713928,
      "rewards/rejected": -0.4254467487335205,
      "step": 3820
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.924494595073517e-06,
      "logits/chosen": -2.4782519340515137,
      "logits/rejected": -2.4902045726776123,
      "logps/chosen": -256.9552001953125,
      "logps/rejected": -268.467041015625,
      "loss": 0.5822,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.12104479223489761,
      "rewards/margins": 0.33370283246040344,
      "rewards/rejected": -0.45474761724472046,
      "step": 3830
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9132353941526575e-06,
      "logits/chosen": -2.640956401824951,
      "logits/rejected": -2.7130298614501953,
      "logps/chosen": -322.9231262207031,
      "logps/rejected": -281.8253479003906,
      "loss": 0.584,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13268163800239563,
      "rewards/margins": 0.3568686246871948,
      "rewards/rejected": -0.48955026268959045,
      "step": 3840
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.901967569436209e-06,
      "logits/chosen": -2.459228038787842,
      "logits/rejected": -2.4970715045928955,
      "logps/chosen": -276.39727783203125,
      "logps/rejected": -308.2503967285156,
      "loss": 0.643,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.23280692100524902,
      "rewards/margins": 0.25167709589004517,
      "rewards/rejected": -0.4844840168952942,
      "step": 3850
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.89069135607203e-06,
      "logits/chosen": -2.556077718734741,
      "logits/rejected": -2.4842946529388428,
      "logps/chosen": -309.0411071777344,
      "logps/rejected": -302.27496337890625,
      "loss": 0.5564,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.06841972470283508,
      "rewards/margins": 0.40710344910621643,
      "rewards/rejected": -0.4755231440067291,
      "step": 3860
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8794069893830386e-06,
      "logits/chosen": -2.4000508785247803,
      "logits/rejected": -2.368074655532837,
      "logps/chosen": -318.82696533203125,
      "logps/rejected": -374.33514404296875,
      "loss": 0.5781,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.09505738317966461,
      "rewards/margins": 0.2986064553260803,
      "rewards/rejected": -0.39366385340690613,
      "step": 3870
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8681147048623038e-06,
      "logits/chosen": -2.6203765869140625,
      "logits/rejected": -2.622159242630005,
      "logps/chosen": -339.40447998046875,
      "logps/rejected": -289.11962890625,
      "loss": 0.6172,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.20032520592212677,
      "rewards/margins": 0.13905063271522522,
      "rewards/rejected": -0.33937588334083557,
      "step": 3880
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8568147381681333e-06,
      "logits/chosen": -2.635042190551758,
      "logits/rejected": -2.4665164947509766,
      "logps/chosen": -334.91741943359375,
      "logps/rejected": -312.35784912109375,
      "loss": 0.5751,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1310148388147354,
      "rewards/margins": 0.31944572925567627,
      "rewards/rejected": -0.4504605829715729,
      "step": 3890
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8455073251191533e-06,
      "logits/chosen": -2.6655802726745605,
      "logits/rejected": -2.5834178924560547,
      "logps/chosen": -306.61749267578125,
      "logps/rejected": -317.67901611328125,
      "loss": 0.5782,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.17261703312397003,
      "rewards/margins": 0.29314273595809937,
      "rewards/rejected": -0.4657597541809082,
      "step": 3900
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8341927016893887e-06,
      "logits/chosen": -2.6398985385894775,
      "logits/rejected": -2.5800163745880127,
      "logps/chosen": -316.98712158203125,
      "logps/rejected": -316.6258850097656,
      "loss": 0.631,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.25684791803359985,
      "rewards/margins": 0.24398711323738098,
      "rewards/rejected": -0.5008348822593689,
      "step": 3910
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.822871104003335e-06,
      "logits/chosen": -2.5508549213409424,
      "logits/rejected": -2.56037974357605,
      "logps/chosen": -268.0732116699219,
      "logps/rejected": -326.3709411621094,
      "loss": 0.5836,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20186085999011993,
      "rewards/margins": 0.32339686155319214,
      "rewards/rejected": -0.5252577066421509,
      "step": 3920
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8115427683310355e-06,
      "logits/chosen": -2.626038074493408,
      "logits/rejected": -2.6347310543060303,
      "logps/chosen": -272.03289794921875,
      "logps/rejected": -298.04791259765625,
      "loss": 0.5635,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1626121997833252,
      "rewards/margins": 0.28776103258132935,
      "rewards/rejected": -0.45037323236465454,
      "step": 3930
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.8002079310831477e-06,
      "logits/chosen": -2.581829071044922,
      "logits/rejected": -2.4708094596862793,
      "logps/chosen": -306.98431396484375,
      "logps/rejected": -275.0307922363281,
      "loss": 0.589,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.2808947265148163,
      "rewards/margins": 0.3008837401866913,
      "rewards/rejected": -0.5817785859107971,
      "step": 3940
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7888668288060095e-06,
      "logits/chosen": -2.575941562652588,
      "logits/rejected": -2.555351495742798,
      "logps/chosen": -281.966796875,
      "logps/rejected": -320.59222412109375,
      "loss": 0.5753,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.3274231255054474,
      "rewards/margins": 0.3251662850379944,
      "rewards/rejected": -0.6525893807411194,
      "step": 3950
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7775196981767044e-06,
      "logits/chosen": -2.67581844329834,
      "logits/rejected": -2.6105871200561523,
      "logps/chosen": -275.9292907714844,
      "logps/rejected": -278.40631103515625,
      "loss": 0.5964,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.32996195554733276,
      "rewards/margins": 0.20140254497528076,
      "rewards/rejected": -0.5313645601272583,
      "step": 3960
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7661667759981213e-06,
      "logits/chosen": -2.4484341144561768,
      "logits/rejected": -2.5259575843811035,
      "logps/chosen": -239.6707763671875,
      "logps/rejected": -248.3018035888672,
      "loss": 0.6396,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.2669145166873932,
      "rewards/margins": 0.1268802285194397,
      "rewards/rejected": -0.3937947750091553,
      "step": 3970
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7548082991940137e-06,
      "logits/chosen": -2.6260974407196045,
      "logits/rejected": -2.5910682678222656,
      "logps/chosen": -371.46014404296875,
      "logps/rejected": -330.17901611328125,
      "loss": 0.5669,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2598438858985901,
      "rewards/margins": 0.3069099485874176,
      "rewards/rejected": -0.5667537450790405,
      "step": 3980
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.743444504804051e-06,
      "logits/chosen": -2.5706257820129395,
      "logits/rejected": -2.472374200820923,
      "logps/chosen": -288.25201416015625,
      "logps/rejected": -244.6339569091797,
      "loss": 0.6152,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.19544066488742828,
      "rewards/margins": 0.24673247337341309,
      "rewards/rejected": -0.44217315316200256,
      "step": 3990
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7320756299788788e-06,
      "logits/chosen": -2.502800464630127,
      "logits/rejected": -2.5665602684020996,
      "logps/chosen": -273.9783630371094,
      "logps/rejected": -309.87042236328125,
      "loss": 0.5997,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.17435002326965332,
      "rewards/margins": 0.3144958019256592,
      "rewards/rejected": -0.4888457655906677,
      "step": 4000
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7207019119751644e-06,
      "logits/chosen": -2.469245195388794,
      "logits/rejected": -2.538689613342285,
      "logps/chosen": -318.72247314453125,
      "logps/rejected": -307.48931884765625,
      "loss": 0.5864,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.19669604301452637,
      "rewards/margins": 0.3518393635749817,
      "rewards/rejected": -0.5485354661941528,
      "step": 4010
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.7093235881506474e-06,
      "logits/chosen": -2.592909812927246,
      "logits/rejected": -2.5366950035095215,
      "logps/chosen": -321.53375244140625,
      "logps/rejected": -306.61614990234375,
      "loss": 0.5879,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.30671948194503784,
      "rewards/margins": 0.25378814339637756,
      "rewards/rejected": -0.5605076551437378,
      "step": 4020
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6979408959591863e-06,
      "logits/chosen": -2.5961556434631348,
      "logits/rejected": -2.5295894145965576,
      "logps/chosen": -316.2151184082031,
      "logps/rejected": -316.78057861328125,
      "loss": 0.5385,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.21914584934711456,
      "rewards/margins": 0.5072210431098938,
      "rewards/rejected": -0.7263668179512024,
      "step": 4030
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6865540729458034e-06,
      "logits/chosen": -2.607487440109253,
      "logits/rejected": -2.5697755813598633,
      "logps/chosen": -334.29986572265625,
      "logps/rejected": -336.15380859375,
      "loss": 0.552,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.19254732131958008,
      "rewards/margins": 0.26985567808151245,
      "rewards/rejected": -0.46240299940109253,
      "step": 4040
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.675163356741726e-06,
      "logits/chosen": -2.5060925483703613,
      "logits/rejected": -2.631974697113037,
      "logps/chosen": -270.8483581542969,
      "logps/rejected": -266.566650390625,
      "loss": 0.542,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.05409348011016846,
      "rewards/margins": 0.4574395716190338,
      "rewards/rejected": -0.5115330815315247,
      "step": 4050
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6637689850594285e-06,
      "logits/chosen": -2.318814754486084,
      "logits/rejected": -2.3529763221740723,
      "logps/chosen": -351.1209716796875,
      "logps/rejected": -386.14630126953125,
      "loss": 0.5833,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.30523282289505005,
      "rewards/margins": 0.37871304154396057,
      "rewards/rejected": -0.683945894241333,
      "step": 4060
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.652371195687671e-06,
      "logits/chosen": -2.5676889419555664,
      "logits/rejected": -2.596935749053955,
      "logps/chosen": -326.96649169921875,
      "logps/rejected": -374.5132141113281,
      "loss": 0.573,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.23799224197864532,
      "rewards/margins": 0.4420638084411621,
      "rewards/rejected": -0.6800560355186462,
      "step": 4070
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.64097022648654e-06,
      "logits/chosen": -2.6802048683166504,
      "logits/rejected": -2.548435926437378,
      "logps/chosen": -307.8295593261719,
      "logps/rejected": -279.1652526855469,
      "loss": 0.6306,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.32306724786758423,
      "rewards/margins": 0.1645546853542328,
      "rewards/rejected": -0.487621933221817,
      "step": 4080
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.6295663153824774e-06,
      "logits/chosen": -2.7688498497009277,
      "logits/rejected": -2.663788080215454,
      "logps/chosen": -351.2892761230469,
      "logps/rejected": -305.7450256347656,
      "loss": 0.5699,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.27330145239830017,
      "rewards/margins": 0.373261421918869,
      "rewards/rejected": -0.646562933921814,
      "step": 4090
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.6181597003633218e-06,
      "logits/chosen": -2.6223292350769043,
      "logits/rejected": -2.60467529296875,
      "logps/chosen": -251.3638916015625,
      "logps/rejected": -261.7774963378906,
      "loss": 0.6003,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.21685147285461426,
      "rewards/margins": 0.2799449563026428,
      "rewards/rejected": -0.4967964291572571,
      "step": 4100
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.606750619473342e-06,
      "logits/chosen": -2.49078106880188,
      "logits/rejected": -2.4936556816101074,
      "logps/chosen": -250.99197387695312,
      "logps/rejected": -311.34246826171875,
      "loss": 0.582,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1258927881717682,
      "rewards/margins": 0.3767452836036682,
      "rewards/rejected": -0.502638041973114,
      "step": 4110
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.595339310808262e-06,
      "logits/chosen": -2.5719306468963623,
      "logits/rejected": -2.5432655811309814,
      "logps/chosen": -330.5278015136719,
      "logps/rejected": -290.69866943359375,
      "loss": 0.5878,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.13885828852653503,
      "rewards/margins": 0.3301781117916107,
      "rewards/rejected": -0.46903640031814575,
      "step": 4120
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5839260125103004e-06,
      "logits/chosen": -2.430999517440796,
      "logits/rejected": -2.393784761428833,
      "logps/chosen": -207.4061737060547,
      "logps/rejected": -249.16732788085938,
      "loss": 0.5656,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.24477513134479523,
      "rewards/margins": 0.2816818654537201,
      "rewards/rejected": -0.5264570116996765,
      "step": 4130
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5725109627631984e-06,
      "logits/chosen": -2.6551685333251953,
      "logits/rejected": -2.621466636657715,
      "logps/chosen": -310.831787109375,
      "logps/rejected": -279.20166015625,
      "loss": 0.6171,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.20539292693138123,
      "rewards/margins": 0.2558387517929077,
      "rewards/rejected": -0.46123170852661133,
      "step": 4140
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5610943997872443e-06,
      "logits/chosen": -2.5449509620666504,
      "logits/rejected": -2.489164352416992,
      "logps/chosen": -268.4985656738281,
      "logps/rejected": -314.49078369140625,
      "loss": 0.6003,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.2852312922477722,
      "rewards/margins": 0.25852587819099426,
      "rewards/rejected": -0.5437571406364441,
      "step": 4150
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5496765618343096e-06,
      "logits/chosen": -2.5548152923583984,
      "logits/rejected": -2.5760722160339355,
      "logps/chosen": -292.0989990234375,
      "logps/rejected": -324.4699401855469,
      "loss": 0.4914,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10181447118520737,
      "rewards/margins": 0.6143667101860046,
      "rewards/rejected": -0.7161811590194702,
      "step": 4160
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.538257687182871e-06,
      "logits/chosen": -2.674144744873047,
      "logits/rejected": -2.5367748737335205,
      "logps/chosen": -287.7279968261719,
      "logps/rejected": -282.7435607910156,
      "loss": 0.556,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2096879482269287,
      "rewards/margins": 0.3056090772151947,
      "rewards/rejected": -0.515296995639801,
      "step": 4170
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.526838014133041e-06,
      "logits/chosen": -2.5615897178649902,
      "logits/rejected": -2.5196616649627686,
      "logps/chosen": -296.09625244140625,
      "logps/rejected": -345.0995178222656,
      "loss": 0.5686,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.15199770033359528,
      "rewards/margins": 0.41160517930984497,
      "rewards/rejected": -0.5636029243469238,
      "step": 4180
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.515417781001594e-06,
      "logits/chosen": -2.6643834114074707,
      "logits/rejected": -2.660529851913452,
      "logps/chosen": -329.8970642089844,
      "logps/rejected": -324.8268127441406,
      "loss": 0.6113,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.19869431853294373,
      "rewards/margins": 0.2704014182090759,
      "rewards/rejected": -0.46909570693969727,
      "step": 4190
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.503997226116992e-06,
      "logits/chosen": -2.6225388050079346,
      "logits/rejected": -2.5563571453094482,
      "logps/chosen": -337.92828369140625,
      "logps/rejected": -320.9072265625,
      "loss": 0.5509,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.2428453415632248,
      "rewards/margins": 0.45858412981033325,
      "rewards/rejected": -0.7014294862747192,
      "step": 4200
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.4925765878144115e-06,
      "logits/chosen": -2.5668654441833496,
      "logits/rejected": -2.444279193878174,
      "logps/chosen": -268.70355224609375,
      "logps/rejected": -360.464599609375,
      "loss": 0.5916,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.24730440974235535,
      "rewards/margins": 0.29829341173171997,
      "rewards/rejected": -0.5455979108810425,
      "step": 4210
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.4811561044307727e-06,
      "logits/chosen": -2.598015308380127,
      "logits/rejected": -2.473515033721924,
      "logps/chosen": -314.43084716796875,
      "logps/rejected": -324.11981201171875,
      "loss": 0.5587,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.16435906291007996,
      "rewards/margins": 0.4245055317878723,
      "rewards/rejected": -0.5888645052909851,
      "step": 4220
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.469736014299758e-06,
      "logits/chosen": -2.6424834728240967,
      "logits/rejected": -2.5358104705810547,
      "logps/chosen": -342.76776123046875,
      "logps/rejected": -308.48419189453125,
      "loss": 0.5953,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.18019364774227142,
      "rewards/margins": 0.3455636501312256,
      "rewards/rejected": -0.5257573127746582,
      "step": 4230
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.458316555746846e-06,
      "logits/chosen": -2.5592801570892334,
      "logits/rejected": -2.50714373588562,
      "logps/chosen": -283.90289306640625,
      "logps/rejected": -320.06597900390625,
      "loss": 0.5762,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.09222878515720367,
      "rewards/margins": 0.32841163873672485,
      "rewards/rejected": -0.42064040899276733,
      "step": 4240
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.446897967084334e-06,
      "logits/chosen": -2.55417537689209,
      "logits/rejected": -2.5206234455108643,
      "logps/chosen": -304.2807312011719,
      "logps/rejected": -322.7982482910156,
      "loss": 0.6128,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.13540779054164886,
      "rewards/margins": 0.32829588651657104,
      "rewards/rejected": -0.4637036919593811,
      "step": 4250
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4354804866063684e-06,
      "logits/chosen": -2.6554081439971924,
      "logits/rejected": -2.5631651878356934,
      "logps/chosen": -298.30670166015625,
      "logps/rejected": -310.69866943359375,
      "loss": 0.5985,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.1584414690732956,
      "rewards/margins": 0.32091015577316284,
      "rewards/rejected": -0.479351669549942,
      "step": 4260
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.424064352583964e-06,
      "logits/chosen": -2.586653232574463,
      "logits/rejected": -2.4521918296813965,
      "logps/chosen": -332.4245300292969,
      "logps/rejected": -311.0771484375,
      "loss": 0.5617,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.17719905078411102,
      "rewards/margins": 0.3603389859199524,
      "rewards/rejected": -0.5375381112098694,
      "step": 4270
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4126498032600403e-06,
      "logits/chosen": -2.5424716472625732,
      "logits/rejected": -2.579455614089966,
      "logps/chosen": -248.1239471435547,
      "logps/rejected": -285.8912048339844,
      "loss": 0.5824,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.24778568744659424,
      "rewards/margins": 0.2968307137489319,
      "rewards/rejected": -0.5446164011955261,
      "step": 4280
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.401237076844445e-06,
      "logits/chosen": -2.4384758472442627,
      "logits/rejected": -2.4419591426849365,
      "logps/chosen": -309.3309631347656,
      "logps/rejected": -324.6368713378906,
      "loss": 0.6007,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.24136705696582794,
      "rewards/margins": 0.2990710437297821,
      "rewards/rejected": -0.5404380559921265,
      "step": 4290
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.38982641150898e-06,
      "logits/chosen": -2.574749231338501,
      "logits/rejected": -2.4584412574768066,
      "logps/chosen": -316.0588073730469,
      "logps/rejected": -311.87408447265625,
      "loss": 0.5542,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.18666191399097443,
      "rewards/margins": 0.36896970868110657,
      "rewards/rejected": -0.5556316375732422,
      "step": 4300
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.3784180453824414e-06,
      "logits/chosen": -2.554856061935425,
      "logits/rejected": -2.616472005844116,
      "logps/chosen": -292.73944091796875,
      "logps/rejected": -310.92938232421875,
      "loss": 0.5842,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.2169373482465744,
      "rewards/margins": 0.3188151717185974,
      "rewards/rejected": -0.535752534866333,
      "step": 4310
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.367012216545638e-06,
      "logits/chosen": -2.7256081104278564,
      "logits/rejected": -2.644930601119995,
      "logps/chosen": -323.84515380859375,
      "logps/rejected": -291.82061767578125,
      "loss": 0.5909,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.10996238887310028,
      "rewards/margins": 0.3115832209587097,
      "rewards/rejected": -0.4215455949306488,
      "step": 4320
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3556091630264294e-06,
      "logits/chosen": -2.5826218128204346,
      "logits/rejected": -2.5682921409606934,
      "logps/chosen": -319.13128662109375,
      "logps/rejected": -362.3716125488281,
      "loss": 0.544,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.2128753364086151,
      "rewards/margins": 0.38813719153404236,
      "rewards/rejected": -0.6010125875473022,
      "step": 4330
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.344209122794757e-06,
      "logits/chosen": -2.5008184909820557,
      "logits/rejected": -2.4876294136047363,
      "logps/chosen": -320.45379638671875,
      "logps/rejected": -331.88226318359375,
      "loss": 0.5932,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.2762475609779358,
      "rewards/margins": 0.3098161220550537,
      "rewards/rejected": -0.5860637426376343,
      "step": 4340
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3328123337576787e-06,
      "logits/chosen": -2.6126558780670166,
      "logits/rejected": -2.5322470664978027,
      "logps/chosen": -302.9745178222656,
      "logps/rejected": -350.0872802734375,
      "loss": 0.6005,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.24269449710845947,
      "rewards/margins": 0.32287895679473877,
      "rewards/rejected": -0.565573513507843,
      "step": 4350
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3214190337544017e-06,
      "logits/chosen": -2.5594842433929443,
      "logits/rejected": -2.561929941177368,
      "logps/chosen": -245.7132568359375,
      "logps/rejected": -274.37982177734375,
      "loss": 0.5811,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.1834389567375183,
      "rewards/margins": 0.3247998356819153,
      "rewards/rejected": -0.5082387328147888,
      "step": 4360
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.310029460551323e-06,
      "logits/chosen": -2.6649985313415527,
      "logits/rejected": -2.5254311561584473,
      "logps/chosen": -293.27239990234375,
      "logps/rejected": -306.5220642089844,
      "loss": 0.5821,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.14221002161502838,
      "rewards/margins": 0.25821369886398315,
      "rewards/rejected": -0.4004237651824951,
      "step": 4370
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.2986438518370645e-06,
      "logits/chosen": -2.477169990539551,
      "logits/rejected": -2.546952486038208,
      "logps/chosen": -269.667236328125,
      "logps/rejected": -292.5154113769531,
      "loss": 0.5845,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1302667260169983,
      "rewards/margins": 0.3379213809967041,
      "rewards/rejected": -0.4681881070137024,
      "step": 4380
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.2872624452175123e-06,
      "logits/chosen": -2.5775961875915527,
      "logits/rejected": -2.598184108734131,
      "logps/chosen": -293.9779357910156,
      "logps/rejected": -306.27972412109375,
      "loss": 0.6237,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1786755621433258,
      "rewards/margins": 0.22471539676189423,
      "rewards/rejected": -0.40339094400405884,
      "step": 4390
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2758854782108584e-06,
      "logits/chosen": -2.5487232208251953,
      "logits/rejected": -2.5394325256347656,
      "logps/chosen": -273.4031066894531,
      "logps/rejected": -308.8743896484375,
      "loss": 0.6033,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.2166653573513031,
      "rewards/margins": 0.2811164855957031,
      "rewards/rejected": -0.49778181314468384,
      "step": 4400
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2645131882426458e-06,
      "logits/chosen": -2.589221954345703,
      "logits/rejected": -2.563668727874756,
      "logps/chosen": -310.12298583984375,
      "logps/rejected": -232.107177734375,
      "loss": 0.6248,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.13823625445365906,
      "rewards/margins": 0.3391169607639313,
      "rewards/rejected": -0.47735318541526794,
      "step": 4410
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2531458126408154e-06,
      "logits/chosen": -2.642242193222046,
      "logits/rejected": -2.566376209259033,
      "logps/chosen": -288.0643615722656,
      "logps/rejected": -283.2187805175781,
      "loss": 0.5621,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.18885095417499542,
      "rewards/margins": 0.4437170922756195,
      "rewards/rejected": -0.6325680017471313,
      "step": 4420
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2417835886307452e-06,
      "logits/chosen": -2.405334711074829,
      "logits/rejected": -2.3908848762512207,
      "logps/chosen": -293.4359436035156,
      "logps/rejected": -289.71917724609375,
      "loss": 0.561,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13935764133930206,
      "rewards/margins": 0.4529578685760498,
      "rewards/rejected": -0.5923154950141907,
      "step": 4430
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2304267533303075e-06,
      "logits/chosen": -2.5580015182495117,
      "logits/rejected": -2.4934747219085693,
      "logps/chosen": -393.26812744140625,
      "logps/rejected": -347.17547607421875,
      "loss": 0.6359,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.26866209506988525,
      "rewards/margins": 0.199972465634346,
      "rewards/rejected": -0.4686345160007477,
      "step": 4440
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.219075543744918e-06,
      "logits/chosen": -2.5444393157958984,
      "logits/rejected": -2.4769625663757324,
      "logps/chosen": -371.289794921875,
      "logps/rejected": -325.30242919921875,
      "loss": 0.6073,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.19787821173667908,
      "rewards/margins": 0.2811738848686218,
      "rewards/rejected": -0.4790521562099457,
      "step": 4450
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.207730196762589e-06,
      "logits/chosen": -2.571669340133667,
      "logits/rejected": -2.502657175064087,
      "logps/chosen": -299.11480712890625,
      "logps/rejected": -299.9048767089844,
      "loss": 0.5921,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.17682352662086487,
      "rewards/margins": 0.4167998731136322,
      "rewards/rejected": -0.5936234593391418,
      "step": 4460
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.1963909491489846e-06,
      "logits/chosen": -2.601297616958618,
      "logits/rejected": -2.589207649230957,
      "logps/chosen": -237.0451202392578,
      "logps/rejected": -242.80563354492188,
      "loss": 0.5888,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.18458524346351624,
      "rewards/margins": 0.26062318682670593,
      "rewards/rejected": -0.4452084004878998,
      "step": 4470
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.185058037542486e-06,
      "logits/chosen": -2.523491382598877,
      "logits/rejected": -2.50538969039917,
      "logps/chosen": -334.57525634765625,
      "logps/rejected": -322.3340148925781,
      "loss": 0.5553,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.20119798183441162,
      "rewards/margins": 0.4498375356197357,
      "rewards/rejected": -0.6510355472564697,
      "step": 4480
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.173731698449244e-06,
      "logits/chosen": -2.4815447330474854,
      "logits/rejected": -2.400696277618408,
      "logps/chosen": -344.4715881347656,
      "logps/rejected": -311.84320068359375,
      "loss": 0.5858,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.17006592452526093,
      "rewards/margins": 0.2964300215244293,
      "rewards/rejected": -0.46649593114852905,
      "step": 4490
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1624121682382495e-06,
      "logits/chosen": -2.4832749366760254,
      "logits/rejected": -2.4193451404571533,
      "logps/chosen": -273.87847900390625,
      "logps/rejected": -334.41436767578125,
      "loss": 0.5469,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.20237097144126892,
      "rewards/margins": 0.33962470293045044,
      "rewards/rejected": -0.541995644569397,
      "step": 4500
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1510996831363993e-06,
      "logits/chosen": -2.645885944366455,
      "logits/rejected": -2.535335063934326,
      "logps/chosen": -345.0766906738281,
      "logps/rejected": -382.795654296875,
      "loss": 0.6197,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.26461997628211975,
      "rewards/margins": 0.2347315549850464,
      "rewards/rejected": -0.4993515908718109,
      "step": 4510
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.139794479223565e-06,
      "logits/chosen": -2.5707051753997803,
      "logits/rejected": -2.5140109062194824,
      "logps/chosen": -303.35137939453125,
      "logps/rejected": -307.03851318359375,
      "loss": 0.6228,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.2665776312351227,
      "rewards/margins": 0.2199481725692749,
      "rewards/rejected": -0.48652583360671997,
      "step": 4520
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.128496792427669e-06,
      "logits/chosen": -2.678926944732666,
      "logits/rejected": -2.627936840057373,
      "logps/chosen": -313.7235412597656,
      "logps/rejected": -349.46246337890625,
      "loss": 0.5948,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18646176159381866,
      "rewards/margins": 0.2873113453388214,
      "rewards/rejected": -0.4737730920314789,
      "step": 4530
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.117206858519758e-06,
      "logits/chosen": -2.635826349258423,
      "logits/rejected": -2.6556289196014404,
      "logps/chosen": -328.98748779296875,
      "logps/rejected": -319.97430419921875,
      "loss": 0.6051,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.13268058001995087,
      "rewards/margins": 0.26753780245780945,
      "rewards/rejected": -0.4002183973789215,
      "step": 4540
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.1059249131090844e-06,
      "logits/chosen": -2.457590341567993,
      "logits/rejected": -2.4332821369171143,
      "logps/chosen": -250.3773956298828,
      "logps/rejected": -240.7942352294922,
      "loss": 0.602,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.19220121204853058,
      "rewards/margins": 0.16399028897285461,
      "rewards/rejected": -0.356191486120224,
      "step": 4550
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.094651191638189e-06,
      "logits/chosen": -2.625013828277588,
      "logits/rejected": -2.550257444381714,
      "logps/chosen": -319.27130126953125,
      "logps/rejected": -304.85931396484375,
      "loss": 0.5871,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.21907266974449158,
      "rewards/margins": 0.3216410279273987,
      "rewards/rejected": -0.5407137274742126,
      "step": 4560
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0833859293779867e-06,
      "logits/chosen": -2.594148635864258,
      "logits/rejected": -2.5711936950683594,
      "logps/chosen": -306.7306823730469,
      "logps/rejected": -315.5806884765625,
      "loss": 0.5735,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.14544863998889923,
      "rewards/margins": 0.29394346475601196,
      "rewards/rejected": -0.4393920302391052,
      "step": 4570
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0721293614228568e-06,
      "logits/chosen": -2.5141139030456543,
      "logits/rejected": -2.489750385284424,
      "logps/chosen": -298.4672546386719,
      "logps/rejected": -302.94805908203125,
      "loss": 0.5933,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2640727162361145,
      "rewards/margins": 0.13427096605300903,
      "rewards/rejected": -0.39834368228912354,
      "step": 4580
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.060881722685742e-06,
      "logits/chosen": -2.4600939750671387,
      "logits/rejected": -2.37214994430542,
      "logps/chosen": -319.441650390625,
      "logps/rejected": -313.952880859375,
      "loss": 0.5497,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.1119556650519371,
      "rewards/margins": 0.43312448263168335,
      "rewards/rejected": -0.5450801849365234,
      "step": 4590
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.049643247893235e-06,
      "logits/chosen": -2.7086758613586426,
      "logits/rejected": -2.5885002613067627,
      "logps/chosen": -354.56170654296875,
      "logps/rejected": -338.67431640625,
      "loss": 0.5966,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.19458802044391632,
      "rewards/margins": 0.35264259576797485,
      "rewards/rejected": -0.54723060131073,
      "step": 4600
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0384141715806903e-06,
      "logits/chosen": -2.538681983947754,
      "logits/rejected": -2.620173931121826,
      "logps/chosen": -272.54425048828125,
      "logps/rejected": -302.4497985839844,
      "loss": 0.6154,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1432553231716156,
      "rewards/margins": 0.21607744693756104,
      "rewards/rejected": -0.35933274030685425,
      "step": 4610
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0271947280873255e-06,
      "logits/chosen": -2.521843433380127,
      "logits/rejected": -2.465176582336426,
      "logps/chosen": -216.7253875732422,
      "logps/rejected": -281.3014221191406,
      "loss": 0.6147,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.09545087814331055,
      "rewards/margins": 0.41773247718811035,
      "rewards/rejected": -0.5131834149360657,
      "step": 4620
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.0159851515513302e-06,
      "logits/chosen": -2.434443950653076,
      "logits/rejected": -2.464874744415283,
      "logps/chosen": -273.6073303222656,
      "logps/rejected": -292.5726013183594,
      "loss": 0.5812,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.2675468325614929,
      "rewards/margins": 0.30386072397232056,
      "rewards/rejected": -0.5714076161384583,
      "step": 4630
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.004785675904982e-06,
      "logits/chosen": -2.4416751861572266,
      "logits/rejected": -2.5264148712158203,
      "logps/chosen": -297.03271484375,
      "logps/rejected": -301.3223876953125,
      "loss": 0.593,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.2851831614971161,
      "rewards/margins": 0.32416126132011414,
      "rewards/rejected": -0.6093443632125854,
      "step": 4640
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.9935965348697624e-06,
      "logits/chosen": -2.518084764480591,
      "logits/rejected": -2.5146846771240234,
      "logps/chosen": -279.87762451171875,
      "logps/rejected": -327.8019714355469,
      "loss": 0.6034,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.20778203010559082,
      "rewards/margins": 0.26329201459884644,
      "rewards/rejected": -0.47107404470443726,
      "step": 4650
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.9824179619514807e-06,
      "logits/chosen": -2.521066188812256,
      "logits/rejected": -2.5882599353790283,
      "logps/chosen": -185.96694946289062,
      "logps/rejected": -223.3569793701172,
      "loss": 0.5953,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.09492836892604828,
      "rewards/margins": 0.2776508033275604,
      "rewards/rejected": -0.3725792169570923,
      "step": 4660
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.9712501904354004e-06,
      "logits/chosen": -2.577836513519287,
      "logits/rejected": -2.4661128520965576,
      "logps/chosen": -274.2032470703125,
      "logps/rejected": -246.4644012451172,
      "loss": 0.6235,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2577691972255707,
      "rewards/margins": 0.20414555072784424,
      "rewards/rejected": -0.4619147777557373,
      "step": 4670
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.960093453381369e-06,
      "logits/chosen": -2.4722354412078857,
      "logits/rejected": -2.5045969486236572,
      "logps/chosen": -254.60546875,
      "logps/rejected": -283.6824951171875,
      "loss": 0.605,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14864405989646912,
      "rewards/margins": 0.29489949345588684,
      "rewards/rejected": -0.44354352355003357,
      "step": 4680
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.948947983618962e-06,
      "logits/chosen": -2.572479248046875,
      "logits/rejected": -2.542299270629883,
      "logps/chosen": -325.290283203125,
      "logps/rejected": -301.44476318359375,
      "loss": 0.6139,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20716340839862823,
      "rewards/margins": 0.2456679344177246,
      "rewards/rejected": -0.45283135771751404,
      "step": 4690
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.937814013742611e-06,
      "logits/chosen": -2.552811622619629,
      "logits/rejected": -2.507504940032959,
      "logps/chosen": -395.3104553222656,
      "logps/rejected": -375.67572021484375,
      "loss": 0.6076,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.27121731638908386,
      "rewards/margins": 0.22323930263519287,
      "rewards/rejected": -0.49445658922195435,
      "step": 4700
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.9266917761067617e-06,
      "logits/chosen": -2.515617847442627,
      "logits/rejected": -2.471935749053955,
      "logps/chosen": -309.8771667480469,
      "logps/rejected": -280.6637878417969,
      "loss": 0.581,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.24826793372631073,
      "rewards/margins": 0.3144586980342865,
      "rewards/rejected": -0.5627266764640808,
      "step": 4710
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.915581502821017e-06,
      "logits/chosen": -2.4324276447296143,
      "logits/rejected": -2.3578710556030273,
      "logps/chosen": -307.38092041015625,
      "logps/rejected": -407.55059814453125,
      "loss": 0.5642,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2369718849658966,
      "rewards/margins": 0.39206990599632263,
      "rewards/rejected": -0.6290417909622192,
      "step": 4720
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.9044834257452997e-06,
      "logits/chosen": -2.588669538497925,
      "logits/rejected": -2.521408796310425,
      "logps/chosen": -289.35467529296875,
      "logps/rejected": -281.88519287109375,
      "loss": 0.5973,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13194029033184052,
      "rewards/margins": 0.21657545864582062,
      "rewards/rejected": -0.3485157787799835,
      "step": 4730
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.893397776485006e-06,
      "logits/chosen": -2.596656084060669,
      "logits/rejected": -2.514173984527588,
      "logps/chosen": -304.2091369628906,
      "logps/rejected": -317.41107177734375,
      "loss": 0.5589,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.1646658480167389,
      "rewards/margins": 0.4921610951423645,
      "rewards/rejected": -0.6568268537521362,
      "step": 4740
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8823247863861804e-06,
      "logits/chosen": -2.598585605621338,
      "logits/rejected": -2.5877842903137207,
      "logps/chosen": -315.4193420410156,
      "logps/rejected": -328.97784423828125,
      "loss": 0.6086,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2035294473171234,
      "rewards/margins": 0.2403070479631424,
      "rewards/rejected": -0.4438364505767822,
      "step": 4750
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8712646865306822e-06,
      "logits/chosen": -2.604114532470703,
      "logits/rejected": -2.5948071479797363,
      "logps/chosen": -368.4901123046875,
      "logps/rejected": -339.9342956542969,
      "loss": 0.581,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.22862637042999268,
      "rewards/margins": 0.3091500699520111,
      "rewards/rejected": -0.5377764701843262,
      "step": 4760
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8602177077313631e-06,
      "logits/chosen": -2.5544817447662354,
      "logits/rejected": -2.4318108558654785,
      "logps/chosen": -290.60748291015625,
      "logps/rejected": -346.29559326171875,
      "loss": 0.596,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.17622289061546326,
      "rewards/margins": 0.2590067982673645,
      "rewards/rejected": -0.43522968888282776,
      "step": 4770
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8491840805272546e-06,
      "logits/chosen": -2.5918362140655518,
      "logits/rejected": -2.5588793754577637,
      "logps/chosen": -311.6506652832031,
      "logps/rejected": -301.0184631347656,
      "loss": 0.6,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2586202025413513,
      "rewards/margins": 0.17031022906303406,
      "rewards/rejected": -0.4289304316043854,
      "step": 4780
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8381640351787516e-06,
      "logits/chosen": -2.4994304180145264,
      "logits/rejected": -2.529038429260254,
      "logps/chosen": -275.3616638183594,
      "logps/rejected": -320.9551696777344,
      "loss": 0.5659,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.10712920129299164,
      "rewards/margins": 0.3530879318714142,
      "rewards/rejected": -0.460217148065567,
      "step": 4790
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8271578016628122e-06,
      "logits/chosen": -2.506784677505493,
      "logits/rejected": -2.5386555194854736,
      "logps/chosen": -275.3206481933594,
      "logps/rejected": -279.49774169921875,
      "loss": 0.6083,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.22943325340747833,
      "rewards/margins": 0.12677840888500214,
      "rewards/rejected": -0.35621166229248047,
      "step": 4800
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8161656096681546e-06,
      "logits/chosen": -2.5047996044158936,
      "logits/rejected": -2.439415454864502,
      "logps/chosen": -243.07089233398438,
      "logps/rejected": -295.4671325683594,
      "loss": 0.5634,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.12545737624168396,
      "rewards/margins": 0.5444375276565552,
      "rewards/rejected": -0.669894814491272,
      "step": 4810
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8051876885904645e-06,
      "logits/chosen": -2.4755449295043945,
      "logits/rejected": -2.517735004425049,
      "logps/chosen": -320.04449462890625,
      "logps/rejected": -315.7063903808594,
      "loss": 0.5535,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.18441326916217804,
      "rewards/margins": 0.4230337142944336,
      "rewards/rejected": -0.6074470281600952,
      "step": 4820
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.7942242675276098e-06,
      "logits/chosen": -2.54813814163208,
      "logits/rejected": -2.5858755111694336,
      "logps/chosen": -217.2032928466797,
      "logps/rejected": -291.31768798828125,
      "loss": 0.5967,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.23552434146404266,
      "rewards/margins": 0.26469406485557556,
      "rewards/rejected": -0.500218391418457,
      "step": 4830
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.783275575274856e-06,
      "logits/chosen": -2.6607840061187744,
      "logits/rejected": -2.5804603099823,
      "logps/chosen": -345.90032958984375,
      "logps/rejected": -292.3533630371094,
      "loss": 0.5858,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.07496364414691925,
      "rewards/margins": 0.43582382798194885,
      "rewards/rejected": -0.5107874274253845,
      "step": 4840
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.7723418403200943e-06,
      "logits/chosen": -2.649951457977295,
      "logits/rejected": -2.559494733810425,
      "logps/chosen": -356.70001220703125,
      "logps/rejected": -309.3967590332031,
      "loss": 0.5836,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.2421417236328125,
      "rewards/margins": 0.306281715631485,
      "rewards/rejected": -0.5484234094619751,
      "step": 4850
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7614232908390748e-06,
      "logits/chosen": -2.5764052867889404,
      "logits/rejected": -2.4994382858276367,
      "logps/chosen": -314.18359375,
      "logps/rejected": -336.04644775390625,
      "loss": 0.6323,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.25903838872909546,
      "rewards/margins": 0.19894324243068695,
      "rewards/rejected": -0.4579816460609436,
      "step": 4860
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7505201546906398e-06,
      "logits/chosen": -2.516021490097046,
      "logits/rejected": -2.580906867980957,
      "logps/chosen": -351.5046081542969,
      "logps/rejected": -300.8806457519531,
      "loss": 0.5847,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.26838454604148865,
      "rewards/margins": 0.27423301339149475,
      "rewards/rejected": -0.5426175594329834,
      "step": 4870
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7396326594119717e-06,
      "logits/chosen": -2.498110294342041,
      "logits/rejected": -2.5909085273742676,
      "logps/chosen": -282.8095703125,
      "logps/rejected": -283.3281555175781,
      "loss": 0.5658,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.13294625282287598,
      "rewards/margins": 0.37030714750289917,
      "rewards/rejected": -0.5032534003257751,
      "step": 4880
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7287610322138449e-06,
      "logits/chosen": -2.5078094005584717,
      "logits/rejected": -2.500873565673828,
      "logps/chosen": -319.1579284667969,
      "logps/rejected": -317.5836486816406,
      "loss": 0.5706,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.205757737159729,
      "rewards/margins": 0.40956979990005493,
      "rewards/rejected": -0.6153275370597839,
      "step": 4890
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7179054999758817e-06,
      "logits/chosen": -2.6467504501342773,
      "logits/rejected": -2.6136515140533447,
      "logps/chosen": -284.30902099609375,
      "logps/rejected": -292.63519287109375,
      "loss": 0.5841,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.28274065256118774,
      "rewards/margins": 0.25528156757354736,
      "rewards/rejected": -0.5380222797393799,
      "step": 4900
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7070662892418225e-06,
      "logits/chosen": -2.596738815307617,
      "logits/rejected": -2.5798354148864746,
      "logps/chosen": -261.60443115234375,
      "logps/rejected": -283.7542419433594,
      "loss": 0.5948,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.32567527890205383,
      "rewards/margins": 0.2797202467918396,
      "rewards/rejected": -0.6053955554962158,
      "step": 4910
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.6962436262147913e-06,
      "logits/chosen": -2.4732182025909424,
      "logits/rejected": -2.435441493988037,
      "logps/chosen": -241.5220947265625,
      "logps/rejected": -286.060302734375,
      "loss": 0.6128,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.3028459846973419,
      "rewards/margins": 0.2888830304145813,
      "rewards/rejected": -0.5917289853096008,
      "step": 4920
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6854377367525814e-06,
      "logits/chosen": -2.4663939476013184,
      "logits/rejected": -2.487776279449463,
      "logps/chosen": -319.4354248046875,
      "logps/rejected": -306.67352294921875,
      "loss": 0.5711,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.15108667314052582,
      "rewards/margins": 0.3576280474662781,
      "rewards/rejected": -0.5087146759033203,
      "step": 4930
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6746488463629362e-06,
      "logits/chosen": -2.7180588245391846,
      "logits/rejected": -2.760690689086914,
      "logps/chosen": -348.2672119140625,
      "logps/rejected": -379.86920166015625,
      "loss": 0.5902,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.26233556866645813,
      "rewards/margins": 0.28122225403785706,
      "rewards/rejected": -0.5435577630996704,
      "step": 4940
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6638771801988483e-06,
      "logits/chosen": -2.604320526123047,
      "logits/rejected": -2.592014789581299,
      "logps/chosen": -343.21807861328125,
      "logps/rejected": -367.0710144042969,
      "loss": 0.5935,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.20770473778247833,
      "rewards/margins": 0.39076295495033264,
      "rewards/rejected": -0.5984677076339722,
      "step": 4950
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.653122963053857e-06,
      "logits/chosen": -2.466764450073242,
      "logits/rejected": -2.38940167427063,
      "logps/chosen": -309.2972106933594,
      "logps/rejected": -305.632080078125,
      "loss": 0.5599,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.16606129705905914,
      "rewards/margins": 0.3350328803062439,
      "rewards/rejected": -0.5010942220687866,
      "step": 4960
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6423864193573606e-06,
      "logits/chosen": -2.334670305252075,
      "logits/rejected": -2.3478381633758545,
      "logps/chosen": -261.6419677734375,
      "logps/rejected": -290.67047119140625,
      "loss": 0.5637,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16364139318466187,
      "rewards/margins": 0.3203355371952057,
      "rewards/rejected": -0.48397690057754517,
      "step": 4970
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6316677731699286e-06,
      "logits/chosen": -2.607004165649414,
      "logits/rejected": -2.4803566932678223,
      "logps/chosen": -345.6315612792969,
      "logps/rejected": -337.2192687988281,
      "loss": 0.5759,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.35969680547714233,
      "rewards/margins": 0.36782944202423096,
      "rewards/rejected": -0.7275261878967285,
      "step": 4980
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6209672481786302e-06,
      "logits/chosen": -2.478020668029785,
      "logits/rejected": -2.502074718475342,
      "logps/chosen": -288.4017333984375,
      "logps/rejected": -283.6263427734375,
      "loss": 0.5328,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.1178351417183876,
      "rewards/margins": 0.4654744267463684,
      "rewards/rejected": -0.583309531211853,
      "step": 4990
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6102850676923616e-06,
      "logits/chosen": -2.546351194381714,
      "logits/rejected": -2.5766713619232178,
      "logps/chosen": -293.3699645996094,
      "logps/rejected": -316.5007629394531,
      "loss": 0.6016,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2899512052536011,
      "rewards/margins": 0.37485018372535706,
      "rewards/rejected": -0.6648014187812805,
      "step": 5000
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5996214546371888e-06,
      "logits/chosen": -2.685042381286621,
      "logits/rejected": -2.593893051147461,
      "logps/chosen": -272.0263366699219,
      "logps/rejected": -256.42279052734375,
      "loss": 0.5752,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.16017282009124756,
      "rewards/margins": 0.3072182834148407,
      "rewards/rejected": -0.46739110350608826,
      "step": 5010
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.588976631551697e-06,
      "logits/chosen": -2.617985248565674,
      "logits/rejected": -2.576514482498169,
      "logps/chosen": -304.63189697265625,
      "logps/rejected": -308.66558837890625,
      "loss": 0.5844,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.190621480345726,
      "rewards/margins": 0.27106973528862,
      "rewards/rejected": -0.4616912007331848,
      "step": 5020
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5783508205823412e-06,
      "logits/chosen": -2.4902517795562744,
      "logits/rejected": -2.58402156829834,
      "logps/chosen": -277.86285400390625,
      "logps/rejected": -292.60687255859375,
      "loss": 0.6241,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.16955803334712982,
      "rewards/margins": 0.3012115955352783,
      "rewards/rejected": -0.47076964378356934,
      "step": 5030
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5677442434788143e-06,
      "logits/chosen": -2.6300415992736816,
      "logits/rejected": -2.523313045501709,
      "logps/chosen": -319.5474548339844,
      "logps/rejected": -330.72418212890625,
      "loss": 0.5937,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16097375750541687,
      "rewards/margins": 0.34245041012763977,
      "rewards/rejected": -0.5034242272377014,
      "step": 5040
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5571571215894181e-06,
      "logits/chosen": -2.7039828300476074,
      "logits/rejected": -2.668703556060791,
      "logps/chosen": -307.61248779296875,
      "logps/rejected": -362.8529052734375,
      "loss": 0.5754,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.25565141439437866,
      "rewards/margins": 0.298455148935318,
      "rewards/rejected": -0.554106593132019,
      "step": 5050
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5465896758564452e-06,
      "logits/chosen": -2.6605722904205322,
      "logits/rejected": -2.6176297664642334,
      "logps/chosen": -319.3802795410156,
      "logps/rejected": -349.5055236816406,
      "loss": 0.6221,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1468852460384369,
      "rewards/margins": 0.27727001905441284,
      "rewards/rejected": -0.4241552948951721,
      "step": 5060
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5360421268115653e-06,
      "logits/chosen": -2.662797212600708,
      "logits/rejected": -2.5455074310302734,
      "logps/chosen": -338.2200622558594,
      "logps/rejected": -312.7371826171875,
      "loss": 0.5569,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.25399890542030334,
      "rewards/margins": 0.3884117305278778,
      "rewards/rejected": -0.6424106359481812,
      "step": 5070
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5255146945712267e-06,
      "logits/chosen": -2.5705513954162598,
      "logits/rejected": -2.4772160053253174,
      "logps/chosen": -326.19989013671875,
      "logps/rejected": -275.66412353515625,
      "loss": 0.5953,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.21252810955047607,
      "rewards/margins": 0.3057176470756531,
      "rewards/rejected": -0.5182457566261292,
      "step": 5080
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.5150075988320594e-06,
      "logits/chosen": -2.485109806060791,
      "logits/rejected": -2.40926194190979,
      "logps/chosen": -261.7632141113281,
      "logps/rejected": -284.9822692871094,
      "loss": 0.5824,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.23927578330039978,
      "rewards/margins": 0.3335501253604889,
      "rewards/rejected": -0.5728258490562439,
      "step": 5090
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.5045210588662929e-06,
      "logits/chosen": -2.541929244995117,
      "logits/rejected": -2.529249668121338,
      "logps/chosen": -272.0650634765625,
      "logps/rejected": -292.97467041015625,
      "loss": 0.5757,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11239515244960785,
      "rewards/margins": 0.4191061854362488,
      "rewards/rejected": -0.5315013527870178,
      "step": 5100
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4940552935171781e-06,
      "logits/chosen": -2.7147622108459473,
      "logits/rejected": -2.5909676551818848,
      "logps/chosen": -323.33221435546875,
      "logps/rejected": -350.9828796386719,
      "loss": 0.5739,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.25958338379859924,
      "rewards/margins": 0.29021376371383667,
      "rewards/rejected": -0.5497971773147583,
      "step": 5110
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.483610521194419e-06,
      "logits/chosen": -2.485908269882202,
      "logits/rejected": -2.4038195610046387,
      "logps/chosen": -340.5307312011719,
      "logps/rejected": -317.41571044921875,
      "loss": 0.5661,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.1522163301706314,
      "rewards/margins": 0.5749739408493042,
      "rewards/rejected": -0.7271903157234192,
      "step": 5120
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4731869598696226e-06,
      "logits/chosen": -2.465968608856201,
      "logits/rejected": -2.5231192111968994,
      "logps/chosen": -259.185546875,
      "logps/rejected": -318.861572265625,
      "loss": 0.5764,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.21996121108531952,
      "rewards/margins": 0.2890729308128357,
      "rewards/rejected": -0.5090342164039612,
      "step": 5130
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4627848270717387e-06,
      "logits/chosen": -2.5720362663269043,
      "logits/rejected": -2.435134172439575,
      "logps/chosen": -305.77734375,
      "logps/rejected": -305.54583740234375,
      "loss": 0.5563,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.2069060057401657,
      "rewards/margins": 0.3971417546272278,
      "rewards/rejected": -0.6040477752685547,
      "step": 5140
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4524043398825277e-06,
      "logits/chosen": -2.3851141929626465,
      "logits/rejected": -2.382739305496216,
      "logps/chosen": -343.54998779296875,
      "logps/rejected": -292.6429138183594,
      "loss": 0.6035,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.19103741645812988,
      "rewards/margins": 0.27970466017723083,
      "rewards/rejected": -0.4707420766353607,
      "step": 5150
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4420457149320299e-06,
      "logits/chosen": -2.481121778488159,
      "logits/rejected": -2.4870851039886475,
      "logps/chosen": -281.44415283203125,
      "logps/rejected": -332.45391845703125,
      "loss": 0.6341,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.2218354046344757,
      "rewards/margins": 0.15597988665103912,
      "rewards/rejected": -0.377815306186676,
      "step": 5160
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.431709168394042e-06,
      "logits/chosen": -2.6676456928253174,
      "logits/rejected": -2.6511683464050293,
      "logps/chosen": -373.96893310546875,
      "logps/rejected": -315.47015380859375,
      "loss": 0.5551,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3234941363334656,
      "rewards/margins": 0.19331450760364532,
      "rewards/rejected": -0.5168086290359497,
      "step": 5170
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4213949159816059e-06,
      "logits/chosen": -2.498394727706909,
      "logits/rejected": -2.3926968574523926,
      "logps/chosen": -276.31793212890625,
      "logps/rejected": -293.73260498046875,
      "loss": 0.5533,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19067999720573425,
      "rewards/margins": 0.4055241048336029,
      "rewards/rejected": -0.5962041616439819,
      "step": 5180
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4111031729425103e-06,
      "logits/chosen": -2.5006566047668457,
      "logits/rejected": -2.4794576168060303,
      "logps/chosen": -324.97137451171875,
      "logps/rejected": -303.35662841796875,
      "loss": 0.5635,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.24338562786579132,
      "rewards/margins": 0.33955326676368713,
      "rewards/rejected": -0.5829388499259949,
      "step": 5190
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4008341540547965e-06,
      "logits/chosen": -2.4405434131622314,
      "logits/rejected": -2.5150179862976074,
      "logps/chosen": -291.26898193359375,
      "logps/rejected": -311.96185302734375,
      "loss": 0.5954,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.16561874747276306,
      "rewards/margins": 0.2473074197769165,
      "rewards/rejected": -0.41292619705200195,
      "step": 5200
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.3905880736222737e-06,
      "logits/chosen": -2.4981284141540527,
      "logits/rejected": -2.52009916305542,
      "logps/chosen": -255.4973602294922,
      "logps/rejected": -305.76678466796875,
      "loss": 0.5787,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.20114591717720032,
      "rewards/margins": 0.2925758957862854,
      "rewards/rejected": -0.4937218725681305,
      "step": 5210
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.3803651454700531e-06,
      "logits/chosen": -2.5396883487701416,
      "logits/rejected": -2.4904818534851074,
      "logps/chosen": -298.5966491699219,
      "logps/rejected": -282.314208984375,
      "loss": 0.5623,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.19929958879947662,
      "rewards/margins": 0.3365853428840637,
      "rewards/rejected": -0.5358849763870239,
      "step": 5220
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.3701655829400773e-06,
      "logits/chosen": -2.695084571838379,
      "logits/rejected": -2.5825228691101074,
      "logps/chosen": -399.9344787597656,
      "logps/rejected": -328.28179931640625,
      "loss": 0.5791,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.23696422576904297,
      "rewards/margins": 0.32420456409454346,
      "rewards/rejected": -0.5611687898635864,
      "step": 5230
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3599895988866756e-06,
      "logits/chosen": -2.622859477996826,
      "logits/rejected": -2.580024242401123,
      "logps/chosen": -272.3352355957031,
      "logps/rejected": -299.87139892578125,
      "loss": 0.6425,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.22998254001140594,
      "rewards/margins": 0.18282505869865417,
      "rewards/rejected": -0.4128076434135437,
      "step": 5240
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3498374056721198e-06,
      "logits/chosen": -2.707981824874878,
      "logits/rejected": -2.7110109329223633,
      "logps/chosen": -274.9527587890625,
      "logps/rejected": -382.5824279785156,
      "loss": 0.5442,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1651863157749176,
      "rewards/margins": 0.3817494511604309,
      "rewards/rejected": -0.5469357371330261,
      "step": 5250
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3397092151621883e-06,
      "logits/chosen": -2.554539442062378,
      "logits/rejected": -2.482882022857666,
      "logps/chosen": -286.2304992675781,
      "logps/rejected": -266.2547607421875,
      "loss": 0.6205,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.26898160576820374,
      "rewards/margins": 0.21374158561229706,
      "rewards/rejected": -0.4827231764793396,
      "step": 5260
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3296052387217484e-06,
      "logits/chosen": -2.45316481590271,
      "logits/rejected": -2.5683462619781494,
      "logps/chosen": -238.8728790283203,
      "logps/rejected": -235.2001495361328,
      "loss": 0.5939,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.16806179285049438,
      "rewards/margins": 0.333972692489624,
      "rewards/rejected": -0.5020344853401184,
      "step": 5270
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3195256872103476e-06,
      "logits/chosen": -2.648393392562866,
      "logits/rejected": -2.59840726852417,
      "logps/chosen": -324.327880859375,
      "logps/rejected": -378.9349670410156,
      "loss": 0.6047,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1936226338148117,
      "rewards/margins": 0.20749232172966003,
      "rewards/rejected": -0.40111494064331055,
      "step": 5280
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3094707709778068e-06,
      "logits/chosen": -2.4787323474884033,
      "logits/rejected": -2.448561906814575,
      "logps/chosen": -247.78439331054688,
      "logps/rejected": -260.88128662109375,
      "loss": 0.5579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1471039354801178,
      "rewards/margins": 0.37695735692977905,
      "rewards/rejected": -0.5240613222122192,
      "step": 5290
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.2994406998598364e-06,
      "logits/chosen": -2.550649642944336,
      "logits/rejected": -2.446993350982666,
      "logps/chosen": -208.6945343017578,
      "logps/rejected": -247.4278564453125,
      "loss": 0.5701,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17239803075790405,
      "rewards/margins": 0.4012460708618164,
      "rewards/rejected": -0.5736441016197205,
      "step": 5300
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.2894356831736558e-06,
      "logits/chosen": -2.5386788845062256,
      "logits/rejected": -2.5003950595855713,
      "logps/chosen": -270.95428466796875,
      "logps/rejected": -281.563720703125,
      "loss": 0.5692,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.2243034839630127,
      "rewards/margins": 0.36400306224823,
      "rewards/rejected": -0.5883064866065979,
      "step": 5310
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2794559297136203e-06,
      "logits/chosen": -2.4010531902313232,
      "logits/rejected": -2.4677743911743164,
      "logps/chosen": -268.36102294921875,
      "logps/rejected": -312.61871337890625,
      "loss": 0.6012,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11416293680667877,
      "rewards/margins": 0.3250843286514282,
      "rewards/rejected": -0.4392472803592682,
      "step": 5320
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2695016477468724e-06,
      "logits/chosen": -2.6123104095458984,
      "logits/rejected": -2.658905506134033,
      "logps/chosen": -315.5183410644531,
      "logps/rejected": -280.52191162109375,
      "loss": 0.645,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1765400469303131,
      "rewards/margins": 0.17122364044189453,
      "rewards/rejected": -0.34776371717453003,
      "step": 5330
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2595730450089874e-06,
      "logits/chosen": -2.68192720413208,
      "logits/rejected": -2.6289165019989014,
      "logps/chosen": -316.01898193359375,
      "logps/rejected": -322.196533203125,
      "loss": 0.5852,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16447719931602478,
      "rewards/margins": 0.36463385820388794,
      "rewards/rejected": -0.5291110873222351,
      "step": 5340
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2496703286996433e-06,
      "logits/chosen": -2.685610055923462,
      "logits/rejected": -2.589296340942383,
      "logps/chosen": -365.1530456542969,
      "logps/rejected": -335.67156982421875,
      "loss": 0.6027,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.22830632328987122,
      "rewards/margins": 0.3892638087272644,
      "rewards/rejected": -0.6175701022148132,
      "step": 5350
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2397937054782961e-06,
      "logits/chosen": -2.555673122406006,
      "logits/rejected": -2.625791549682617,
      "logps/chosen": -346.9773864746094,
      "logps/rejected": -334.68756103515625,
      "loss": 0.5705,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.23506096005439758,
      "rewards/margins": 0.3681856393814087,
      "rewards/rejected": -0.6032465696334839,
      "step": 5360
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2299433814598635e-06,
      "logits/chosen": -2.5669150352478027,
      "logits/rejected": -2.6253628730773926,
      "logps/chosen": -268.91680908203125,
      "logps/rejected": -296.8352355957031,
      "loss": 0.5799,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.10326936095952988,
      "rewards/margins": 0.25865739583969116,
      "rewards/rejected": -0.36192673444747925,
      "step": 5370
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2201195622104265e-06,
      "logits/chosen": -2.5529732704162598,
      "logits/rejected": -2.570631504058838,
      "logps/chosen": -280.7418212890625,
      "logps/rejected": -277.921630859375,
      "loss": 0.5804,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.21117191016674042,
      "rewards/margins": 0.3797195553779602,
      "rewards/rejected": -0.5908914804458618,
      "step": 5380
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.2103224527429417e-06,
      "logits/chosen": -2.5260720252990723,
      "logits/rejected": -2.5511975288391113,
      "logps/chosen": -303.44635009765625,
      "logps/rejected": -246.2676239013672,
      "loss": 0.6542,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.12673085927963257,
      "rewards/margins": 0.29531019926071167,
      "rewards/rejected": -0.42204099893569946,
      "step": 5390
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.2005522575129559e-06,
      "logits/chosen": -2.6706199645996094,
      "logits/rejected": -2.5881125926971436,
      "logps/chosen": -391.4795837402344,
      "logps/rejected": -295.01031494140625,
      "loss": 0.655,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2009950876235962,
      "rewards/margins": 0.23872563242912292,
      "rewards/rejected": -0.4397208094596863,
      "step": 5400
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1908091804143469e-06,
      "logits/chosen": -2.438290596008301,
      "logits/rejected": -2.404784679412842,
      "logps/chosen": -300.5682373046875,
      "logps/rejected": -316.4365234375,
      "loss": 0.6282,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.2771514058113098,
      "rewards/margins": 0.25950735807418823,
      "rewards/rejected": -0.536658763885498,
      "step": 5410
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1810934247750649e-06,
      "logits/chosen": -2.5039753913879395,
      "logits/rejected": -2.472374200820923,
      "logps/chosen": -355.4488220214844,
      "logps/rejected": -306.5274353027344,
      "loss": 0.5836,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.11722154915332794,
      "rewards/margins": 0.4215324819087982,
      "rewards/rejected": -0.5387539863586426,
      "step": 5420
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1714051933528881e-06,
      "logits/chosen": -2.6620097160339355,
      "logits/rejected": -2.5606446266174316,
      "logps/chosen": -340.6926574707031,
      "logps/rejected": -322.7391052246094,
      "loss": 0.5843,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.18550732731819153,
      "rewards/margins": 0.3209478259086609,
      "rewards/rejected": -0.50645512342453,
      "step": 5430
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.161744688331192e-06,
      "logits/chosen": -2.569016695022583,
      "logits/rejected": -2.427980422973633,
      "logps/chosen": -297.58782958984375,
      "logps/rejected": -343.2862854003906,
      "loss": 0.622,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.30308234691619873,
      "rewards/margins": 0.1463664025068283,
      "rewards/rejected": -0.44944876432418823,
      "step": 5440
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.152112111314733e-06,
      "logits/chosen": -2.584120512008667,
      "logits/rejected": -2.5221121311187744,
      "logps/chosen": -292.50823974609375,
      "logps/rejected": -288.62921142578125,
      "loss": 0.6346,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.20627093315124512,
      "rewards/margins": 0.20762047171592712,
      "rewards/rejected": -0.413891464471817,
      "step": 5450
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.142507663325439e-06,
      "logits/chosen": -2.5839767456054688,
      "logits/rejected": -2.5707592964172363,
      "logps/chosen": -337.36578369140625,
      "logps/rejected": -340.32684326171875,
      "loss": 0.5433,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1801554262638092,
      "rewards/margins": 0.41654491424560547,
      "rewards/rejected": -0.5967002511024475,
      "step": 5460
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.132931544798211e-06,
      "logits/chosen": -2.521233558654785,
      "logits/rejected": -2.5143444538116455,
      "logps/chosen": -303.11968994140625,
      "logps/rejected": -321.09661865234375,
      "loss": 0.5342,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.12275087833404541,
      "rewards/margins": 0.49955272674560547,
      "rewards/rejected": -0.6223036050796509,
      "step": 5470
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.1233839555767482e-06,
      "logits/chosen": -2.4385015964508057,
      "logits/rejected": -2.3833563327789307,
      "logps/chosen": -290.13604736328125,
      "logps/rejected": -249.4829559326172,
      "loss": 0.5508,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.15207554399967194,
      "rewards/margins": 0.4083792567253113,
      "rewards/rejected": -0.5604548454284668,
      "step": 5480
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.1138650949093668e-06,
      "logits/chosen": -2.5767691135406494,
      "logits/rejected": -2.4980454444885254,
      "logps/chosen": -255.419189453125,
      "logps/rejected": -224.4200439453125,
      "loss": 0.6602,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.31322962045669556,
      "rewards/margins": 0.19506050646305084,
      "rewards/rejected": -0.5082900524139404,
      "step": 5490
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.1043751614448543e-06,
      "logits/chosen": -2.431666135787964,
      "logits/rejected": -2.3092381954193115,
      "logps/chosen": -237.71817016601562,
      "logps/rejected": -221.69140625,
      "loss": 0.5965,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1798931211233139,
      "rewards/margins": 0.2392689287662506,
      "rewards/rejected": -0.4191620349884033,
      "step": 5500
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.0949143532283107e-06,
      "logits/chosen": -2.580761432647705,
      "logits/rejected": -2.549431800842285,
      "logps/chosen": -321.2472839355469,
      "logps/rejected": -338.0050048828125,
      "loss": 0.5863,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.2106788456439972,
      "rewards/margins": 0.34467029571533203,
      "rewards/rejected": -0.5553491711616516,
      "step": 5510
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.0854828676970275e-06,
      "logits/chosen": -2.4911673069000244,
      "logits/rejected": -2.5411815643310547,
      "logps/chosen": -248.7816925048828,
      "logps/rejected": -274.0561218261719,
      "loss": 0.5994,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.13380002975463867,
      "rewards/margins": 0.3159855604171753,
      "rewards/rejected": -0.4497855305671692,
      "step": 5520
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.076080901676361e-06,
      "logits/chosen": -2.6060454845428467,
      "logits/rejected": -2.626995325088501,
      "logps/chosen": -280.73358154296875,
      "logps/rejected": -289.40032958984375,
      "loss": 0.6006,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2465798407793045,
      "rewards/margins": 0.2739890515804291,
      "rewards/rejected": -0.5205689072608948,
      "step": 5530
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.0667086513756234e-06,
      "logits/chosen": -2.6200010776519775,
      "logits/rejected": -2.5084142684936523,
      "logps/chosen": -293.21099853515625,
      "logps/rejected": -301.6272888183594,
      "loss": 0.5946,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.18248774111270905,
      "rewards/margins": 0.34508463740348816,
      "rewards/rejected": -0.5275723338127136,
      "step": 5540
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0573663123839912e-06,
      "logits/chosen": -2.5958545207977295,
      "logits/rejected": -2.530036449432373,
      "logps/chosen": -302.2039489746094,
      "logps/rejected": -267.8409118652344,
      "loss": 0.5642,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1686379760503769,
      "rewards/margins": 0.46634167432785034,
      "rewards/rejected": -0.6349795460700989,
      "step": 5550
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0480540796664251e-06,
      "logits/chosen": -2.4992308616638184,
      "logits/rejected": -2.491286039352417,
      "logps/chosen": -319.95916748046875,
      "logps/rejected": -346.5733337402344,
      "loss": 0.5925,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.28914397954940796,
      "rewards/margins": 0.20945701003074646,
      "rewards/rejected": -0.4986010193824768,
      "step": 5560
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0387721475595978e-06,
      "logits/chosen": -2.5275931358337402,
      "logits/rejected": -2.4925732612609863,
      "logps/chosen": -221.79342651367188,
      "logps/rejected": -234.3103485107422,
      "loss": 0.6016,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.24439530074596405,
      "rewards/margins": 0.23605410754680634,
      "rewards/rejected": -0.480449378490448,
      "step": 5570
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0295207097678378e-06,
      "logits/chosen": -2.433387279510498,
      "logits/rejected": -2.4333786964416504,
      "logps/chosen": -226.1207275390625,
      "logps/rejected": -324.7813415527344,
      "loss": 0.5561,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.16782227158546448,
      "rewards/margins": 0.3883552551269531,
      "rewards/rejected": -0.5561774969100952,
      "step": 5580
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0202999593590924e-06,
      "logits/chosen": -2.5502381324768066,
      "logits/rejected": -2.5282599925994873,
      "logps/chosen": -267.7656555175781,
      "logps/rejected": -308.3643493652344,
      "loss": 0.5295,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.12993329763412476,
      "rewards/margins": 0.4945352077484131,
      "rewards/rejected": -0.6244684457778931,
      "step": 5590
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.011110088760891e-06,
      "logits/chosen": -2.455394744873047,
      "logits/rejected": -2.5353024005889893,
      "logps/chosen": -244.3286590576172,
      "logps/rejected": -283.96429443359375,
      "loss": 0.5477,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.20858506858348846,
      "rewards/margins": 0.46013203263282776,
      "rewards/rejected": -0.6687171459197998,
      "step": 5600
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0019512897563347e-06,
      "logits/chosen": -2.6215782165527344,
      "logits/rejected": -2.4959828853607178,
      "logps/chosen": -300.98876953125,
      "logps/rejected": -297.37158203125,
      "loss": 0.6051,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.14473457634449005,
      "rewards/margins": 0.34757936000823975,
      "rewards/rejected": -0.492313951253891,
      "step": 5610
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.928237534800935e-07,
      "logits/chosen": -2.6514501571655273,
      "logits/rejected": -2.6429600715637207,
      "logps/chosen": -281.0267028808594,
      "logps/rejected": -294.16644287109375,
      "loss": 0.5812,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.11892740428447723,
      "rewards/margins": 0.35566264390945435,
      "rewards/rejected": -0.4745900630950928,
      "step": 5620
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.837276704144174e-07,
      "logits/chosen": -2.627713918685913,
      "logits/rejected": -2.588146686553955,
      "logps/chosen": -265.66455078125,
      "logps/rejected": -259.3338928222656,
      "loss": 0.5904,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1974610537290573,
      "rewards/margins": 0.2754455506801605,
      "rewards/rejected": -0.47290658950805664,
      "step": 5630
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.746632303851569e-07,
      "logits/chosen": -2.6073737144470215,
      "logits/rejected": -2.5223560333251953,
      "logps/chosen": -310.0323791503906,
      "logps/rejected": -273.57391357421875,
      "loss": 0.6077,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2162187099456787,
      "rewards/margins": 0.34184861183166504,
      "rewards/rejected": -0.5580673217773438,
      "step": 5640
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.65630622557809e-07,
      "logits/chosen": -2.5930376052856445,
      "logits/rejected": -2.4731199741363525,
      "logps/chosen": -358.55145263671875,
      "logps/rejected": -344.0464172363281,
      "loss": 0.6056,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.2608993649482727,
      "rewards/margins": 0.24014616012573242,
      "rewards/rejected": -0.5010455846786499,
      "step": 5650
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.56630035433561e-07,
      "logits/chosen": -2.6305251121520996,
      "logits/rejected": -2.5761353969573975,
      "logps/chosen": -330.01239013671875,
      "logps/rejected": -283.33489990234375,
      "loss": 0.5968,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1723465621471405,
      "rewards/margins": 0.23930080235004425,
      "rewards/rejected": -0.41164731979370117,
      "step": 5660
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.476616568453659e-07,
      "logits/chosen": -2.5701346397399902,
      "logits/rejected": -2.505082368850708,
      "logps/chosen": -317.55303955078125,
      "logps/rejected": -285.5252990722656,
      "loss": 0.537,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2099696844816208,
      "rewards/margins": 0.4133245348930359,
      "rewards/rejected": -0.6232942342758179,
      "step": 5670
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.387256739540162e-07,
      "logits/chosen": -2.589440107345581,
      "logits/rejected": -2.581523895263672,
      "logps/chosen": -336.957763671875,
      "logps/rejected": -325.838623046875,
      "loss": 0.5688,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.15233464539051056,
      "rewards/margins": 0.35094717144966125,
      "rewards/rejected": -0.5032817721366882,
      "step": 5680
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.298222732442377e-07,
      "logits/chosen": -2.658050537109375,
      "logits/rejected": -2.5099217891693115,
      "logps/chosen": -319.69879150390625,
      "logps/rejected": -303.26031494140625,
      "loss": 0.5772,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.15199032425880432,
      "rewards/margins": 0.4483245313167572,
      "rewards/rejected": -0.600314736366272,
      "step": 5690
    },
    {
      "epoch": 0.75,
      "learning_rate": 9.20951640520803e-07,
      "logits/chosen": -2.556957244873047,
      "logits/rejected": -2.5528512001037598,
      "logps/chosen": -270.5416259765625,
      "logps/rejected": -284.24444580078125,
      "loss": 0.578,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.15435631573200226,
      "rewards/margins": 0.3883664011955261,
      "rewards/rejected": -0.542722761631012,
      "step": 5700
    },
    {
      "epoch": 0.75,
      "learning_rate": 9.121139609046484e-07,
      "logits/chosen": -2.516080379486084,
      "logits/rejected": -2.489114999771118,
      "logps/chosen": -303.9247741699219,
      "logps/rejected": -362.7187805175781,
      "loss": 0.6415,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.21308767795562744,
      "rewards/margins": 0.3083495497703552,
      "rewards/rejected": -0.5214372873306274,
      "step": 5710
    },
    {
      "epoch": 0.75,
      "learning_rate": 9.033094188290121e-07,
      "logits/chosen": -2.392362356185913,
      "logits/rejected": -2.425485134124756,
      "logps/chosen": -325.4200439453125,
      "logps/rejected": -310.832275390625,
      "loss": 0.6042,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2987435460090637,
      "rewards/margins": 0.34238141775131226,
      "rewards/rejected": -0.641124963760376,
      "step": 5720
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.945381980355889e-07,
      "logits/chosen": -2.576991558074951,
      "logits/rejected": -2.5154693126678467,
      "logps/chosen": -310.90142822265625,
      "logps/rejected": -355.0868225097656,
      "loss": 0.6146,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.24633660912513733,
      "rewards/margins": 0.30161046981811523,
      "rewards/rejected": -0.5479470491409302,
      "step": 5730
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.858004815706919e-07,
      "logits/chosen": -2.481250762939453,
      "logits/rejected": -2.510680675506592,
      "logps/chosen": -277.41778564453125,
      "logps/rejected": -293.6460266113281,
      "loss": 0.5435,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.20409002900123596,
      "rewards/margins": 0.4168508052825928,
      "rewards/rejected": -0.6209408044815063,
      "step": 5740
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.77096451781432e-07,
      "logits/chosen": -2.537360668182373,
      "logits/rejected": -2.456080913543701,
      "logps/chosen": -254.56448364257812,
      "logps/rejected": -274.6441345214844,
      "loss": 0.6138,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2944655120372772,
      "rewards/margins": 0.24884769320487976,
      "rewards/rejected": -0.543313205242157,
      "step": 5750
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.684262903119165e-07,
      "logits/chosen": -2.5056889057159424,
      "logits/rejected": -2.6055495738983154,
      "logps/chosen": -276.7388610839844,
      "logps/rejected": -314.30108642578125,
      "loss": 0.6008,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.23513415455818176,
      "rewards/margins": 0.2173004448413849,
      "rewards/rejected": -0.45243462920188904,
      "step": 5760
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.597901780994525e-07,
      "logits/chosen": -2.653364658355713,
      "logits/rejected": -2.60335111618042,
      "logps/chosen": -279.13238525390625,
      "logps/rejected": -248.5230255126953,
      "loss": 0.626,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.2263440191745758,
      "rewards/margins": 0.20855498313903809,
      "rewards/rejected": -0.4348990321159363,
      "step": 5770
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.511882953707773e-07,
      "logits/chosen": -2.5802361965179443,
      "logits/rejected": -2.5868887901306152,
      "logps/chosen": -342.547119140625,
      "logps/rejected": -336.5263671875,
      "loss": 0.596,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.21341674029827118,
      "rewards/margins": 0.34919142723083496,
      "rewards/rejected": -0.5626082420349121,
      "step": 5780
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.426208216382944e-07,
      "logits/chosen": -2.4884839057922363,
      "logits/rejected": -2.538118362426758,
      "logps/chosen": -254.6638946533203,
      "logps/rejected": -303.5052490234375,
      "loss": 0.5528,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.20429837703704834,
      "rewards/margins": 0.37576302886009216,
      "rewards/rejected": -0.5800613760948181,
      "step": 5790
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.340879356963245e-07,
      "logits/chosen": -2.4506642818450928,
      "logits/rejected": -2.382664203643799,
      "logps/chosen": -295.90997314453125,
      "logps/rejected": -309.56756591796875,
      "loss": 0.5646,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.23028168082237244,
      "rewards/margins": 0.252223402261734,
      "rewards/rejected": -0.48250502347946167,
      "step": 5800
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.255898156173777e-07,
      "logits/chosen": -2.572566270828247,
      "logits/rejected": -2.5592901706695557,
      "logps/chosen": -250.8806610107422,
      "logps/rejected": -271.90069580078125,
      "loss": 0.5678,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.18372826278209686,
      "rewards/margins": 0.3871956765651703,
      "rewards/rejected": -0.570923924446106,
      "step": 5810
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.171266387484389e-07,
      "logits/chosen": -2.6025376319885254,
      "logits/rejected": -2.593602418899536,
      "logps/chosen": -296.9878845214844,
      "logps/rejected": -274.2995300292969,
      "loss": 0.6177,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.19963833689689636,
      "rewards/margins": 0.2611692547798157,
      "rewards/rejected": -0.46080756187438965,
      "step": 5820
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.086985817072604e-07,
      "logits/chosen": -2.5865213871002197,
      "logits/rejected": -2.521101713180542,
      "logps/chosen": -290.48248291015625,
      "logps/rejected": -290.9474182128906,
      "loss": 0.5939,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.16447794437408447,
      "rewards/margins": 0.32579711079597473,
      "rewards/rejected": -0.4902750849723816,
      "step": 5830
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.003058203786835e-07,
      "logits/chosen": -2.486022472381592,
      "logits/rejected": -2.4313018321990967,
      "logps/chosen": -333.7433776855469,
      "logps/rejected": -302.3613586425781,
      "loss": 0.622,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.27535480260849,
      "rewards/margins": 0.2524801790714264,
      "rewards/rejected": -0.527834951877594,
      "step": 5840
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.91948529910963e-07,
      "logits/chosen": -2.485321521759033,
      "logits/rejected": -2.4735782146453857,
      "logps/chosen": -293.6622009277344,
      "logps/rejected": -316.9804992675781,
      "loss": 0.5743,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.19359636306762695,
      "rewards/margins": 0.4723263382911682,
      "rewards/rejected": -0.6659227013587952,
      "step": 5850
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.836268847121126e-07,
      "logits/chosen": -2.5226197242736816,
      "logits/rejected": -2.5177345275878906,
      "logps/chosen": -333.4381103515625,
      "logps/rejected": -344.66888427734375,
      "loss": 0.6019,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.2280873954296112,
      "rewards/margins": 0.2676352858543396,
      "rewards/rejected": -0.4957226812839508,
      "step": 5860
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.753410584462681e-07,
      "logits/chosen": -2.4178872108459473,
      "logits/rejected": -2.3296265602111816,
      "logps/chosen": -314.51025390625,
      "logps/rejected": -292.57550048828125,
      "loss": 0.5726,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.19157357513904572,
      "rewards/margins": 0.46646785736083984,
      "rewards/rejected": -0.6580413579940796,
      "step": 5870
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.670912240300596e-07,
      "logits/chosen": -2.529723644256592,
      "logits/rejected": -2.554422616958618,
      "logps/chosen": -318.37469482421875,
      "logps/rejected": -314.1541442871094,
      "loss": 0.5606,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.09865571558475494,
      "rewards/margins": 0.3737652599811554,
      "rewards/rejected": -0.47242099046707153,
      "step": 5880
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.588775536290035e-07,
      "logits/chosen": -2.5802218914031982,
      "logits/rejected": -2.4884378910064697,
      "logps/chosen": -291.59027099609375,
      "logps/rejected": -292.0467529296875,
      "loss": 0.5638,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.22754530608654022,
      "rewards/margins": 0.4772590100765228,
      "rewards/rejected": -0.7048042416572571,
      "step": 5890
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.507002186539147e-07,
      "logits/chosen": -2.395439624786377,
      "logits/rejected": -2.5001044273376465,
      "logps/chosen": -360.6567077636719,
      "logps/rejected": -345.16259765625,
      "loss": 0.6263,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.28635045886039734,
      "rewards/margins": 0.23526303470134735,
      "rewards/rejected": -0.5216134786605835,
      "step": 5900
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.425593897573216e-07,
      "logits/chosen": -2.476325511932373,
      "logits/rejected": -2.4704856872558594,
      "logps/chosen": -365.66754150390625,
      "logps/rejected": -364.56768798828125,
      "loss": 0.595,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.23280945420265198,
      "rewards/margins": 0.36780303716659546,
      "rewards/rejected": -0.6006125211715698,
      "step": 5910
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.344552368299088e-07,
      "logits/chosen": -2.561272621154785,
      "logits/rejected": -2.5794219970703125,
      "logps/chosen": -313.61297607421875,
      "logps/rejected": -329.70770263671875,
      "loss": 0.5875,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.26516374945640564,
      "rewards/margins": 0.23184308409690857,
      "rewards/rejected": -0.4970068037509918,
      "step": 5920
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.26387928996973e-07,
      "logits/chosen": -2.4242825508117676,
      "logits/rejected": -2.4046523571014404,
      "logps/chosen": -290.2831115722656,
      "logps/rejected": -275.2210998535156,
      "loss": 0.5484,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.41711360216140747,
      "rewards/margins": 0.27641454339027405,
      "rewards/rejected": -0.6935281157493591,
      "step": 5930
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.183576346148899e-07,
      "logits/chosen": -2.6386637687683105,
      "logits/rejected": -2.579282283782959,
      "logps/chosen": -284.7298278808594,
      "logps/rejected": -318.6064758300781,
      "loss": 0.5745,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.2168664038181305,
      "rewards/margins": 0.37781259417533875,
      "rewards/rejected": -0.594679057598114,
      "step": 5940
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.103645212676044e-07,
      "logits/chosen": -2.486691951751709,
      "logits/rejected": -2.5294435024261475,
      "logps/chosen": -260.233154296875,
      "logps/rejected": -269.3633728027344,
      "loss": 0.5684,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.12218284606933594,
      "rewards/margins": 0.41997766494750977,
      "rewards/rejected": -0.5421606302261353,
      "step": 5950
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.024087557631318e-07,
      "logits/chosen": -2.462345838546753,
      "logits/rejected": -2.4638350009918213,
      "logps/chosen": -264.4425048828125,
      "logps/rejected": -300.456787109375,
      "loss": 0.6159,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3224496841430664,
      "rewards/margins": 0.18295326828956604,
      "rewards/rejected": -0.5054029226303101,
      "step": 5960
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.944905041300739e-07,
      "logits/chosen": -2.4791269302368164,
      "logits/rejected": -2.4125843048095703,
      "logps/chosen": -333.6778869628906,
      "logps/rejected": -301.9044189453125,
      "loss": 0.5248,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.21129567921161652,
      "rewards/margins": 0.4146009385585785,
      "rewards/rejected": -0.6258966326713562,
      "step": 5970
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.866099316141606e-07,
      "logits/chosen": -2.6854865550994873,
      "logits/rejected": -2.6372344493865967,
      "logps/chosen": -300.1522216796875,
      "logps/rejected": -308.7620544433594,
      "loss": 0.5992,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.29093360900878906,
      "rewards/margins": 0.2515958249568939,
      "rewards/rejected": -0.5425294637680054,
      "step": 5980
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.787672026747946e-07,
      "logits/chosen": -2.477449893951416,
      "logits/rejected": -2.531902313232422,
      "logps/chosen": -295.63287353515625,
      "logps/rejected": -329.295654296875,
      "loss": 0.617,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.28388139605522156,
      "rewards/margins": 0.18577614426612854,
      "rewards/rejected": -0.4696575105190277,
      "step": 5990
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.709624809816223e-07,
      "logits/chosen": -2.5245113372802734,
      "logits/rejected": -2.5861029624938965,
      "logps/chosen": -321.34124755859375,
      "logps/rejected": -345.78643798828125,
      "loss": 0.5814,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.1564144492149353,
      "rewards/margins": 0.43052688241004944,
      "rewards/rejected": -0.5869413614273071,
      "step": 6000
    },
    {
      "epoch": 0.79,
      "eval_logits/chosen": -2.5504660606384277,
      "eval_logits/rejected": -2.520192861557007,
      "eval_logps/chosen": -308.9028625488281,
      "eval_logps/rejected": -305.8824462890625,
      "eval_loss": 0.586705207824707,
      "eval_rewards/accuracies": 0.7080000042915344,
      "eval_rewards/chosen": -0.24183543026447296,
      "eval_rewards/margins": 0.30403584241867065,
      "eval_rewards/rejected": -0.5458711981773376,
      "eval_runtime": 469.9296,
      "eval_samples_per_second": 4.256,
      "eval_steps_per_second": 1.064,
      "step": 6000
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.6319592941112e-07,
      "logits/chosen": -2.6814560890197754,
      "logits/rejected": -2.6334147453308105,
      "logps/chosen": -373.9043273925781,
      "logps/rejected": -349.0970153808594,
      "loss": 0.5531,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.13861213624477386,
      "rewards/margins": 0.39211505651474,
      "rewards/rejected": -0.5307271480560303,
      "step": 6010
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.554677100431927e-07,
      "logits/chosen": -2.579408645629883,
      "logits/rejected": -2.473529100418091,
      "logps/chosen": -315.31787109375,
      "logps/rejected": -308.83428955078125,
      "loss": 0.6099,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2672146260738373,
      "rewards/margins": 0.30611953139305115,
      "rewards/rejected": -0.5733340978622437,
      "step": 6020
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.4777798415779e-07,
      "logits/chosen": -2.5432662963867188,
      "logits/rejected": -2.6698403358459473,
      "logps/chosen": -305.22607421875,
      "logps/rejected": -312.6539001464844,
      "loss": 0.5846,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.24335560202598572,
      "rewards/margins": 0.3419436514377594,
      "rewards/rejected": -0.5852991938591003,
      "step": 6030
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.401269122315451e-07,
      "logits/chosen": -2.3377437591552734,
      "logits/rejected": -2.317566394805908,
      "logps/chosen": -245.70132446289062,
      "logps/rejected": -307.01837158203125,
      "loss": 0.5634,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.21264919638633728,
      "rewards/margins": 0.40377864241600037,
      "rewards/rejected": -0.6164278984069824,
      "step": 6040
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.325146539344196e-07,
      "logits/chosen": -2.454629898071289,
      "logits/rejected": -2.4771924018859863,
      "logps/chosen": -363.04827880859375,
      "logps/rejected": -350.6267395019531,
      "loss": 0.5681,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.37394410371780396,
      "rewards/margins": 0.2525387406349182,
      "rewards/rejected": -0.6264828443527222,
      "step": 6050
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.249413681263782e-07,
      "logits/chosen": -2.641986846923828,
      "logits/rejected": -2.6534454822540283,
      "logps/chosen": -259.7637023925781,
      "logps/rejected": -276.17962646484375,
      "loss": 0.5925,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.19203992187976837,
      "rewards/margins": 0.2957462668418884,
      "rewards/rejected": -0.4877861440181732,
      "step": 6060
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.174072128540686e-07,
      "logits/chosen": -2.4908456802368164,
      "logits/rejected": -2.480006217956543,
      "logps/chosen": -293.46746826171875,
      "logps/rejected": -304.4007568359375,
      "loss": 0.5475,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.1637772023677826,
      "rewards/margins": 0.4057283401489258,
      "rewards/rejected": -0.569505512714386,
      "step": 6070
    },
    {
      "epoch": 0.8,
      "learning_rate": 6.099123453475245e-07,
      "logits/chosen": -2.514127016067505,
      "logits/rejected": -2.460484504699707,
      "logps/chosen": -310.1918640136719,
      "logps/rejected": -283.559326171875,
      "loss": 0.5942,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.2885943055152893,
      "rewards/margins": 0.2289300411939621,
      "rewards/rejected": -0.5175243616104126,
      "step": 6080
    },
    {
      "epoch": 0.8,
      "learning_rate": 6.024569220168836e-07,
      "logits/chosen": -2.5178470611572266,
      "logits/rejected": -2.5889010429382324,
      "logps/chosen": -299.262451171875,
      "logps/rejected": -286.537109375,
      "loss": 0.5779,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.23814013600349426,
      "rewards/margins": 0.2815389335155487,
      "rewards/rejected": -0.519679069519043,
      "step": 6090
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.950410984491268e-07,
      "logits/chosen": -2.7100613117218018,
      "logits/rejected": -2.7072670459747314,
      "logps/chosen": -364.88330078125,
      "logps/rejected": -345.30145263671875,
      "loss": 0.5909,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.23071260750293732,
      "rewards/margins": 0.32682785391807556,
      "rewards/rejected": -0.5575404763221741,
      "step": 6100
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.876650294048262e-07,
      "logits/chosen": -2.4635324478149414,
      "logits/rejected": -2.5075535774230957,
      "logps/chosen": -290.86737060546875,
      "logps/rejected": -348.6126403808594,
      "loss": 0.5878,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.21459011733531952,
      "rewards/margins": 0.36013904213905334,
      "rewards/rejected": -0.5747290849685669,
      "step": 6110
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.8032886881492e-07,
      "logits/chosen": -2.7076010704040527,
      "logits/rejected": -2.550320863723755,
      "logps/chosen": -326.60009765625,
      "logps/rejected": -403.6198425292969,
      "loss": 0.6187,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.31724706292152405,
      "rewards/margins": 0.22199532389640808,
      "rewards/rejected": -0.5392423868179321,
      "step": 6120
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.730327697774988e-07,
      "logits/chosen": -2.5117106437683105,
      "logits/rejected": -2.506225347518921,
      "logps/chosen": -316.81219482421875,
      "logps/rejected": -331.5554504394531,
      "loss": 0.6249,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.2280002385377884,
      "rewards/margins": 0.21184459328651428,
      "rewards/rejected": -0.43984484672546387,
      "step": 6130
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.657768845546068e-07,
      "logits/chosen": -2.4038338661193848,
      "logits/rejected": -2.5388214588165283,
      "logps/chosen": -307.8619079589844,
      "logps/rejected": -383.5082702636719,
      "loss": 0.605,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.2060864418745041,
      "rewards/margins": 0.13893654942512512,
      "rewards/rejected": -0.345022976398468,
      "step": 6140
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.585613645690713e-07,
      "logits/chosen": -2.517221450805664,
      "logits/rejected": -2.4837019443511963,
      "logps/chosen": -259.75616455078125,
      "logps/rejected": -294.917236328125,
      "loss": 0.5567,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.29537954926490784,
      "rewards/margins": 0.40257397294044495,
      "rewards/rejected": -0.6979535818099976,
      "step": 6150
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.513863604013355e-07,
      "logits/chosen": -2.53302001953125,
      "logits/rejected": -2.406219720840454,
      "logps/chosen": -396.5958251953125,
      "logps/rejected": -334.18170166015625,
      "loss": 0.5657,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.08400538563728333,
      "rewards/margins": 0.5109080076217651,
      "rewards/rejected": -0.5949134230613708,
      "step": 6160
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.442520217863215e-07,
      "logits/chosen": -2.689875364303589,
      "logits/rejected": -2.602809190750122,
      "logps/chosen": -288.01361083984375,
      "logps/rejected": -299.61407470703125,
      "loss": 0.5664,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.2592290937900543,
      "rewards/margins": 0.291963666677475,
      "rewards/rejected": -0.5511927604675293,
      "step": 6170
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.371584976103034e-07,
      "logits/chosen": -2.6083056926727295,
      "logits/rejected": -2.544882297515869,
      "logps/chosen": -305.5341796875,
      "logps/rejected": -331.2020568847656,
      "loss": 0.5763,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.14928746223449707,
      "rewards/margins": 0.43168264627456665,
      "rewards/rejected": -0.5809701681137085,
      "step": 6180
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.301059359077987e-07,
      "logits/chosen": -2.5736923217773438,
      "logits/rejected": -2.5212347507476807,
      "logps/chosen": -296.933349609375,
      "logps/rejected": -293.7825622558594,
      "loss": 0.5969,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.20169103145599365,
      "rewards/margins": 0.3371420204639435,
      "rewards/rejected": -0.5388330221176147,
      "step": 6190
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.230944838584806e-07,
      "logits/chosen": -2.5412113666534424,
      "logits/rejected": -2.5593161582946777,
      "logps/chosen": -317.4584655761719,
      "logps/rejected": -338.98431396484375,
      "loss": 0.5554,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.21396486461162567,
      "rewards/margins": 0.44962701201438904,
      "rewards/rejected": -0.6635918617248535,
      "step": 6200
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.161242877841083e-07,
      "logits/chosen": -2.5664896965026855,
      "logits/rejected": -2.5260627269744873,
      "logps/chosen": -327.7658386230469,
      "logps/rejected": -291.395263671875,
      "loss": 0.5891,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.3413836359977722,
      "rewards/margins": 0.1652899980545044,
      "rewards/rejected": -0.5066736340522766,
      "step": 6210
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.091954931454682e-07,
      "logits/chosen": -2.5482285022735596,
      "logits/rejected": -2.5847091674804688,
      "logps/chosen": -298.89947509765625,
      "logps/rejected": -294.7454833984375,
      "loss": 0.6506,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.22662201523780823,
      "rewards/margins": 0.11685802787542343,
      "rewards/rejected": -0.34348005056381226,
      "step": 6220
    },
    {
      "epoch": 0.82,
      "learning_rate": 5.023082445393446e-07,
      "logits/chosen": -2.6040854454040527,
      "logits/rejected": -2.5687479972839355,
      "logps/chosen": -270.38751220703125,
      "logps/rejected": -280.75469970703125,
      "loss": 0.5924,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.250823050737381,
      "rewards/margins": 0.2815064787864685,
      "rewards/rejected": -0.5323294997215271,
      "step": 6230
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.95462685695498e-07,
      "logits/chosen": -2.544628620147705,
      "logits/rejected": -2.547745704650879,
      "logps/chosen": -323.119140625,
      "logps/rejected": -350.0447998046875,
      "loss": 0.5791,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.22325992584228516,
      "rewards/margins": 0.27086228132247925,
      "rewards/rejected": -0.4941222071647644,
      "step": 6240
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.88658959473666e-07,
      "logits/chosen": -2.5207772254943848,
      "logits/rejected": -2.5802581310272217,
      "logps/chosen": -308.8432312011719,
      "logps/rejected": -359.64288330078125,
      "loss": 0.5673,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.14637571573257446,
      "rewards/margins": 0.41771894693374634,
      "rewards/rejected": -0.5640946626663208,
      "step": 6250
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.818972078605821e-07,
      "logits/chosen": -2.5128281116485596,
      "logits/rejected": -2.457853317260742,
      "logps/chosen": -299.13055419921875,
      "logps/rejected": -281.8258361816406,
      "loss": 0.6092,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.31060346961021423,
      "rewards/margins": 0.19784040749073029,
      "rewards/rejected": -0.5084439516067505,
      "step": 6260
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.7517757196701514e-07,
      "logits/chosen": -2.509042263031006,
      "logits/rejected": -2.427283525466919,
      "logps/chosen": -240.78555297851562,
      "logps/rejected": -285.3486633300781,
      "loss": 0.6152,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.3734577000141144,
      "rewards/margins": 0.23900923132896423,
      "rewards/rejected": -0.6124669313430786,
      "step": 6270
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.6850019202482193e-07,
      "logits/chosen": -2.512200117111206,
      "logits/rejected": -2.500453472137451,
      "logps/chosen": -273.4225158691406,
      "logps/rejected": -334.0801696777344,
      "loss": 0.5791,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.18041428923606873,
      "rewards/margins": 0.3033163249492645,
      "rewards/rejected": -0.48373061418533325,
      "step": 6280
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.618652073840188e-07,
      "logits/chosen": -2.600834369659424,
      "logits/rejected": -2.5488085746765137,
      "logps/chosen": -323.1775817871094,
      "logps/rejected": -324.8509216308594,
      "loss": 0.5777,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.25158780813217163,
      "rewards/margins": 0.28678256273269653,
      "rewards/rejected": -0.5383704304695129,
      "step": 6290
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.5527275650987965e-07,
      "logits/chosen": -2.522247552871704,
      "logits/rejected": -2.547168254852295,
      "logps/chosen": -266.584228515625,
      "logps/rejected": -314.6968994140625,
      "loss": 0.5841,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.25916263461112976,
      "rewards/margins": 0.39021292328834534,
      "rewards/rejected": -0.6493755578994751,
      "step": 6300
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.487229769800394e-07,
      "logits/chosen": -2.5155081748962402,
      "logits/rejected": -2.467679262161255,
      "logps/chosen": -289.71258544921875,
      "logps/rejected": -309.3189697265625,
      "loss": 0.602,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.24935400485992432,
      "rewards/margins": 0.3160592019557953,
      "rewards/rejected": -0.5654131770133972,
      "step": 6310
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.422160054816285e-07,
      "logits/chosen": -2.5616321563720703,
      "logits/rejected": -2.6143195629119873,
      "logps/chosen": -299.6599426269531,
      "logps/rejected": -307.5855407714844,
      "loss": 0.5826,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.254104882478714,
      "rewards/margins": 0.33058133721351624,
      "rewards/rejected": -0.5846861600875854,
      "step": 6320
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.35751977808416e-07,
      "logits/chosen": -2.583028793334961,
      "logits/rejected": -2.5282931327819824,
      "logps/chosen": -372.45013427734375,
      "logps/rejected": -405.28350830078125,
      "loss": 0.5127,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.14416107535362244,
      "rewards/margins": 0.551115870475769,
      "rewards/rejected": -0.6952770352363586,
      "step": 6330
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.293310288579794e-07,
      "logits/chosen": -2.588747262954712,
      "logits/rejected": -2.5295772552490234,
      "logps/chosen": -273.9884338378906,
      "logps/rejected": -282.23907470703125,
      "loss": 0.5938,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19281110167503357,
      "rewards/margins": 0.29851463437080383,
      "rewards/rejected": -0.4913257658481598,
      "step": 6340
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.2295329262888733e-07,
      "logits/chosen": -2.4053893089294434,
      "logits/rejected": -2.4115614891052246,
      "logps/chosen": -342.3263854980469,
      "logps/rejected": -388.1925048828125,
      "loss": 0.5984,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3017456531524658,
      "rewards/margins": 0.3246362805366516,
      "rewards/rejected": -0.6263819932937622,
      "step": 6350
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.1661890221790316e-07,
      "logits/chosen": -2.5638811588287354,
      "logits/rejected": -2.525207996368408,
      "logps/chosen": -340.7550964355469,
      "logps/rejected": -313.72015380859375,
      "loss": 0.5626,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.23626506328582764,
      "rewards/margins": 0.39344462752342224,
      "rewards/rejected": -0.6297096014022827,
      "step": 6360
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.103279898172072e-07,
      "logits/chosen": -2.5897655487060547,
      "logits/rejected": -2.541827917098999,
      "logps/chosen": -323.07861328125,
      "logps/rejected": -299.7076110839844,
      "loss": 0.5254,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.22562608122825623,
      "rewards/margins": 0.38975054025650024,
      "rewards/rejected": -0.6153766512870789,
      "step": 6370
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.040806867116401e-07,
      "logits/chosen": -2.6776676177978516,
      "logits/rejected": -2.617333173751831,
      "logps/chosen": -325.8877258300781,
      "logps/rejected": -345.1810607910156,
      "loss": 0.5894,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2222873866558075,
      "rewards/margins": 0.29427021741867065,
      "rewards/rejected": -0.5165575742721558,
      "step": 6380
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.978771232759615e-07,
      "logits/chosen": -2.5058422088623047,
      "logits/rejected": -2.6096720695495605,
      "logps/chosen": -261.63604736328125,
      "logps/rejected": -275.4947204589844,
      "loss": 0.565,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17887890338897705,
      "rewards/margins": 0.3644290268421173,
      "rewards/rejected": -0.5433079600334167,
      "step": 6390
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.917174289721276e-07,
      "logits/chosen": -2.387831211090088,
      "logits/rejected": -2.41300630569458,
      "logps/chosen": -294.5731506347656,
      "logps/rejected": -296.50115966796875,
      "loss": 0.5222,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1628241389989853,
      "rewards/margins": 0.5051959753036499,
      "rewards/rejected": -0.6680201292037964,
      "step": 6400
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.856017323465938e-07,
      "logits/chosen": -2.6898727416992188,
      "logits/rejected": -2.568655490875244,
      "logps/chosen": -296.9098815917969,
      "logps/rejected": -276.76495361328125,
      "loss": 0.5912,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.3579789996147156,
      "rewards/margins": 0.22693082690238953,
      "rewards/rejected": -0.5849098563194275,
      "step": 6410
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.7953016102762695e-07,
      "logits/chosen": -2.5304763317108154,
      "logits/rejected": -2.507497549057007,
      "logps/chosen": -321.936767578125,
      "logps/rejected": -273.9273986816406,
      "loss": 0.5651,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.1546071320772171,
      "rewards/margins": 0.3859509229660034,
      "rewards/rejected": -0.5405580401420593,
      "step": 6420
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.7350284172264493e-07,
      "logits/chosen": -2.495654582977295,
      "logits/rejected": -2.462907314300537,
      "logps/chosen": -303.0787353515625,
      "logps/rejected": -300.4488830566406,
      "loss": 0.5997,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.30195024609565735,
      "rewards/margins": 0.36328279972076416,
      "rewards/rejected": -0.6652330160140991,
      "step": 6430
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.67519900215573e-07,
      "logits/chosen": -2.623809814453125,
      "logits/rejected": -2.636713743209839,
      "logps/chosen": -305.46063232421875,
      "logps/rejected": -310.02642822265625,
      "loss": 0.5753,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.18001635372638702,
      "rewards/margins": 0.3698897361755371,
      "rewards/rejected": -0.5499060750007629,
      "step": 6440
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.615814613642174e-07,
      "logits/chosen": -2.5246150493621826,
      "logits/rejected": -2.5499939918518066,
      "logps/chosen": -316.9003601074219,
      "logps/rejected": -327.5944519042969,
      "loss": 0.6326,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.28238850831985474,
      "rewards/margins": 0.1322154551744461,
      "rewards/rejected": -0.4146040380001068,
      "step": 6450
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.5568764909765795e-07,
      "logits/chosen": -2.6263697147369385,
      "logits/rejected": -2.6150715351104736,
      "logps/chosen": -328.6458435058594,
      "logps/rejected": -268.51361083984375,
      "loss": 0.6494,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.15244045853614807,
      "rewards/margins": 0.2182413786649704,
      "rewards/rejected": -0.3706818222999573,
      "step": 6460
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.498385864136672e-07,
      "logits/chosen": -2.556443691253662,
      "logits/rejected": -2.5505318641662598,
      "logps/chosen": -343.93829345703125,
      "logps/rejected": -317.4654846191406,
      "loss": 0.5867,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.25182396173477173,
      "rewards/margins": 0.3008098006248474,
      "rewards/rejected": -0.5526337027549744,
      "step": 6470
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.440343953761363e-07,
      "logits/chosen": -2.4140422344207764,
      "logits/rejected": -2.3615925312042236,
      "logps/chosen": -269.6112976074219,
      "logps/rejected": -271.3714294433594,
      "loss": 0.5731,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.1203186959028244,
      "rewards/margins": 0.4729042649269104,
      "rewards/rejected": -0.5932229161262512,
      "step": 6480
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.382751971125345e-07,
      "logits/chosen": -2.5431666374206543,
      "logits/rejected": -2.562822103500366,
      "logps/chosen": -324.5274353027344,
      "logps/rejected": -372.4595642089844,
      "loss": 0.6288,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.18169930577278137,
      "rewards/margins": 0.21922583878040314,
      "rewards/rejected": -0.4009251594543457,
      "step": 6490
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.3256111181137753e-07,
      "logits/chosen": -2.593381643295288,
      "logits/rejected": -2.5534257888793945,
      "logps/chosen": -278.71295166015625,
      "logps/rejected": -332.32916259765625,
      "loss": 0.6147,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.2595116198062897,
      "rewards/margins": 0.2278112918138504,
      "rewards/rejected": -0.4873228967189789,
      "step": 6500
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.2689225871971905e-07,
      "logits/chosen": -2.5177037715911865,
      "logits/rejected": -2.5049195289611816,
      "logps/chosen": -307.04730224609375,
      "logps/rejected": -303.7582702636719,
      "loss": 0.6577,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.3698388636112213,
      "rewards/margins": 0.12160823494195938,
      "rewards/rejected": -0.4914471209049225,
      "step": 6510
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.2126875614066523e-07,
      "logits/chosen": -2.610668420791626,
      "logits/rejected": -2.5742154121398926,
      "logps/chosen": -333.54290771484375,
      "logps/rejected": -353.10052490234375,
      "loss": 0.5332,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.18249043822288513,
      "rewards/margins": 0.4640236794948578,
      "rewards/rejected": -0.6465141773223877,
      "step": 6520
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.156907214309024e-07,
      "logits/chosen": -2.494969367980957,
      "logits/rejected": -2.5566751956939697,
      "logps/chosen": -263.52508544921875,
      "logps/rejected": -259.69818115234375,
      "loss": 0.6364,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.2828473448753357,
      "rewards/margins": 0.06948140263557434,
      "rewards/rejected": -0.3523287773132324,
      "step": 6530
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.1015827099824923e-07,
      "logits/chosen": -2.586190938949585,
      "logits/rejected": -2.579084634780884,
      "logps/chosen": -332.7265319824219,
      "logps/rejected": -330.85589599609375,
      "loss": 0.6042,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.14104333519935608,
      "rewards/margins": 0.2698945701122284,
      "rewards/rejected": -0.41093793511390686,
      "step": 6540
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.0467152029922926e-07,
      "logits/chosen": -2.380070447921753,
      "logits/rejected": -2.4754655361175537,
      "logps/chosen": -284.60626220703125,
      "logps/rejected": -284.9497985839844,
      "loss": 0.5451,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.1199837327003479,
      "rewards/margins": 0.3513313829898834,
      "rewards/rejected": -0.4713151454925537,
      "step": 6550
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.992305838366591e-07,
      "logits/chosen": -2.6500418186187744,
      "logits/rejected": -2.5698771476745605,
      "logps/chosen": -240.611083984375,
      "logps/rejected": -251.97286987304688,
      "loss": 0.5578,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.20479588210582733,
      "rewards/margins": 0.309184730052948,
      "rewards/rejected": -0.5139805674552917,
      "step": 6560
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.938355751572583e-07,
      "logits/chosen": -2.569633960723877,
      "logits/rejected": -2.511861562728882,
      "logps/chosen": -250.4905242919922,
      "logps/rejected": -290.0649719238281,
      "loss": 0.5421,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.14301282167434692,
      "rewards/margins": 0.4621787965297699,
      "rewards/rejected": -0.6051915884017944,
      "step": 6570
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.8848660684928307e-07,
      "logits/chosen": -2.5781993865966797,
      "logits/rejected": -2.5404911041259766,
      "logps/chosen": -262.2217102050781,
      "logps/rejected": -348.08282470703125,
      "loss": 0.5624,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.24088053405284882,
      "rewards/margins": 0.3800331652164459,
      "rewards/rejected": -0.6209137439727783,
      "step": 6580
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.8318379054017383e-07,
      "logits/chosen": -2.5964581966400146,
      "logits/rejected": -2.445565700531006,
      "logps/chosen": -320.2331848144531,
      "logps/rejected": -380.0176696777344,
      "loss": 0.5722,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.28656232357025146,
      "rewards/margins": 0.2690221667289734,
      "rewards/rejected": -0.5555844306945801,
      "step": 6590
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.779272368942246e-07,
      "logits/chosen": -2.514643669128418,
      "logits/rejected": -2.478142738342285,
      "logps/chosen": -310.20135498046875,
      "logps/rejected": -284.0975646972656,
      "loss": 0.6536,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.3290698230266571,
      "rewards/margins": 0.18459972739219666,
      "rewards/rejected": -0.5136696100234985,
      "step": 6600
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.7271705561027986e-07,
      "logits/chosen": -2.6962289810180664,
      "logits/rejected": -2.6481692790985107,
      "logps/chosen": -344.97625732421875,
      "logps/rejected": -396.3194885253906,
      "loss": 0.5275,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14827750623226166,
      "rewards/margins": 0.4757654070854187,
      "rewards/rejected": -0.6240429282188416,
      "step": 6610
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.6755335541943677e-07,
      "logits/chosen": -2.4567790031433105,
      "logits/rejected": -2.4796199798583984,
      "logps/chosen": -289.46539306640625,
      "logps/rejected": -292.6962890625,
      "loss": 0.5973,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.349151611328125,
      "rewards/margins": 0.12624169886112213,
      "rewards/rejected": -0.47539329528808594,
      "step": 6620
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.62436244082781e-07,
      "logits/chosen": -2.536022424697876,
      "logits/rejected": -2.5415444374084473,
      "logps/chosen": -295.7829895019531,
      "logps/rejected": -287.15252685546875,
      "loss": 0.6223,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.29702284932136536,
      "rewards/margins": 0.15536999702453613,
      "rewards/rejected": -0.4523928165435791,
      "step": 6630
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.5736582838913836e-07,
      "logits/chosen": -2.590641975402832,
      "logits/rejected": -2.5924506187438965,
      "logps/chosen": -281.2153625488281,
      "logps/rejected": -286.9208984375,
      "loss": 0.5854,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1338876485824585,
      "rewards/margins": 0.29984337091445923,
      "rewards/rejected": -0.43373093008995056,
      "step": 6640
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.5234221415284363e-07,
      "logits/chosen": -2.6226627826690674,
      "logits/rejected": -2.599210262298584,
      "logps/chosen": -312.03912353515625,
      "logps/rejected": -315.6419982910156,
      "loss": 0.5975,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.24639169871807098,
      "rewards/margins": 0.3372633457183838,
      "rewards/rejected": -0.583655059337616,
      "step": 6650
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.4736550621153375e-07,
      "logits/chosen": -2.5119102001190186,
      "logits/rejected": -2.5432655811309814,
      "logps/chosen": -354.1219177246094,
      "logps/rejected": -350.56298828125,
      "loss": 0.6117,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.26676005125045776,
      "rewards/margins": 0.3218335211277008,
      "rewards/rejected": -0.588593602180481,
      "step": 6660
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.424358084239609e-07,
      "logits/chosen": -2.6142399311065674,
      "logits/rejected": -2.713916301727295,
      "logps/chosen": -411.18341064453125,
      "logps/rejected": -339.60308837890625,
      "loss": 0.6018,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21380481123924255,
      "rewards/margins": 0.25104427337646484,
      "rewards/rejected": -0.464849054813385,
      "step": 6670
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.3755322366782158e-07,
      "logits/chosen": -2.48545503616333,
      "logits/rejected": -2.5491490364074707,
      "logps/chosen": -332.91790771484375,
      "logps/rejected": -318.849853515625,
      "loss": 0.5744,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.24387934803962708,
      "rewards/margins": 0.32038286328315735,
      "rewards/rejected": -0.5642622709274292,
      "step": 6680
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.3271785383761431e-07,
      "logits/chosen": -2.4943230152130127,
      "logits/rejected": -2.4346394538879395,
      "logps/chosen": -380.49920654296875,
      "logps/rejected": -364.5450134277344,
      "loss": 0.6466,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.2653929591178894,
      "rewards/margins": 0.2565198540687561,
      "rewards/rejected": -0.5219127535820007,
      "step": 6690
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.2792979984250978e-07,
      "logits/chosen": -2.5065841674804688,
      "logits/rejected": -2.488462448120117,
      "logps/chosen": -316.1371765136719,
      "logps/rejected": -299.7496643066406,
      "loss": 0.5848,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.21795110404491425,
      "rewards/margins": 0.313754141330719,
      "rewards/rejected": -0.5317052602767944,
      "step": 6700
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.231891616042453e-07,
      "logits/chosen": -2.511505365371704,
      "logits/rejected": -2.5903966426849365,
      "logps/chosen": -285.35870361328125,
      "logps/rejected": -365.3995361328125,
      "loss": 0.6514,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.26048222184181213,
      "rewards/margins": 0.2787513732910156,
      "rewards/rejected": -0.5392335653305054,
      "step": 6710
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.1849603805504328e-07,
      "logits/chosen": -2.441913604736328,
      "logits/rejected": -2.460020065307617,
      "logps/chosen": -327.166015625,
      "logps/rejected": -331.92279052734375,
      "loss": 0.5754,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.2605854868888855,
      "rewards/margins": 0.294552743434906,
      "rewards/rejected": -0.5551381707191467,
      "step": 6720
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.1385052713554066e-07,
      "logits/chosen": -2.595977306365967,
      "logits/rejected": -2.589634418487549,
      "logps/chosen": -242.5916748046875,
      "logps/rejected": -287.7140197753906,
      "loss": 0.6079,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.36479222774505615,
      "rewards/margins": 0.20024845004081726,
      "rewards/rejected": -0.5650407075881958,
      "step": 6730
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.0925272579274873e-07,
      "logits/chosen": -2.63620662689209,
      "logits/rejected": -2.611586093902588,
      "logps/chosen": -342.5591735839844,
      "logps/rejected": -348.40460205078125,
      "loss": 0.5766,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.21620401740074158,
      "rewards/margins": 0.3205227553844452,
      "rewards/rejected": -0.5367268323898315,
      "step": 6740
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.047027299780302e-07,
      "logits/chosen": -2.6527209281921387,
      "logits/rejected": -2.7160065174102783,
      "logps/chosen": -344.5036315917969,
      "logps/rejected": -357.87042236328125,
      "loss": 0.5907,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.3077239990234375,
      "rewards/margins": 0.29998379945755005,
      "rewards/rejected": -0.6077077984809875,
      "step": 6750
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.0020063464509492e-07,
      "logits/chosen": -2.5766146183013916,
      "logits/rejected": -2.5696825981140137,
      "logps/chosen": -284.848388671875,
      "logps/rejected": -241.0084991455078,
      "loss": 0.5643,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.12015555053949356,
      "rewards/margins": 0.4285253584384918,
      "rewards/rejected": -0.5486809015274048,
      "step": 6760
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.957465337480191e-07,
      "logits/chosen": -2.433406352996826,
      "logits/rejected": -2.4728941917419434,
      "logps/chosen": -270.2312316894531,
      "logps/rejected": -258.283447265625,
      "loss": 0.5863,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16889183223247528,
      "rewards/margins": 0.33343085646629333,
      "rewards/rejected": -0.5023227334022522,
      "step": 6770
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.9134052023928622e-07,
      "logits/chosen": -2.490142345428467,
      "logits/rejected": -2.5064761638641357,
      "logps/chosen": -260.85211181640625,
      "logps/rejected": -286.789306640625,
      "loss": 0.6311,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.26439350843429565,
      "rewards/margins": 0.2775447964668274,
      "rewards/rejected": -0.541938304901123,
      "step": 6780
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.8698268606784392e-07,
      "logits/chosen": -2.5506930351257324,
      "logits/rejected": -2.472028970718384,
      "logps/chosen": -279.47833251953125,
      "logps/rejected": -317.755126953125,
      "loss": 0.5259,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.1397266685962677,
      "rewards/margins": 0.4272634983062744,
      "rewards/rejected": -0.5669901371002197,
      "step": 6790
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.826731221771866e-07,
      "logits/chosen": -2.5460689067840576,
      "logits/rejected": -2.5425117015838623,
      "logps/chosen": -354.0918884277344,
      "logps/rejected": -331.4277038574219,
      "loss": 0.5989,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.21315228939056396,
      "rewards/margins": 0.4096096456050873,
      "rewards/rejected": -0.6227619647979736,
      "step": 6800
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.7841191850345967e-07,
      "logits/chosen": -2.4992847442626953,
      "logits/rejected": -2.553622245788574,
      "logps/chosen": -304.04803466796875,
      "logps/rejected": -341.6429138183594,
      "loss": 0.5936,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.27363067865371704,
      "rewards/margins": 0.3625665307044983,
      "rewards/rejected": -0.6361972093582153,
      "step": 6810
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.7419916397357905e-07,
      "logits/chosen": -2.529766798019409,
      "logits/rejected": -2.4962515830993652,
      "logps/chosen": -255.84994506835938,
      "logps/rejected": -275.2969055175781,
      "loss": 0.5974,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2806466221809387,
      "rewards/margins": 0.21514853835105896,
      "rewards/rejected": -0.49579519033432007,
      "step": 6820
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.700349465033782e-07,
      "logits/chosen": -2.6027166843414307,
      "logits/rejected": -2.534972906112671,
      "logps/chosen": -345.79766845703125,
      "logps/rejected": -291.8948669433594,
      "loss": 0.6274,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.36087167263031006,
      "rewards/margins": 0.25738582015037537,
      "rewards/rejected": -0.618257462978363,
      "step": 6830
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.6591935299577227e-07,
      "logits/chosen": -2.5771901607513428,
      "logits/rejected": -2.568636178970337,
      "logps/chosen": -335.08758544921875,
      "logps/rejected": -357.94329833984375,
      "loss": 0.5739,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2538537085056305,
      "rewards/margins": 0.3991164565086365,
      "rewards/rejected": -0.6529701948165894,
      "step": 6840
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.6185246933894338e-07,
      "logits/chosen": -2.544599771499634,
      "logits/rejected": -2.4817824363708496,
      "logps/chosen": -338.9317932128906,
      "logps/rejected": -354.46575927734375,
      "loss": 0.5834,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.35685110092163086,
      "rewards/margins": 0.30958980321884155,
      "rewards/rejected": -0.6664408445358276,
      "step": 6850
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.5783438040455097e-07,
      "logits/chosen": -2.5295493602752686,
      "logits/rejected": -2.6433188915252686,
      "logps/chosen": -347.1949768066406,
      "logps/rejected": -312.98980712890625,
      "loss": 0.5991,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.3010823130607605,
      "rewards/margins": 0.13934998214244843,
      "rewards/rejected": -0.4404323101043701,
      "step": 6860
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.538651700459576e-07,
      "logits/chosen": -2.6250827312469482,
      "logits/rejected": -2.590571403503418,
      "logps/chosen": -387.9285583496094,
      "logps/rejected": -334.91522216796875,
      "loss": 0.6188,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.4069536328315735,
      "rewards/margins": 0.23420529067516327,
      "rewards/rejected": -0.641158938407898,
      "step": 6870
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.4994492109648151e-07,
      "logits/chosen": -2.4459919929504395,
      "logits/rejected": -2.3544578552246094,
      "logps/chosen": -252.25808715820312,
      "logps/rejected": -341.5327453613281,
      "loss": 0.5787,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.20545756816864014,
      "rewards/margins": 0.2764742970466614,
      "rewards/rejected": -0.4819318652153015,
      "step": 6880
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.4607371536766695e-07,
      "logits/chosen": -2.5497701168060303,
      "logits/rejected": -2.6141464710235596,
      "logps/chosen": -268.9443359375,
      "logps/rejected": -308.8302001953125,
      "loss": 0.6344,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.32575878500938416,
      "rewards/margins": 0.21937373280525208,
      "rewards/rejected": -0.5451325178146362,
      "step": 6890
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.4225163364757655e-07,
      "logits/chosen": -2.6052086353302,
      "logits/rejected": -2.554774522781372,
      "logps/chosen": -365.8276062011719,
      "logps/rejected": -333.3267517089844,
      "loss": 0.5578,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.2463061809539795,
      "rewards/margins": 0.2694723904132843,
      "rewards/rejected": -0.5157786011695862,
      "step": 6900
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.3847875569910462e-07,
      "logits/chosen": -2.527517080307007,
      "logits/rejected": -2.5814757347106934,
      "logps/chosen": -260.64996337890625,
      "logps/rejected": -265.3381652832031,
      "loss": 0.6107,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.3884786367416382,
      "rewards/margins": 0.19082988798618317,
      "rewards/rejected": -0.5793085098266602,
      "step": 6910
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.3475516025831552e-07,
      "logits/chosen": -2.553699493408203,
      "logits/rejected": -2.5805094242095947,
      "logps/chosen": -235.4549102783203,
      "logps/rejected": -278.7104797363281,
      "loss": 0.5879,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.24513201415538788,
      "rewards/margins": 0.22519740462303162,
      "rewards/rejected": -0.4703294336795807,
      "step": 6920
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.310809250327974e-07,
      "logits/chosen": -2.5498898029327393,
      "logits/rejected": -2.5694942474365234,
      "logps/chosen": -260.2915344238281,
      "logps/rejected": -301.33331298828125,
      "loss": 0.5541,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.2523956000804901,
      "rewards/margins": 0.3117953836917877,
      "rewards/rejected": -0.5641909837722778,
      "step": 6930
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.2745612670004153e-07,
      "logits/chosen": -2.6088833808898926,
      "logits/rejected": -2.4972448348999023,
      "logps/chosen": -282.6246032714844,
      "logps/rejected": -294.3809509277344,
      "loss": 0.5616,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.30884820222854614,
      "rewards/margins": 0.40127071738243103,
      "rewards/rejected": -0.7101189494132996,
      "step": 6940
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.2388084090584395e-07,
      "logits/chosen": -2.553621530532837,
      "logits/rejected": -2.5755157470703125,
      "logps/chosen": -285.8392333984375,
      "logps/rejected": -273.127685546875,
      "loss": 0.5947,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.21612191200256348,
      "rewards/margins": 0.37774962186813354,
      "rewards/rejected": -0.5938714742660522,
      "step": 6950
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.2035514226272305e-07,
      "logits/chosen": -2.54499888420105,
      "logits/rejected": -2.6355156898498535,
      "logps/chosen": -309.70703125,
      "logps/rejected": -313.765869140625,
      "loss": 0.6071,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.29586124420166016,
      "rewards/margins": 0.153177410364151,
      "rewards/rejected": -0.4490385949611664,
      "step": 6960
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1687910434836607e-07,
      "logits/chosen": -2.36899995803833,
      "logits/rejected": -2.435290575027466,
      "logps/chosen": -256.7616271972656,
      "logps/rejected": -355.43634033203125,
      "loss": 0.5517,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.23554377257823944,
      "rewards/margins": 0.4043079912662506,
      "rewards/rejected": -0.6398516893386841,
      "step": 6970
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1345279970409128e-07,
      "logits/chosen": -2.604055881500244,
      "logits/rejected": -2.6000876426696777,
      "logps/chosen": -293.7242431640625,
      "logps/rejected": -350.16375732421875,
      "loss": 0.6043,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.23609530925750732,
      "rewards/margins": 0.22831125557422638,
      "rewards/rejected": -0.4644065499305725,
      "step": 6980
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1007629983333629e-07,
      "logits/chosen": -2.3824760913848877,
      "logits/rejected": -2.38716983795166,
      "logps/chosen": -292.387939453125,
      "logps/rejected": -310.6404724121094,
      "loss": 0.5431,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.18008548021316528,
      "rewards/margins": 0.550257682800293,
      "rewards/rejected": -0.7303431630134583,
      "step": 6990
    },
    {
      "epoch": 0.92,
      "learning_rate": 1.067496752001626e-07,
      "logits/chosen": -2.5673344135284424,
      "logits/rejected": -2.538301467895508,
      "logps/chosen": -250.818115234375,
      "logps/rejected": -313.76220703125,
      "loss": 0.5811,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.2165127545595169,
      "rewards/margins": 0.34249112010002136,
      "rewards/rejected": -0.5590038299560547,
      "step": 7000
    },
    {
      "epoch": 0.92,
      "learning_rate": 1.0347299522778909e-07,
      "logits/chosen": -2.5746231079101562,
      "logits/rejected": -2.670893430709839,
      "logps/chosen": -262.069580078125,
      "logps/rejected": -300.20928955078125,
      "loss": 0.6302,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.27484551072120667,
      "rewards/margins": 0.19697365164756775,
      "rewards/rejected": -0.4718192219734192,
      "step": 7010
    },
    {
      "epoch": 0.92,
      "learning_rate": 1.0024632829713971e-07,
      "logits/chosen": -2.5796866416931152,
      "logits/rejected": -2.5593037605285645,
      "logps/chosen": -247.50619506835938,
      "logps/rejected": -334.5338439941406,
      "loss": 0.5687,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.16089312732219696,
      "rewards/margins": 0.3748016953468323,
      "rewards/rejected": -0.5356948375701904,
      "step": 7020
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.706974174541889e-08,
      "logits/chosen": -2.612639904022217,
      "logits/rejected": -2.5660667419433594,
      "logps/chosen": -328.50079345703125,
      "logps/rejected": -310.83734130859375,
      "loss": 0.587,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2703118920326233,
      "rewards/margins": 0.21857671439647675,
      "rewards/rejected": -0.48888859152793884,
      "step": 7030
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.39433018647043e-08,
      "logits/chosen": -2.5860233306884766,
      "logits/rejected": -2.4884932041168213,
      "logps/chosen": -287.6629333496094,
      "logps/rejected": -283.45001220703125,
      "loss": 0.5869,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.26323238015174866,
      "rewards/margins": 0.3591728210449219,
      "rewards/rejected": -0.6224051713943481,
      "step": 7040
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.086707390056543e-08,
      "logits/chosen": -2.6537437438964844,
      "logits/rejected": -2.6398043632507324,
      "logps/chosen": -323.03753662109375,
      "logps/rejected": -351.5426330566406,
      "loss": 0.6427,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3780897259712219,
      "rewards/margins": 0.2068289965391159,
      "rewards/rejected": -0.584918737411499,
      "step": 7050
    },
    {
      "epoch": 0.92,
      "learning_rate": 8.784112205070083e-08,
      "logits/chosen": -2.5830557346343994,
      "logits/rejected": -2.62549090385437,
      "logps/chosen": -274.672119140625,
      "logps/rejected": -305.0771484375,
      "loss": 0.5494,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.09539235383272171,
      "rewards/margins": 0.4771255850791931,
      "rewards/rejected": -0.572517991065979,
      "step": 7060
    },
    {
      "epoch": 0.93,
      "learning_rate": 8.486550946359779e-08,
      "logits/chosen": -2.622972011566162,
      "logits/rejected": -2.623706102371216,
      "logps/chosen": -286.2050476074219,
      "logps/rejected": -285.8695373535156,
      "loss": 0.6466,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.33009567856788635,
      "rewards/margins": 0.13764071464538574,
      "rewards/rejected": -0.4677363336086273,
      "step": 7070
    },
    {
      "epoch": 0.93,
      "learning_rate": 8.194029823721556e-08,
      "logits/chosen": -2.66804575920105,
      "logits/rejected": -2.6642093658447266,
      "logps/chosen": -335.0277099609375,
      "logps/rejected": -316.1394348144531,
      "loss": 0.6425,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.2565035820007324,
      "rewards/margins": 0.15874430537223816,
      "rewards/rejected": -0.4152478575706482,
      "step": 7080
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.906554941768896e-08,
      "logits/chosen": -2.480766534805298,
      "logits/rejected": -2.4899449348449707,
      "logps/chosen": -271.68438720703125,
      "logps/rejected": -331.60064697265625,
      "loss": 0.607,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.17925897240638733,
      "rewards/margins": 0.2871427536010742,
      "rewards/rejected": -0.46640172600746155,
      "step": 7090
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.624132299805575e-08,
      "logits/chosen": -2.6367640495300293,
      "logits/rejected": -2.642698287963867,
      "logps/chosen": -314.950927734375,
      "logps/rejected": -329.8536071777344,
      "loss": 0.5392,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.128908172249794,
      "rewards/margins": 0.4371621608734131,
      "rewards/rejected": -0.5660702586174011,
      "step": 7100
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.346767791700127e-08,
      "logits/chosen": -2.549582004547119,
      "logits/rejected": -2.59175443649292,
      "logps/chosen": -292.8898010253906,
      "logps/rejected": -312.5947265625,
      "loss": 0.5883,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.26378917694091797,
      "rewards/margins": 0.34494879841804504,
      "rewards/rejected": -0.6087379455566406,
      "step": 7110
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.07446720576327e-08,
      "logits/chosen": -2.580864667892456,
      "logits/rejected": -2.5099356174468994,
      "logps/chosen": -319.59539794921875,
      "logps/rejected": -322.5418395996094,
      "loss": 0.5841,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.23274675011634827,
      "rewards/margins": 0.3921774625778198,
      "rewards/rejected": -0.6249242424964905,
      "step": 7120
    },
    {
      "epoch": 0.93,
      "learning_rate": 6.807236224626701e-08,
      "logits/chosen": -2.480311155319214,
      "logits/rejected": -2.445802688598633,
      "logps/chosen": -252.0524444580078,
      "logps/rejected": -291.4988098144531,
      "loss": 0.5848,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.313507616519928,
      "rewards/margins": 0.20375177264213562,
      "rewards/rejected": -0.517259418964386,
      "step": 7130
    },
    {
      "epoch": 0.93,
      "learning_rate": 6.545080425124888e-08,
      "logits/chosen": -2.58805251121521,
      "logits/rejected": -2.58807373046875,
      "logps/chosen": -279.99005126953125,
      "logps/rejected": -277.5595397949219,
      "loss": 0.6408,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2955562174320221,
      "rewards/margins": 0.27668994665145874,
      "rewards/rejected": -0.5722461938858032,
      "step": 7140
    },
    {
      "epoch": 0.94,
      "learning_rate": 6.288005278178382e-08,
      "logits/chosen": -2.661332130432129,
      "logits/rejected": -2.63032603263855,
      "logps/chosen": -300.8346252441406,
      "logps/rejected": -303.6176452636719,
      "loss": 0.6006,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1539689600467682,
      "rewards/margins": 0.30857062339782715,
      "rewards/rejected": -0.4625396132469177,
      "step": 7150
    },
    {
      "epoch": 0.94,
      "learning_rate": 6.036016148679825e-08,
      "logits/chosen": -2.4713082313537598,
      "logits/rejected": -2.405475616455078,
      "logps/chosen": -299.3686218261719,
      "logps/rejected": -321.78045654296875,
      "loss": 0.567,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.3375488221645355,
      "rewards/margins": 0.2744831442832947,
      "rewards/rejected": -0.6120319962501526,
      "step": 7160
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.7891182953819235e-08,
      "logits/chosen": -2.4739296436309814,
      "logits/rejected": -2.4340744018554688,
      "logps/chosen": -314.0517883300781,
      "logps/rejected": -305.4825744628906,
      "loss": 0.6257,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.3760947585105896,
      "rewards/margins": 0.10057481378316879,
      "rewards/rejected": -0.4766695499420166,
      "step": 7170
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.547316870787689e-08,
      "logits/chosen": -2.6653645038604736,
      "logits/rejected": -2.6970224380493164,
      "logps/chosen": -338.0147399902344,
      "logps/rejected": -332.74420166015625,
      "loss": 0.6123,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.2274622619152069,
      "rewards/margins": 0.31383103132247925,
      "rewards/rejected": -0.5412932634353638,
      "step": 7180
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.310616921042927e-08,
      "logits/chosen": -2.5177793502807617,
      "logits/rejected": -2.498081684112549,
      "logps/chosen": -274.1446838378906,
      "logps/rejected": -251.3614501953125,
      "loss": 0.5866,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.16666170954704285,
      "rewards/margins": 0.31953534483909607,
      "rewards/rejected": -0.4861970841884613,
      "step": 7190
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.079023385830939e-08,
      "logits/chosen": -2.4965853691101074,
      "logits/rejected": -2.548297643661499,
      "logps/chosen": -303.9081115722656,
      "logps/rejected": -291.8494873046875,
      "loss": 0.5912,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.19232706725597382,
      "rewards/margins": 0.3016923666000366,
      "rewards/rejected": -0.49401944875717163,
      "step": 7200
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.8525410982695476e-08,
      "logits/chosen": -2.664166212081909,
      "logits/rejected": -2.532313823699951,
      "logps/chosen": -365.9681091308594,
      "logps/rejected": -338.4162902832031,
      "loss": 0.605,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.3020409643650055,
      "rewards/margins": 0.2812599241733551,
      "rewards/rejected": -0.5833008885383606,
      "step": 7210
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.6311747848099e-08,
      "logits/chosen": -2.6504900455474854,
      "logits/rejected": -2.489622116088867,
      "logps/chosen": -325.5758056640625,
      "logps/rejected": -324.2012634277344,
      "loss": 0.5608,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.21951039135456085,
      "rewards/margins": 0.33448880910873413,
      "rewards/rejected": -0.5539992451667786,
      "step": 7220
    },
    {
      "epoch": 0.95,
      "learning_rate": 4.4149290651382405e-08,
      "logits/chosen": -2.489150047302246,
      "logits/rejected": -2.5522258281707764,
      "logps/chosen": -243.5304412841797,
      "logps/rejected": -324.83746337890625,
      "loss": 0.5436,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.22564832866191864,
      "rewards/margins": 0.46516934037208557,
      "rewards/rejected": -0.690817654132843,
      "step": 7230
    },
    {
      "epoch": 0.95,
      "learning_rate": 4.203808452079211e-08,
      "logits/chosen": -2.4958624839782715,
      "logits/rejected": -2.4828903675079346,
      "logps/chosen": -232.5818634033203,
      "logps/rejected": -258.32452392578125,
      "loss": 0.5691,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.276003897190094,
      "rewards/margins": 0.24557428061962128,
      "rewards/rejected": -0.5215781927108765,
      "step": 7240
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.9978173515018427e-08,
      "logits/chosen": -2.5811498165130615,
      "logits/rejected": -2.5918076038360596,
      "logps/chosen": -302.5559997558594,
      "logps/rejected": -328.662109375,
      "loss": 0.6272,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.3498912751674652,
      "rewards/margins": 0.17439976334571838,
      "rewards/rejected": -0.5242910385131836,
      "step": 7250
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.7969600622274614e-08,
      "logits/chosen": -2.495222806930542,
      "logits/rejected": -2.4114508628845215,
      "logps/chosen": -374.94720458984375,
      "logps/rejected": -313.7791442871094,
      "loss": 0.5368,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.24385342001914978,
      "rewards/margins": 0.41693535447120667,
      "rewards/rejected": -0.6607887148857117,
      "step": 7260
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.601240775940151e-08,
      "logits/chosen": -2.627872943878174,
      "logits/rejected": -2.585951805114746,
      "logps/chosen": -384.087158203125,
      "logps/rejected": -376.13232421875,
      "loss": 0.601,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.20892560482025146,
      "rewards/margins": 0.41255682706832886,
      "rewards/rejected": -0.6214824914932251,
      "step": 7270
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.410663577099071e-08,
      "logits/chosen": -2.462028980255127,
      "logits/rejected": -2.428433418273926,
      "logps/chosen": -296.00628662109375,
      "logps/rejected": -314.38885498046875,
      "loss": 0.5723,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17949989438056946,
      "rewards/margins": 0.40048688650131226,
      "rewards/rejected": -0.5799866914749146,
      "step": 7280
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.2252324428534986e-08,
      "logits/chosen": -2.5399367809295654,
      "logits/rejected": -2.384171962738037,
      "logps/chosen": -287.9708251953125,
      "logps/rejected": -294.95611572265625,
      "loss": 0.554,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.1900123953819275,
      "rewards/margins": 0.44957780838012695,
      "rewards/rejected": -0.6395902037620544,
      "step": 7290
    },
    {
      "epoch": 0.96,
      "learning_rate": 3.0449512429594486e-08,
      "logits/chosen": -2.656339406967163,
      "logits/rejected": -2.596925973892212,
      "logps/chosen": -322.1007995605469,
      "logps/rejected": -355.7225646972656,
      "loss": 0.5746,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.21370860934257507,
      "rewards/margins": 0.38849660754203796,
      "rewards/rejected": -0.6022052764892578,
      "step": 7300
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.8698237396992956e-08,
      "logits/chosen": -2.4962539672851562,
      "logits/rejected": -2.4656410217285156,
      "logps/chosen": -263.3753662109375,
      "logps/rejected": -287.70123291015625,
      "loss": 0.5934,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.19082583487033844,
      "rewards/margins": 0.47203341126441956,
      "rewards/rejected": -0.6628592014312744,
      "step": 7310
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.6998535878030584e-08,
      "logits/chosen": -2.484363555908203,
      "logits/rejected": -2.4631125926971436,
      "logps/chosen": -315.1961669921875,
      "logps/rejected": -283.8462829589844,
      "loss": 0.5741,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2730049788951874,
      "rewards/margins": 0.30888694524765015,
      "rewards/rejected": -0.5818918943405151,
      "step": 7320
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.535044334372072e-08,
      "logits/chosen": -2.54293155670166,
      "logits/rejected": -2.618013620376587,
      "logps/chosen": -318.7171325683594,
      "logps/rejected": -346.13250732421875,
      "loss": 0.5512,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.2942810654640198,
      "rewards/margins": 0.4349061846733093,
      "rewards/rejected": -0.7291873693466187,
      "step": 7330
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.3753994188051853e-08,
      "logits/chosen": -2.343754529953003,
      "logits/rejected": -2.3049113750457764,
      "logps/chosen": -321.657470703125,
      "logps/rejected": -334.9903869628906,
      "loss": 0.5791,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.2927539646625519,
      "rewards/margins": 0.3356403410434723,
      "rewards/rejected": -0.6283942461013794,
      "step": 7340
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.220922172726764e-08,
      "logits/chosen": -2.6127965450286865,
      "logits/rejected": -2.5224764347076416,
      "logps/chosen": -301.496337890625,
      "logps/rejected": -272.4375305175781,
      "loss": 0.6179,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.27615025639533997,
      "rewards/margins": 0.1150190606713295,
      "rewards/rejected": -0.39116930961608887,
      "step": 7350
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.071615819917244e-08,
      "logits/chosen": -2.5463995933532715,
      "logits/rejected": -2.495800495147705,
      "logps/chosen": -286.66082763671875,
      "logps/rejected": -307.84967041015625,
      "loss": 0.6042,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.19279077649116516,
      "rewards/margins": 0.3158416152000427,
      "rewards/rejected": -0.5086323618888855,
      "step": 7360
    },
    {
      "epoch": 0.96,
      "learning_rate": 1.9274834762459393e-08,
      "logits/chosen": -2.4749820232391357,
      "logits/rejected": -2.576807975769043,
      "logps/chosen": -349.1869201660156,
      "logps/rejected": -311.72119140625,
      "loss": 0.5868,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.31071510910987854,
      "rewards/margins": 0.22670575976371765,
      "rewards/rejected": -0.5374208688735962,
      "step": 7370
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.7885281496058947e-08,
      "logits/chosen": -2.5542681217193604,
      "logits/rejected": -2.601315975189209,
      "logps/chosen": -292.5190734863281,
      "logps/rejected": -291.88873291015625,
      "loss": 0.5945,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.09982474148273468,
      "rewards/margins": 0.30825820565223694,
      "rewards/rejected": -0.40808290243148804,
      "step": 7380
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.654752739851134e-08,
      "logits/chosen": -2.478485107421875,
      "logits/rejected": -2.5681591033935547,
      "logps/chosen": -242.53207397460938,
      "logps/rejected": -303.34698486328125,
      "loss": 0.5478,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.27765676379203796,
      "rewards/margins": 0.29515042901039124,
      "rewards/rejected": -0.5728071928024292,
      "step": 7390
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.526160038736235e-08,
      "logits/chosen": -2.537532329559326,
      "logits/rejected": -2.5868353843688965,
      "logps/chosen": -318.7858581542969,
      "logps/rejected": -292.2711181640625,
      "loss": 0.5937,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.29203277826309204,
      "rewards/margins": 0.3736723065376282,
      "rewards/rejected": -0.6657050848007202,
      "step": 7400
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.402752729857959e-08,
      "logits/chosen": -2.590498685836792,
      "logits/rejected": -2.5806334018707275,
      "logps/chosen": -261.1465759277344,
      "logps/rejected": -281.0189514160156,
      "loss": 0.5891,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.2757474482059479,
      "rewards/margins": 0.19477182626724243,
      "rewards/rejected": -0.4705192446708679,
      "step": 7410
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.2845333885992683e-08,
      "logits/chosen": -2.4128005504608154,
      "logits/rejected": -2.5475845336914062,
      "logps/chosen": -273.72332763671875,
      "logps/rejected": -293.3398742675781,
      "loss": 0.539,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.13520777225494385,
      "rewards/margins": 0.4506809115409851,
      "rewards/rejected": -0.5858887434005737,
      "step": 7420
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.171504482075675e-08,
      "logits/chosen": -2.54884934425354,
      "logits/rejected": -2.5620005130767822,
      "logps/chosen": -278.064208984375,
      "logps/rejected": -289.39794921875,
      "loss": 0.5428,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.16827186942100525,
      "rewards/margins": 0.4255130887031555,
      "rewards/rejected": -0.5937849283218384,
      "step": 7430
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.0636683690836147e-08,
      "logits/chosen": -2.526398181915283,
      "logits/rejected": -2.4657576084136963,
      "logps/chosen": -262.78839111328125,
      "logps/rejected": -258.28857421875,
      "loss": 0.582,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.17425993084907532,
      "rewards/margins": 0.3398754894733429,
      "rewards/rejected": -0.5141354203224182,
      "step": 7440
    },
    {
      "epoch": 0.97,
      "learning_rate": 9.610273000513203e-09,
      "logits/chosen": -2.614753246307373,
      "logits/rejected": -2.573422908782959,
      "logps/chosen": -308.4884338378906,
      "logps/rejected": -377.4931335449219,
      "loss": 0.5838,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.32458263635635376,
      "rewards/margins": 0.32800227403640747,
      "rewards/rejected": -0.652584969997406,
      "step": 7450
    },
    {
      "epoch": 0.98,
      "learning_rate": 8.635834169918312e-09,
      "logits/chosen": -2.557628631591797,
      "logits/rejected": -2.5104641914367676,
      "logps/chosen": -318.6761474609375,
      "logps/rejected": -305.15374755859375,
      "loss": 0.6005,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.39393773674964905,
      "rewards/margins": 0.14898169040679932,
      "rewards/rejected": -0.5429194569587708,
      "step": 7460
    },
    {
      "epoch": 0.98,
      "learning_rate": 7.713387534582506e-09,
      "logits/chosen": -2.605743408203125,
      "logits/rejected": -2.6812820434570312,
      "logps/chosen": -354.9667663574219,
      "logps/rejected": -365.39080810546875,
      "loss": 0.5956,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3010857403278351,
      "rewards/margins": 0.38392147421836853,
      "rewards/rejected": -0.6850072145462036,
      "step": 7470
    },
    {
      "epoch": 0.98,
      "learning_rate": 6.84295234501392e-09,
      "logits/chosen": -2.485640048980713,
      "logits/rejected": -2.4678311347961426,
      "logps/chosen": -286.43231201171875,
      "logps/rejected": -268.5378723144531,
      "loss": 0.5629,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17162033915519714,
      "rewards/margins": 0.5318527817726135,
      "rewards/rejected": -0.7034731507301331,
      "step": 7480
    },
    {
      "epoch": 0.98,
      "learning_rate": 6.024546766295325e-09,
      "logits/chosen": -2.6134684085845947,
      "logits/rejected": -2.539670944213867,
      "logps/chosen": -296.3213806152344,
      "logps/rejected": -301.2705383300781,
      "loss": 0.5466,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.14228995144367218,
      "rewards/margins": 0.40069717168807983,
      "rewards/rejected": -0.542987048625946,
      "step": 7490
    },
    {
      "epoch": 0.98,
      "learning_rate": 5.2581878777049895e-09,
      "logits/chosen": -2.583683490753174,
      "logits/rejected": -2.539593458175659,
      "logps/chosen": -265.1103820800781,
      "logps/rejected": -264.25726318359375,
      "loss": 0.5898,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2404584139585495,
      "rewards/margins": 0.2934717535972595,
      "rewards/rejected": -0.5339301824569702,
      "step": 7500
    },
    {
      "epoch": 0.98,
      "learning_rate": 4.543891672361411e-09,
      "logits/chosen": -2.632657289505005,
      "logits/rejected": -2.6300811767578125,
      "logps/chosen": -299.6949462890625,
      "logps/rejected": -331.4681091308594,
      "loss": 0.57,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.18346548080444336,
      "rewards/margins": 0.34690719842910767,
      "rewards/rejected": -0.5303727388381958,
      "step": 7510
    },
    {
      "epoch": 0.98,
      "learning_rate": 3.881673056887747e-09,
      "logits/chosen": -2.5574936866760254,
      "logits/rejected": -2.582885265350342,
      "logps/chosen": -237.0987091064453,
      "logps/rejected": -262.08770751953125,
      "loss": 0.5632,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.2373037338256836,
      "rewards/margins": 0.37760406732559204,
      "rewards/rejected": -0.6149078607559204,
      "step": 7520
    },
    {
      "epoch": 0.99,
      "learning_rate": 3.2715458511023425e-09,
      "logits/chosen": -2.514371871948242,
      "logits/rejected": -2.5667572021484375,
      "logps/chosen": -275.7968444824219,
      "logps/rejected": -298.59979248046875,
      "loss": 0.5517,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.21118685603141785,
      "rewards/margins": 0.3040321171283722,
      "rewards/rejected": -0.5152189135551453,
      "step": 7530
    },
    {
      "epoch": 0.99,
      "learning_rate": 2.7135227877289617e-09,
      "logits/chosen": -2.454634428024292,
      "logits/rejected": -2.5675089359283447,
      "logps/chosen": -249.9116668701172,
      "logps/rejected": -345.05108642578125,
      "loss": 0.6025,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.25211089849472046,
      "rewards/margins": 0.18249237537384033,
      "rewards/rejected": -0.4346033036708832,
      "step": 7540
    },
    {
      "epoch": 0.99,
      "learning_rate": 2.2076155121328326e-09,
      "logits/chosen": -2.563410758972168,
      "logits/rejected": -2.6032705307006836,
      "logps/chosen": -331.269775390625,
      "logps/rejected": -366.534423828125,
      "loss": 0.5774,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.24757087230682373,
      "rewards/margins": 0.3087003231048584,
      "rewards/rejected": -0.5562711954116821,
      "step": 7550
    },
    {
      "epoch": 0.99,
      "learning_rate": 1.7538345820755641e-09,
      "logits/chosen": -2.346280574798584,
      "logits/rejected": -2.4170234203338623,
      "logps/chosen": -263.93865966796875,
      "logps/rejected": -321.49615478515625,
      "loss": 0.6129,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.2762508988380432,
      "rewards/margins": 0.27643775939941406,
      "rewards/rejected": -0.5526885986328125,
      "step": 7560
    },
    {
      "epoch": 0.99,
      "learning_rate": 1.3521894674961567e-09,
      "logits/chosen": -2.68477201461792,
      "logits/rejected": -2.602001667022705,
      "logps/chosen": -266.88397216796875,
      "logps/rejected": -297.6312255859375,
      "loss": 0.5684,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.24672797322273254,
      "rewards/margins": 0.292520135641098,
      "rewards/rejected": -0.5392481088638306,
      "step": 7570
    },
    {
      "epoch": 0.99,
      "learning_rate": 1.0026885503131023e-09,
      "logits/chosen": -2.5589654445648193,
      "logits/rejected": -2.558234214782715,
      "logps/chosen": -265.538818359375,
      "logps/rejected": -298.25372314453125,
      "loss": 0.6381,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1422235518693924,
      "rewards/margins": 0.3625614047050476,
      "rewards/rejected": -0.5047849416732788,
      "step": 7580
    },
    {
      "epoch": 0.99,
      "learning_rate": 7.053391242492491e-10,
      "logits/chosen": -2.3764610290527344,
      "logits/rejected": -2.380582332611084,
      "logps/chosen": -280.76300048828125,
      "logps/rejected": -296.69122314453125,
      "loss": 0.5852,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.240085169672966,
      "rewards/margins": 0.3118717074394226,
      "rewards/rejected": -0.551956832408905,
      "step": 7590
    },
    {
      "epoch": 0.99,
      "learning_rate": 4.6014739467997725e-10,
      "logits/chosen": -2.493314504623413,
      "logits/rejected": -2.4609227180480957,
      "logps/chosen": -286.0111083984375,
      "logps/rejected": -331.01507568359375,
      "loss": 0.6265,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.29493117332458496,
      "rewards/margins": 0.3773437440395355,
      "rewards/rejected": -0.6722748279571533,
      "step": 7600
    },
    {
      "epoch": 1.0,
      "learning_rate": 2.671184785033032e-10,
      "logits/chosen": -2.6753334999084473,
      "logits/rejected": -2.719071865081787,
      "logps/chosen": -342.070556640625,
      "logps/rejected": -344.14990234375,
      "loss": 0.6253,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.3149510324001312,
      "rewards/margins": 0.22299817204475403,
      "rewards/rejected": -0.5379492044448853,
      "step": 7610
    },
    {
      "epoch": 1.0,
      "learning_rate": 1.2625640403302054e-10,
      "logits/chosen": -2.422921895980835,
      "logits/rejected": -2.439711332321167,
      "logps/chosen": -251.980712890625,
      "logps/rejected": -311.44805908203125,
      "loss": 0.5459,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1920924037694931,
      "rewards/margins": 0.4203423857688904,
      "rewards/rejected": -0.6124347448348999,
      "step": 7620
    },
    {
      "epoch": 1.0,
      "learning_rate": 3.756411091515588e-11,
      "logits/chosen": -2.6092958450317383,
      "logits/rejected": -2.5790138244628906,
      "logps/chosen": -348.45111083984375,
      "logps/rejected": -310.7505798339844,
      "loss": 0.5965,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.21164055168628693,
      "rewards/margins": 0.3577142059803009,
      "rewards/rejected": -0.569354772567749,
      "step": 7630
    },
    {
      "epoch": 1.0,
      "learning_rate": 1.0434500657963143e-12,
      "logits/chosen": -2.7101950645446777,
      "logits/rejected": -2.600151538848877,
      "logps/chosen": -317.154541015625,
      "logps/rejected": -331.2071228027344,
      "loss": 0.5759,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.22717204689979553,
      "rewards/margins": 0.34210166335105896,
      "rewards/rejected": -0.5692737102508545,
      "step": 7640
    },
    {
      "epoch": 1.0,
      "step": 7642,
      "total_flos": 0.0,
      "train_loss": 0.6071469678956156,
      "train_runtime": 31823.939,
      "train_samples_per_second": 1.921,
      "train_steps_per_second": 0.24
    }
  ],
  "logging_steps": 10,
  "max_steps": 7642,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 1000,
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}