trainer_state.json

{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 3821,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 1.3054830287206268e-08,
      "logits/chosen": 0.9550814628601074,
      "logits/rejected": 1.0664727687835693,
      "logps/chosen": -190.47879028320312,
      "logps/rejected": -177.6958770751953,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.0,
      "learning_rate": 1.3054830287206266e-07,
      "logits/chosen": 1.0219794511795044,
      "logits/rejected": 1.074777364730835,
      "logps/chosen": -277.9150695800781,
      "logps/rejected": -268.2773742675781,
      "loss": 0.6933,
      "rewards/accuracies": 0.4097222089767456,
      "rewards/chosen": -0.00020908194710500538,
      "rewards/margins": -0.000234946608543396,
      "rewards/rejected": 2.586471055110451e-05,
      "step": 10
    },
    {
      "epoch": 0.01,
      "learning_rate": 2.610966057441253e-07,
      "logits/chosen": 1.0537651777267456,
      "logits/rejected": 1.0348730087280273,
      "logps/chosen": -258.07086181640625,
      "logps/rejected": -219.53591918945312,
      "loss": 0.6935,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.000874903635121882,
      "rewards/margins": -0.0006632002769038081,
      "rewards/rejected": -0.0002117032854584977,
      "step": 20
    },
    {
      "epoch": 0.01,
      "learning_rate": 3.9164490861618804e-07,
      "logits/chosen": 0.9774575233459473,
      "logits/rejected": 0.9953697919845581,
      "logps/chosen": -234.4716796875,
      "logps/rejected": -216.33425903320312,
      "loss": 0.6932,
      "rewards/accuracies": 0.41874998807907104,
      "rewards/chosen": -0.0008366929250769317,
      "rewards/margins": -0.00013415786088444293,
      "rewards/rejected": -0.0007025349768809974,
      "step": 30
    },
    {
      "epoch": 0.01,
      "learning_rate": 5.221932114882506e-07,
      "logits/chosen": 1.0602939128875732,
      "logits/rejected": 1.0611933469772339,
      "logps/chosen": -269.39129638671875,
      "logps/rejected": -236.54690551757812,
      "loss": 0.6933,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.000759485294111073,
      "rewards/margins": -0.00026061575044877827,
      "rewards/rejected": -0.0004988695727661252,
      "step": 40
    },
    {
      "epoch": 0.01,
      "learning_rate": 6.527415143603135e-07,
      "logits/chosen": 1.012333631515503,
      "logits/rejected": 1.0495545864105225,
      "logps/chosen": -245.17025756835938,
      "logps/rejected": -241.9575653076172,
      "loss": 0.693,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.0008211232488974929,
      "rewards/margins": 0.00028653821209445596,
      "rewards/rejected": -0.0011076615191996098,
      "step": 50
    },
    {
      "epoch": 0.02,
      "learning_rate": 7.832898172323761e-07,
      "logits/chosen": 0.9769388437271118,
      "logits/rejected": 1.0936685800552368,
      "logps/chosen": -283.7574157714844,
      "logps/rejected": -234.1509552001953,
      "loss": 0.6932,
      "rewards/accuracies": 0.4312500059604645,
      "rewards/chosen": -0.00026667636120691895,
      "rewards/margins": -0.00013940571807324886,
      "rewards/rejected": -0.00012727065768558532,
      "step": 60
    },
    {
      "epoch": 0.02,
      "learning_rate": 9.138381201044387e-07,
      "logits/chosen": 1.0060393810272217,
      "logits/rejected": 1.0819052457809448,
      "logps/chosen": -271.9745178222656,
      "logps/rejected": -231.0491943359375,
      "loss": 0.693,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.00024310217122547328,
      "rewards/margins": 0.00035628705518320203,
      "rewards/rejected": -0.00011318484030198306,
      "step": 70
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.0443864229765013e-06,
      "logits/chosen": 1.0235029458999634,
      "logits/rejected": 1.062941312789917,
      "logps/chosen": -283.92523193359375,
      "logps/rejected": -261.73150634765625,
      "loss": 0.6931,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -0.0006150220870040357,
      "rewards/margins": 9.248043352272362e-05,
      "rewards/rejected": -0.0007075025932863355,
      "step": 80
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.1749347258485642e-06,
      "logits/chosen": 1.043299913406372,
      "logits/rejected": 1.0933589935302734,
      "logps/chosen": -278.43609619140625,
      "logps/rejected": -235.748046875,
      "loss": 0.6934,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.000780908390879631,
      "rewards/margins": -0.0005825563566759229,
      "rewards/rejected": -0.00019835206330753863,
      "step": 90
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.305483028720627e-06,
      "logits/chosen": 0.9930270910263062,
      "logits/rejected": 1.068216323852539,
      "logps/chosen": -237.21041870117188,
      "logps/rejected": -218.50753784179688,
      "loss": 0.6925,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": 0.0006625339738093317,
      "rewards/margins": 0.0012667592382058501,
      "rewards/rejected": -0.0006042252061888576,
      "step": 100
    },
    {
      "epoch": 0.03,
      "eval_logits/chosen": 0.9728049635887146,
      "eval_logits/rejected": 1.0653607845306396,
      "eval_logps/chosen": -277.5415954589844,
      "eval_logps/rejected": -243.8911895751953,
      "eval_loss": 0.6927526593208313,
      "eval_rewards/accuracies": 0.4950000047683716,
      "eval_rewards/chosen": 5.226604480412789e-05,
      "eval_rewards/margins": 0.0008048939635045826,
      "eval_rewards/rejected": -0.0007526279077865183,
      "eval_runtime": 545.5397,
      "eval_samples_per_second": 3.666,
      "eval_steps_per_second": 0.917,
      "step": 100
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.4360313315926894e-06,
      "logits/chosen": 0.9969541430473328,
      "logits/rejected": 1.0819013118743896,
      "logps/chosen": -283.55718994140625,
      "logps/rejected": -250.1197967529297,
      "loss": 0.6924,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": 0.0006327680312097073,
      "rewards/margins": 0.0015511559322476387,
      "rewards/rejected": -0.0009183877264149487,
      "step": 110
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.5665796344647521e-06,
      "logits/chosen": 1.0308067798614502,
      "logits/rejected": 1.0752949714660645,
      "logps/chosen": -227.8155517578125,
      "logps/rejected": -234.15908813476562,
      "loss": 0.6929,
      "rewards/accuracies": 0.4312500059604645,
      "rewards/chosen": -6.81330930092372e-05,
      "rewards/margins": 0.00048325079842470586,
      "rewards/rejected": -0.0005513839423656464,
      "step": 120
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.6971279373368146e-06,
      "logits/chosen": 1.049055814743042,
      "logits/rejected": 1.0949084758758545,
      "logps/chosen": -282.67279052734375,
      "logps/rejected": -239.2623291015625,
      "loss": 0.6923,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.00029724548221565783,
      "rewards/margins": 0.001690825680270791,
      "rewards/rejected": -0.0013935801107436419,
      "step": 130
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.8276762402088774e-06,
      "logits/chosen": 1.026393175125122,
      "logits/rejected": 1.026254415512085,
      "logps/chosen": -264.0199279785156,
      "logps/rejected": -237.1195831298828,
      "loss": 0.6922,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.0003116951265837997,
      "rewards/margins": 0.0020114013459533453,
      "rewards/rejected": -0.001699706306681037,
      "step": 140
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.9582245430809403e-06,
      "logits/chosen": 1.0078837871551514,
      "logits/rejected": 1.0274178981781006,
      "logps/chosen": -262.63385009765625,
      "logps/rejected": -235.02658081054688,
      "loss": 0.6919,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": 0.0008050563046708703,
      "rewards/margins": 0.002434623194858432,
      "rewards/rejected": -0.0016295671230182052,
      "step": 150
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.0887728459530026e-06,
      "logits/chosen": 0.9562094807624817,
      "logits/rejected": 1.085970163345337,
      "logps/chosen": -258.23858642578125,
      "logps/rejected": -240.0513916015625,
      "loss": 0.6921,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0007125878473743796,
      "rewards/margins": 0.002117170486599207,
      "rewards/rejected": -0.0014045826392248273,
      "step": 160
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.2193211488250653e-06,
      "logits/chosen": 0.9959365129470825,
      "logits/rejected": 1.0712645053863525,
      "logps/chosen": -268.434814453125,
      "logps/rejected": -218.3533172607422,
      "loss": 0.6912,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0012755919015035033,
      "rewards/margins": 0.0039804959669709206,
      "rewards/rejected": -0.0027049046475440264,
      "step": 170
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.3498694516971284e-06,
      "logits/chosen": 0.9840243458747864,
      "logits/rejected": 1.0322699546813965,
      "logps/chosen": -272.4624938964844,
      "logps/rejected": -237.8259735107422,
      "loss": 0.6914,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": 0.0022116941399872303,
      "rewards/margins": 0.00362318754196167,
      "rewards/rejected": -0.0014114934019744396,
      "step": 180
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.4804177545691907e-06,
      "logits/chosen": 0.9921697378158569,
      "logits/rejected": 1.0103299617767334,
      "logps/chosen": -269.4023742675781,
      "logps/rejected": -235.68417358398438,
      "loss": 0.6903,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": 0.002223977353423834,
      "rewards/margins": 0.005811163689941168,
      "rewards/rejected": -0.003587186336517334,
      "step": 190
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.610966057441254e-06,
      "logits/chosen": 1.0245262384414673,
      "logits/rejected": 1.1350899934768677,
      "logps/chosen": -277.9847412109375,
      "logps/rejected": -249.67733764648438,
      "loss": 0.6903,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": 0.005080274306237698,
      "rewards/margins": 0.00588814215734601,
      "rewards/rejected": -0.0008078686660155654,
      "step": 200
    },
    {
      "epoch": 0.05,
      "eval_logits/chosen": 0.9732384085655212,
      "eval_logits/rejected": 1.0658739805221558,
      "eval_logps/chosen": -277.0526123046875,
      "eval_logps/rejected": -243.96609497070312,
      "eval_loss": 0.6899830102920532,
      "eval_rewards/accuracies": 0.5830000042915344,
      "eval_rewards/chosen": 0.004941860679537058,
      "eval_rewards/margins": 0.006443744525313377,
      "eval_rewards/rejected": -0.0015018840786069632,
      "eval_runtime": 544.739,
      "eval_samples_per_second": 3.671,
      "eval_steps_per_second": 0.918,
      "step": 200
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.741514360313316e-06,
      "logits/chosen": 1.01772940158844,
      "logits/rejected": 1.0974925756454468,
      "logps/chosen": -260.6138610839844,
      "logps/rejected": -233.33798217773438,
      "loss": 0.6896,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": 0.00463427510112524,
      "rewards/margins": 0.007291147019714117,
      "rewards/rejected": -0.0026568726170808077,
      "step": 210
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.872062663185379e-06,
      "logits/chosen": 1.0488287210464478,
      "logits/rejected": 1.1195874214172363,
      "logps/chosen": -277.33319091796875,
      "logps/rejected": -243.1424560546875,
      "loss": 0.6894,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": 0.006252074148505926,
      "rewards/margins": 0.007730481214821339,
      "rewards/rejected": -0.001478406717069447,
      "step": 220
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.0026109660574416e-06,
      "logits/chosen": 1.0392377376556396,
      "logits/rejected": 1.0487968921661377,
      "logps/chosen": -268.562255859375,
      "logps/rejected": -275.25787353515625,
      "loss": 0.6882,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.008288154378533363,
      "rewards/margins": 0.010209737345576286,
      "rewards/rejected": -0.0019215833162888885,
      "step": 230
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.1331592689295043e-06,
      "logits/chosen": 1.0380384922027588,
      "logits/rejected": 1.0580257177352905,
      "logps/chosen": -271.28057861328125,
      "logps/rejected": -231.651123046875,
      "loss": 0.6849,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.015278227627277374,
      "rewards/margins": 0.016800139099359512,
      "rewards/rejected": -0.0015219092601910233,
      "step": 240
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.263707571801567e-06,
      "logits/chosen": 0.9396646618843079,
      "logits/rejected": 1.0964558124542236,
      "logps/chosen": -261.69482421875,
      "logps/rejected": -207.1199188232422,
      "loss": 0.6867,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": 0.011847788468003273,
      "rewards/margins": 0.01328262872993946,
      "rewards/rejected": -0.0014348386321216822,
      "step": 250
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.3942558746736293e-06,
      "logits/chosen": 1.0348241329193115,
      "logits/rejected": 1.0277621746063232,
      "logps/chosen": -255.30850219726562,
      "logps/rejected": -249.1978759765625,
      "loss": 0.6871,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.01062377355992794,
      "rewards/margins": 0.01277032308280468,
      "rewards/rejected": -0.002146549290046096,
      "step": 260
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.524804177545692e-06,
      "logits/chosen": 0.9442011713981628,
      "logits/rejected": 1.091737985610962,
      "logps/chosen": -249.87490844726562,
      "logps/rejected": -225.23934936523438,
      "loss": 0.6862,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.009906780906021595,
      "rewards/margins": 0.014515595510601997,
      "rewards/rejected": -0.004608814604580402,
      "step": 270
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.6553524804177547e-06,
      "logits/chosen": 0.9841794967651367,
      "logits/rejected": 1.0294774770736694,
      "logps/chosen": -261.4816589355469,
      "logps/rejected": -244.85580444335938,
      "loss": 0.6821,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": 0.01882908120751381,
      "rewards/margins": 0.023043904453516006,
      "rewards/rejected": -0.004214824177324772,
      "step": 280
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.7859007832898174e-06,
      "logits/chosen": 0.9971128702163696,
      "logits/rejected": 1.0910747051239014,
      "logps/chosen": -257.55096435546875,
      "logps/rejected": -229.0856170654297,
      "loss": 0.6801,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02020946703851223,
      "rewards/margins": 0.027215411886572838,
      "rewards/rejected": -0.007005943916738033,
      "step": 290
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.9164490861618806e-06,
      "logits/chosen": 0.972184956073761,
      "logits/rejected": 1.0939315557479858,
      "logps/chosen": -256.2604064941406,
      "logps/rejected": -227.18539428710938,
      "loss": 0.682,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.01802998222410679,
      "rewards/margins": 0.023866988718509674,
      "rewards/rejected": -0.0058370064944028854,
      "step": 300
    },
    {
      "epoch": 0.08,
      "eval_logits/chosen": 1.0022695064544678,
      "eval_logits/rejected": 1.097358226776123,
      "eval_logps/chosen": -275.3941345214844,
      "eval_logps/rejected": -244.4587860107422,
      "eval_loss": 0.6800673604011536,
      "eval_rewards/accuracies": 0.6054999828338623,
      "eval_rewards/chosen": 0.021527061238884926,
      "eval_rewards/margins": 0.027955830097198486,
      "eval_rewards/rejected": -0.006428766064345837,
      "eval_runtime": 545.3023,
      "eval_samples_per_second": 3.668,
      "eval_steps_per_second": 0.917,
      "step": 300
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.046997389033943e-06,
      "logits/chosen": 0.9668880701065063,
      "logits/rejected": 1.1288020610809326,
      "logps/chosen": -255.2298583984375,
      "logps/rejected": -227.0789337158203,
      "loss": 0.6769,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.026618385687470436,
      "rewards/margins": 0.03431355208158493,
      "rewards/rejected": -0.0076951696537435055,
      "step": 310
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.177545691906005e-06,
      "logits/chosen": 0.9519135355949402,
      "logits/rejected": 1.0828487873077393,
      "logps/chosen": -281.8594970703125,
      "logps/rejected": -257.09490966796875,
      "loss": 0.6821,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.01871672458946705,
      "rewards/margins": 0.024688560515642166,
      "rewards/rejected": -0.00597183546051383,
      "step": 320
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.308093994778068e-06,
      "logits/chosen": 1.033220648765564,
      "logits/rejected": 1.1434317827224731,
      "logps/chosen": -278.22552490234375,
      "logps/rejected": -244.3297119140625,
      "loss": 0.6805,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.010118372738361359,
      "rewards/margins": 0.028245192021131516,
      "rewards/rejected": -0.018126821145415306,
      "step": 330
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.4386422976501306e-06,
      "logits/chosen": 1.0310008525848389,
      "logits/rejected": 1.0641546249389648,
      "logps/chosen": -273.8054504394531,
      "logps/rejected": -269.79022216796875,
      "loss": 0.6784,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.008210320957005024,
      "rewards/margins": 0.03238377720117569,
      "rewards/rejected": -0.02417345717549324,
      "step": 340
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.569190600522193e-06,
      "logits/chosen": 1.0388667583465576,
      "logits/rejected": 1.034010887145996,
      "logps/chosen": -284.13043212890625,
      "logps/rejected": -252.32052612304688,
      "loss": 0.6737,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.012403028085827827,
      "rewards/margins": 0.042349107563495636,
      "rewards/rejected": -0.02994607947766781,
      "step": 350
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.699738903394257e-06,
      "logits/chosen": 1.0033949613571167,
      "logits/rejected": 1.097804307937622,
      "logps/chosen": -302.3703918457031,
      "logps/rejected": -261.0248718261719,
      "loss": 0.6727,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.0027267406694591045,
      "rewards/margins": 0.04591723158955574,
      "rewards/rejected": -0.04319049045443535,
      "step": 360
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.8302872062663196e-06,
      "logits/chosen": 1.0511659383773804,
      "logits/rejected": 1.042121171951294,
      "logps/chosen": -305.19305419921875,
      "logps/rejected": -264.2236328125,
      "loss": 0.6659,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.007779981009662151,
      "rewards/margins": 0.06012594699859619,
      "rewards/rejected": -0.052345968782901764,
      "step": 370
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.9608355091383814e-06,
      "logits/chosen": 1.019500970840454,
      "logits/rejected": 1.1177808046340942,
      "logps/chosen": -256.4404296875,
      "logps/rejected": -228.1063232421875,
      "loss": 0.6635,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.001956403721123934,
      "rewards/margins": 0.06523159891366959,
      "rewards/rejected": -0.06718799471855164,
      "step": 380
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.9999488562447675e-06,
      "logits/chosen": 1.0209654569625854,
      "logits/rejected": 1.0544273853302002,
      "logps/chosen": -299.9403991699219,
      "logps/rejected": -262.7241516113281,
      "loss": 0.6558,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.0010883864015340805,
      "rewards/margins": 0.08589887619018555,
      "rewards/rejected": -0.08698725700378418,
      "step": 390
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999698361256577e-06,
      "logits/chosen": 1.0261785984039307,
      "logits/rejected": 1.0322299003601074,
      "logps/chosen": -282.60333251953125,
      "logps/rejected": -271.5989685058594,
      "loss": 0.6574,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.03285291790962219,
      "rewards/margins": 0.0808914452791214,
      "rewards/rejected": -0.11374436318874359,
      "step": 400
    },
    {
      "epoch": 0.1,
      "eval_logits/chosen": 0.9584712982177734,
      "eval_logits/rejected": 1.0540906190872192,
      "eval_logps/chosen": -282.07501220703125,
      "eval_logps/rejected": -255.61891174316406,
      "eval_loss": 0.6623440384864807,
      "eval_rewards/accuracies": 0.6054999828338623,
      "eval_rewards/chosen": -0.045281875878572464,
      "eval_rewards/margins": 0.07274789363145828,
      "eval_rewards/rejected": -0.11802978068590164,
      "eval_runtime": 544.6934,
      "eval_samples_per_second": 3.672,
      "eval_steps_per_second": 0.918,
      "step": 400
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.999239142174581e-06,
      "logits/chosen": 0.9255741834640503,
      "logits/rejected": 0.9569520950317383,
      "logps/chosen": -298.91546630859375,
      "logps/rejected": -255.8893280029297,
      "loss": 0.6584,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.053911127150058746,
      "rewards/margins": 0.07991180568933487,
      "rewards/rejected": -0.13382293283939362,
      "step": 410
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.99857123734344e-06,
      "logits/chosen": 0.9512017965316772,
      "logits/rejected": 1.0617644786834717,
      "logps/chosen": -285.84527587890625,
      "logps/rejected": -258.625732421875,
      "loss": 0.6592,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07313002645969391,
      "rewards/margins": 0.08329417556524277,
      "rewards/rejected": -0.15642420947551727,
      "step": 420
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.997694702533016e-06,
      "logits/chosen": 0.8519557118415833,
      "logits/rejected": 0.9811371564865112,
      "logps/chosen": -277.22845458984375,
      "logps/rejected": -276.25067138671875,
      "loss": 0.6667,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.13439401984214783,
      "rewards/margins": 0.07216326892375946,
      "rewards/rejected": -0.20655731856822968,
      "step": 430
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.996609610933713e-06,
      "logits/chosen": 0.9116706848144531,
      "logits/rejected": 0.8976380228996277,
      "logps/chosen": -296.78948974609375,
      "logps/rejected": -279.92169189453125,
      "loss": 0.6649,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.14868149161338806,
      "rewards/margins": 0.07609296590089798,
      "rewards/rejected": -0.22477443516254425,
      "step": 440
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.995316053150366e-06,
      "logits/chosen": 0.8733074069023132,
      "logits/rejected": 0.9674992561340332,
      "logps/chosen": -276.9858093261719,
      "logps/rejected": -256.2315673828125,
      "loss": 0.6635,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.15514853596687317,
      "rewards/margins": 0.07954682409763336,
      "rewards/rejected": -0.23469536006450653,
      "step": 450
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.9938141371946815e-06,
      "logits/chosen": 0.8927382230758667,
      "logits/rejected": 1.0055015087127686,
      "logps/chosen": -265.1849670410156,
      "logps/rejected": -274.2850036621094,
      "loss": 0.643,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.1328841745853424,
      "rewards/margins": 0.12832005321979523,
      "rewards/rejected": -0.26120424270629883,
      "step": 460
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.992103988476206e-06,
      "logits/chosen": 0.8474317789077759,
      "logits/rejected": 0.9396875500679016,
      "logps/chosen": -310.89520263671875,
      "logps/rejected": -272.8279113769531,
      "loss": 0.6472,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.15546834468841553,
      "rewards/margins": 0.11896806955337524,
      "rewards/rejected": -0.27443641424179077,
      "step": 470
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.990185749791866e-06,
      "logits/chosen": 0.7985225915908813,
      "logits/rejected": 0.8882298469543457,
      "logps/chosen": -259.30877685546875,
      "logps/rejected": -238.6549530029297,
      "loss": 0.6418,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.1691729873418808,
      "rewards/margins": 0.13572049140930176,
      "rewards/rejected": -0.30489346385002136,
      "step": 480
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.9880595813140395e-06,
      "logits/chosen": 0.7800701856613159,
      "logits/rejected": 0.8966795206069946,
      "logps/chosen": -295.586181640625,
      "logps/rejected": -269.9721374511719,
      "loss": 0.6344,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.20812523365020752,
      "rewards/margins": 0.15324506163597107,
      "rewards/rejected": -0.361370325088501,
      "step": 490
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.985725660577184e-06,
      "logits/chosen": 0.8370100259780884,
      "logits/rejected": 0.8300473093986511,
      "logps/chosen": -268.3841247558594,
      "logps/rejected": -272.91815185546875,
      "loss": 0.6262,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.2489767074584961,
      "rewards/margins": 0.1822068989276886,
      "rewards/rejected": -0.4311836361885071,
      "step": 500
    },
    {
      "epoch": 0.13,
      "eval_logits/chosen": 0.7187003493309021,
      "eval_logits/rejected": 0.7971898317337036,
      "eval_logps/chosen": -310.10272216796875,
      "eval_logps/rejected": -292.38580322265625,
      "eval_loss": 0.6406594514846802,
      "eval_rewards/accuracies": 0.6044999957084656,
      "eval_rewards/chosen": -0.32555925846099854,
      "eval_rewards/margins": 0.16013950109481812,
      "eval_rewards/rejected": -0.48569872975349426,
      "eval_runtime": 545.4957,
      "eval_samples_per_second": 3.666,
      "eval_steps_per_second": 0.917,
      "step": 500
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.983184182463009e-06,
      "logits/chosen": 0.6981757879257202,
      "logits/rejected": 0.8250112533569336,
      "logps/chosen": -314.55950927734375,
      "logps/rejected": -264.6604919433594,
      "loss": 0.6356,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.31188660860061646,
      "rewards/margins": 0.15816518664360046,
      "rewards/rejected": -0.4700518548488617,
      "step": 510
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.980435359184203e-06,
      "logits/chosen": 0.7419403791427612,
      "logits/rejected": 0.8123974800109863,
      "logps/chosen": -326.6161193847656,
      "logps/rejected": -308.69049072265625,
      "loss": 0.6274,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.3839537501335144,
      "rewards/margins": 0.20789018273353577,
      "rewards/rejected": -0.5918439626693726,
      "step": 520
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.9774794202667236e-06,
      "logits/chosen": 0.6873521208763123,
      "logits/rejected": 0.7603924870491028,
      "logps/chosen": -304.0896301269531,
      "logps/rejected": -290.05841064453125,
      "loss": 0.6251,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.3807668089866638,
      "rewards/margins": 0.1975090503692627,
      "rewards/rejected": -0.5782758593559265,
      "step": 530
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.974316612530615e-06,
      "logits/chosen": 0.6311649084091187,
      "logits/rejected": 0.6694685816764832,
      "logps/chosen": -272.9194641113281,
      "logps/rejected": -275.1445007324219,
      "loss": 0.6065,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.3134765625,
      "rewards/margins": 0.24742670357227325,
      "rewards/rejected": -0.5609032511711121,
      "step": 540
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.970947200069416e-06,
      "logits/chosen": 0.6108866333961487,
      "logits/rejected": 0.6520397663116455,
      "logps/chosen": -301.78326416015625,
      "logps/rejected": -287.75848388671875,
      "loss": 0.6147,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.3902989625930786,
      "rewards/margins": 0.26976609230041504,
      "rewards/rejected": -0.6600649952888489,
      "step": 550
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.967371464228096e-06,
      "logits/chosen": 0.62494957447052,
      "logits/rejected": 0.6929062604904175,
      "logps/chosen": -332.30120849609375,
      "logps/rejected": -324.767333984375,
      "loss": 0.6461,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.4859549105167389,
      "rewards/margins": 0.181377574801445,
      "rewards/rejected": -0.6673325300216675,
      "step": 560
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.963589703579569e-06,
      "logits/chosen": 0.568712592124939,
      "logits/rejected": 0.7027269601821899,
      "logps/chosen": -308.6661376953125,
      "logps/rejected": -291.0648498535156,
      "loss": 0.5775,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.407268762588501,
      "rewards/margins": 0.34008973836898804,
      "rewards/rejected": -0.747358500957489,
      "step": 570
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.9596022338997615e-06,
      "logits/chosen": 0.5451105833053589,
      "logits/rejected": 0.532687246799469,
      "logps/chosen": -301.73736572265625,
      "logps/rejected": -290.80987548828125,
      "loss": 0.6317,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.4523653984069824,
      "rewards/margins": 0.20574729144573212,
      "rewards/rejected": -0.6581127047538757,
      "step": 580
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.955409388141243e-06,
      "logits/chosen": 0.5721498131752014,
      "logits/rejected": 0.5243967771530151,
      "logps/chosen": -310.00701904296875,
      "logps/rejected": -296.6248779296875,
      "loss": 0.6067,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.42185163497924805,
      "rewards/margins": 0.2717475891113281,
      "rewards/rejected": -0.6935992240905762,
      "step": 590
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.951011516405429e-06,
      "logits/chosen": 0.5027920007705688,
      "logits/rejected": 0.5034610033035278,
      "logps/chosen": -286.9452209472656,
      "logps/rejected": -324.5048828125,
      "loss": 0.6441,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.5031018257141113,
      "rewards/margins": 0.1803509145975113,
      "rewards/rejected": -0.6834527254104614,
      "step": 600
    },
    {
      "epoch": 0.16,
      "eval_logits/chosen": 0.44342371821403503,
      "eval_logits/rejected": 0.5041319727897644,
      "eval_logps/chosen": -327.3852233886719,
      "eval_logps/rejected": -317.38275146484375,
      "eval_loss": 0.6309801340103149,
      "eval_rewards/accuracies": 0.6039999723434448,
      "eval_rewards/chosen": -0.4983837902545929,
      "eval_rewards/margins": 0.23728469014167786,
      "eval_rewards/rejected": -0.7356684803962708,
      "eval_runtime": 544.5805,
      "eval_samples_per_second": 3.673,
      "eval_steps_per_second": 0.918,
      "step": 600
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.946408985913344e-06,
      "logits/chosen": 0.4787842333316803,
      "logits/rejected": 0.49609264731407166,
      "logps/chosen": -300.0178527832031,
      "logps/rejected": -288.23297119140625,
      "loss": 0.6733,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.5682172775268555,
      "rewards/margins": 0.13463091850280762,
      "rewards/rejected": -0.7028483152389526,
      "step": 610
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.941602180974958e-06,
      "logits/chosen": 0.48600974678993225,
      "logits/rejected": 0.5271373987197876,
      "logps/chosen": -327.2370300292969,
      "logps/rejected": -296.0666198730469,
      "loss": 0.6103,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4133662283420563,
      "rewards/margins": 0.28371673822402954,
      "rewards/rejected": -0.6970829963684082,
      "step": 620
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.936591502957101e-06,
      "logits/chosen": 0.43321290612220764,
      "logits/rejected": 0.5329369306564331,
      "logps/chosen": -335.93365478515625,
      "logps/rejected": -309.68084716796875,
      "loss": 0.6186,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.3893541097640991,
      "rewards/margins": 0.21971730887889862,
      "rewards/rejected": -0.6090713739395142,
      "step": 630
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.931377370249946e-06,
      "logits/chosen": 0.5257896184921265,
      "logits/rejected": 0.5845480561256409,
      "logps/chosen": -317.484619140625,
      "logps/rejected": -291.53948974609375,
      "loss": 0.618,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.4368332028388977,
      "rewards/margins": 0.21368706226348877,
      "rewards/rejected": -0.6505202054977417,
      "step": 640
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.925960218232073e-06,
      "logits/chosen": 0.5211079716682434,
      "logits/rejected": 0.5993599891662598,
      "logps/chosen": -356.9900207519531,
      "logps/rejected": -359.18939208984375,
      "loss": 0.5982,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.540452778339386,
      "rewards/margins": 0.29875844717025757,
      "rewards/rejected": -0.8392112851142883,
      "step": 650
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.920340499234116e-06,
      "logits/chosen": 0.4190196394920349,
      "logits/rejected": 0.5146626234054565,
      "logps/chosen": -337.7661437988281,
      "logps/rejected": -283.05902099609375,
      "loss": 0.6362,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.5408000349998474,
      "rewards/margins": 0.21028542518615723,
      "rewards/rejected": -0.7510854005813599,
      "step": 660
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.914518682500995e-06,
      "logits/chosen": 0.5096255540847778,
      "logits/rejected": 0.5135337114334106,
      "logps/chosen": -328.9342346191406,
      "logps/rejected": -347.03399658203125,
      "loss": 0.6074,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.51976478099823,
      "rewards/margins": 0.2629929780960083,
      "rewards/rejected": -0.7827577590942383,
      "step": 670
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.9084952541527315e-06,
      "logits/chosen": 0.4741145968437195,
      "logits/rejected": 0.4941348135471344,
      "logps/chosen": -293.9762268066406,
      "logps/rejected": -308.02655029296875,
      "loss": 0.5911,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.39032864570617676,
      "rewards/margins": 0.3133639395236969,
      "rewards/rejected": -0.7036925554275513,
      "step": 680
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.902270717143858e-06,
      "logits/chosen": 0.44331198930740356,
      "logits/rejected": 0.4858936667442322,
      "logps/chosen": -326.63909912109375,
      "logps/rejected": -310.3343505859375,
      "loss": 0.6161,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.41057485342025757,
      "rewards/margins": 0.2567914128303528,
      "rewards/rejected": -0.6673663258552551,
      "step": 690
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.895845591221427e-06,
      "logits/chosen": 0.3671930730342865,
      "logits/rejected": 0.47033652663230896,
      "logps/chosen": -300.76873779296875,
      "logps/rejected": -305.1224060058594,
      "loss": 0.6238,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.537026047706604,
      "rewards/margins": 0.2782416045665741,
      "rewards/rejected": -0.8152676820755005,
      "step": 700
    },
    {
      "epoch": 0.18,
      "eval_logits/chosen": 0.414013147354126,
      "eval_logits/rejected": 0.4768453538417816,
      "eval_logps/chosen": -328.9062805175781,
      "eval_logps/rejected": -321.1136779785156,
      "eval_loss": 0.6180335879325867,
      "eval_rewards/accuracies": 0.6175000071525574,
      "eval_rewards/chosen": -0.5135945081710815,
      "eval_rewards/margins": 0.25938284397125244,
      "eval_rewards/rejected": -0.7729774117469788,
      "eval_runtime": 544.9185,
      "eval_samples_per_second": 3.67,
      "eval_steps_per_second": 0.918,
      "step": 700
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.8892204128816e-06,
      "logits/chosen": 0.4711977541446686,
      "logits/rejected": 0.5027458071708679,
      "logps/chosen": -335.2975769042969,
      "logps/rejected": -362.6507873535156,
      "loss": 0.6061,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.5243827700614929,
      "rewards/margins": 0.3451407551765442,
      "rewards/rejected": -0.8695236444473267,
      "step": 710
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.882395735324864e-06,
      "logits/chosen": 0.37723851203918457,
      "logits/rejected": 0.37279143929481506,
      "logps/chosen": -324.41204833984375,
      "logps/rejected": -320.57904052734375,
      "loss": 0.6097,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.4609324038028717,
      "rewards/margins": 0.25949710607528687,
      "rewards/rejected": -0.720429539680481,
      "step": 720
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.87537212840983e-06,
      "logits/chosen": 0.41040197014808655,
      "logits/rejected": 0.4262399673461914,
      "logps/chosen": -353.74359130859375,
      "logps/rejected": -339.08740234375,
      "loss": 0.5831,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.5424319505691528,
      "rewards/margins": 0.3541128635406494,
      "rewards/rejected": -0.8965448141098022,
      "step": 730
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.8681501786056545e-06,
      "logits/chosen": 0.37297043204307556,
      "logits/rejected": 0.42292696237564087,
      "logps/chosen": -331.2789306640625,
      "logps/rejected": -298.3685302734375,
      "loss": 0.6372,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.613556981086731,
      "rewards/margins": 0.22487401962280273,
      "rewards/rejected": -0.8384310007095337,
      "step": 740
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.860730488943068e-06,
      "logits/chosen": 0.4059647023677826,
      "logits/rejected": 0.41552990674972534,
      "logps/chosen": -330.83514404296875,
      "logps/rejected": -325.4556884765625,
      "loss": 0.5811,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5344411134719849,
      "rewards/margins": 0.34030893445014954,
      "rewards/rejected": -0.8747501373291016,
      "step": 750
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.853113678964022e-06,
      "logits/chosen": 0.4156310558319092,
      "logits/rejected": 0.4202690124511719,
      "logps/chosen": -309.32623291015625,
      "logps/rejected": -325.4747619628906,
      "loss": 0.5772,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.6245480179786682,
      "rewards/margins": 0.3636637330055237,
      "rewards/rejected": -0.9882117509841919,
      "step": 760
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.845300384669958e-06,
      "logits/chosen": 0.36242616176605225,
      "logits/rejected": 0.37326696515083313,
      "logps/chosen": -326.7525939941406,
      "logps/rejected": -303.91986083984375,
      "loss": 0.6412,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.6412128210067749,
      "rewards/margins": 0.229770228266716,
      "rewards/rejected": -0.8709830045700073,
      "step": 770
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.837291258468701e-06,
      "logits/chosen": 0.3382135033607483,
      "logits/rejected": 0.34883031249046326,
      "logps/chosen": -334.1026306152344,
      "logps/rejected": -330.28582763671875,
      "loss": 0.5985,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5876267552375793,
      "rewards/margins": 0.3265851140022278,
      "rewards/rejected": -0.9142118692398071,
      "step": 780
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.829086969119984e-06,
      "logits/chosen": 0.305641233921051,
      "logits/rejected": 0.3537415862083435,
      "logps/chosen": -300.503662109375,
      "logps/rejected": -319.2359313964844,
      "loss": 0.6035,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5569041967391968,
      "rewards/margins": 0.3058740198612213,
      "rewards/rejected": -0.8627783060073853,
      "step": 790
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.820688201679605e-06,
      "logits/chosen": 0.3215860426425934,
      "logits/rejected": 0.36162325739860535,
      "logps/chosen": -309.1678161621094,
      "logps/rejected": -305.80645751953125,
      "loss": 0.6022,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.5132586359977722,
      "rewards/margins": 0.2897386848926544,
      "rewards/rejected": -0.8029972910881042,
      "step": 800
    },
    {
      "epoch": 0.21,
      "eval_logits/chosen": 0.2919526994228363,
      "eval_logits/rejected": 0.3468739986419678,
      "eval_logps/chosen": -333.6270751953125,
      "eval_logps/rejected": -329.4936828613281,
      "eval_loss": 0.6146319508552551,
      "eval_rewards/accuracies": 0.609499990940094,
      "eval_rewards/chosen": -0.5608024001121521,
      "eval_rewards/margins": 0.29597532749176025,
      "eval_rewards/rejected": -0.8567777276039124,
      "eval_runtime": 544.2266,
      "eval_samples_per_second": 3.675,
      "eval_steps_per_second": 0.919,
      "step": 800
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.8120956574422315e-06,
      "logits/chosen": 0.3398824632167816,
      "logits/rejected": 0.329068660736084,
      "logps/chosen": -338.9511413574219,
      "logps/rejected": -359.6614074707031,
      "loss": 0.6156,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5863468050956726,
      "rewards/margins": 0.282352089881897,
      "rewards/rejected": -0.8686988949775696,
      "step": 810
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.803310053882831e-06,
      "logits/chosen": 0.3125428557395935,
      "logits/rejected": 0.33215147256851196,
      "logps/chosen": -261.2579345703125,
      "logps/rejected": -275.7474365234375,
      "loss": 0.6258,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.5128569602966309,
      "rewards/margins": 0.28050488233566284,
      "rewards/rejected": -0.7933619022369385,
      "step": 820
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.794332124596775e-06,
      "logits/chosen": 0.31949958205223083,
      "logits/rejected": 0.37735581398010254,
      "logps/chosen": -333.3515625,
      "logps/rejected": -333.65374755859375,
      "loss": 0.6235,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.5982740521430969,
      "rewards/margins": 0.2791600823402405,
      "rewards/rejected": -0.8774340748786926,
      "step": 830
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.785162619238575e-06,
      "logits/chosen": 0.36345189809799194,
      "logits/rejected": 0.39270249009132385,
      "logps/chosen": -318.712890625,
      "logps/rejected": -333.59381103515625,
      "loss": 0.588,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5787845849990845,
      "rewards/margins": 0.36206668615341187,
      "rewards/rejected": -0.9408512115478516,
      "step": 840
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.775802303459288e-06,
      "logits/chosen": 0.3391379714012146,
      "logits/rejected": 0.42467838525772095,
      "logps/chosen": -346.51043701171875,
      "logps/rejected": -349.189453125,
      "loss": 0.5874,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.595561683177948,
      "rewards/margins": 0.36882907152175903,
      "rewards/rejected": -0.9643908739089966,
      "step": 850
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.766251958842589e-06,
      "logits/chosen": 0.3606031835079193,
      "logits/rejected": 0.39103537797927856,
      "logps/chosen": -363.67010498046875,
      "logps/rejected": -356.0177001953125,
      "loss": 0.5942,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.6311839818954468,
      "rewards/margins": 0.330700546503067,
      "rewards/rejected": -0.9618844985961914,
      "step": 860
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.7565123828395066e-06,
      "logits/chosen": 0.3178097605705261,
      "logits/rejected": 0.42083635926246643,
      "logps/chosen": -368.1545715332031,
      "logps/rejected": -372.1910095214844,
      "loss": 0.591,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.5468029975891113,
      "rewards/margins": 0.32849472761154175,
      "rewards/rejected": -0.8752977252006531,
      "step": 870
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.746584388701831e-06,
      "logits/chosen": 0.33455774188041687,
      "logits/rejected": 0.43152570724487305,
      "logps/chosen": -313.480712890625,
      "logps/rejected": -316.8352355957031,
      "loss": 0.5702,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.5729404091835022,
      "rewards/margins": 0.43620842695236206,
      "rewards/rejected": -1.0091488361358643,
      "step": 880
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.736468805414218e-06,
      "logits/chosen": 0.3555249571800232,
      "logits/rejected": 0.4613436758518219,
      "logps/chosen": -347.31915283203125,
      "logps/rejected": -359.3041076660156,
      "loss": 0.5614,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5481359362602234,
      "rewards/margins": 0.4673057496547699,
      "rewards/rejected": -1.015441656112671,
      "step": 890
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.7261664776249595e-06,
      "logits/chosen": 0.31781864166259766,
      "logits/rejected": 0.32524436712265015,
      "logps/chosen": -334.6598205566406,
      "logps/rejected": -352.02569580078125,
      "loss": 0.5893,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.6611984372138977,
      "rewards/margins": 0.40746697783470154,
      "rewards/rejected": -1.0686652660369873,
      "step": 900
    },
    {
      "epoch": 0.24,
      "eval_logits/chosen": 0.25759702920913696,
      "eval_logits/rejected": 0.31359511613845825,
      "eval_logps/chosen": -344.19696044921875,
      "eval_logps/rejected": -343.95404052734375,
      "eval_loss": 0.6058722138404846,
      "eval_rewards/accuracies": 0.6169999837875366,
      "eval_rewards/chosen": -0.6665017008781433,
      "eval_rewards/margins": 0.3348793387413025,
      "eval_rewards/rejected": -1.0013810396194458,
      "eval_runtime": 545.3642,
      "eval_samples_per_second": 3.667,
      "eval_steps_per_second": 0.917,
      "step": 900
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.715678265575463e-06,
      "logits/chosen": 0.29794925451278687,
      "logits/rejected": 0.34237638115882874,
      "logps/chosen": -375.47698974609375,
      "logps/rejected": -381.877685546875,
      "loss": 0.6336,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.6799601912498474,
      "rewards/margins": 0.27002081274986267,
      "rewards/rejected": -0.9499810338020325,
      "step": 910
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.705005045028415e-06,
      "logits/chosen": 0.32408756017684937,
      "logits/rejected": 0.34640252590179443,
      "logps/chosen": -400.0458068847656,
      "logps/rejected": -393.19244384765625,
      "loss": 0.5575,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.6836780905723572,
      "rewards/margins": 0.46534833312034607,
      "rewards/rejected": -1.1490265130996704,
      "step": 920
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.694147707194659e-06,
      "logits/chosen": 0.24671033024787903,
      "logits/rejected": 0.2881634533405304,
      "logps/chosen": -361.03912353515625,
      "logps/rejected": -349.8355712890625,
      "loss": 0.5887,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7573670744895935,
      "rewards/margins": 0.39362633228302,
      "rewards/rejected": -1.1509934663772583,
      "step": 930
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.683107158658782e-06,
      "logits/chosen": 0.19585518538951874,
      "logits/rejected": 0.19908159971237183,
      "logps/chosen": -330.7119445800781,
      "logps/rejected": -334.669921875,
      "loss": 0.5961,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.6968773603439331,
      "rewards/margins": 0.3805212378501892,
      "rewards/rejected": -1.077398657798767,
      "step": 940
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.671884321303407e-06,
      "logits/chosen": 0.23753933608531952,
      "logits/rejected": 0.23777754604816437,
      "logps/chosen": -348.0587158203125,
      "logps/rejected": -371.2275695800781,
      "loss": 0.5923,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.6319106817245483,
      "rewards/margins": 0.4165875315666199,
      "rewards/rejected": -1.048498272895813,
      "step": 950
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.660480132232224e-06,
      "logits/chosen": 0.2047911435365677,
      "logits/rejected": 0.23187002539634705,
      "logps/chosen": -346.5743713378906,
      "logps/rejected": -345.38970947265625,
      "loss": 0.6187,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.7073361873626709,
      "rewards/margins": 0.31164541840553284,
      "rewards/rejected": -1.018981695175171,
      "step": 960
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.6488955436917414e-06,
      "logits/chosen": 0.2647855281829834,
      "logits/rejected": 0.3133198320865631,
      "logps/chosen": -343.3387756347656,
      "logps/rejected": -299.07171630859375,
      "loss": 0.6214,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.6582953333854675,
      "rewards/margins": 0.22357884049415588,
      "rewards/rejected": -0.8818742036819458,
      "step": 970
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.6371315229917644e-06,
      "logits/chosen": 0.2728188931941986,
      "logits/rejected": 0.3087821900844574,
      "logps/chosen": -320.83038330078125,
      "logps/rejected": -330.16864013671875,
      "loss": 0.5954,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.5822576284408569,
      "rewards/margins": 0.3170580267906189,
      "rewards/rejected": -0.8993157148361206,
      "step": 980
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.625189052424638e-06,
      "logits/chosen": 0.3076649308204651,
      "logits/rejected": 0.34865492582321167,
      "logps/chosen": -290.0766296386719,
      "logps/rejected": -279.2445068359375,
      "loss": 0.6127,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.4531574249267578,
      "rewards/margins": 0.25979921221733093,
      "rewards/rejected": -0.7129566669464111,
      "step": 990
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.613069129183218e-06,
      "logits/chosen": 0.2790106236934662,
      "logits/rejected": 0.2944566607475281,
      "logps/chosen": -318.7317810058594,
      "logps/rejected": -332.01373291015625,
      "loss": 0.6435,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.5851359367370605,
      "rewards/margins": 0.2526613771915436,
      "rewards/rejected": -0.8377972841262817,
      "step": 1000
    },
    {
      "epoch": 0.26,
      "eval_logits/chosen": 0.2765878736972809,
      "eval_logits/rejected": 0.3378385305404663,
      "eval_logps/chosen": -331.1562194824219,
      "eval_logps/rejected": -330.9462585449219,
      "eval_loss": 0.6007062792778015,
      "eval_rewards/accuracies": 0.6294999718666077,
      "eval_rewards/chosen": -0.5360942482948303,
      "eval_rewards/margins": 0.3352090120315552,
      "eval_rewards/rejected": -0.8713032007217407,
      "eval_runtime": 545.4218,
      "eval_samples_per_second": 3.667,
      "eval_steps_per_second": 0.917,
      "step": 1000
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.600772765277607e-06,
      "logits/chosen": 0.27743756771087646,
      "logits/rejected": 0.34427809715270996,
      "logps/chosen": -349.8276672363281,
      "logps/rejected": -335.66961669921875,
      "loss": 0.5897,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5436006784439087,
      "rewards/margins": 0.3772372603416443,
      "rewards/rejected": -0.9208378791809082,
      "step": 1010
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.588300987450652e-06,
      "logits/chosen": 0.3442625403404236,
      "logits/rejected": 0.4009414613246918,
      "logps/chosen": -328.8561096191406,
      "logps/rejected": -346.3475036621094,
      "loss": 0.5841,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.514030396938324,
      "rewards/margins": 0.38697338104248047,
      "rewards/rejected": -0.9010037183761597,
      "step": 1020
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.5756548370922136e-06,
      "logits/chosen": 0.34263429045677185,
      "logits/rejected": 0.3785105049610138,
      "logps/chosen": -328.25164794921875,
      "logps/rejected": -325.9219665527344,
      "loss": 0.6203,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.5046300292015076,
      "rewards/margins": 0.2802472710609436,
      "rewards/rejected": -0.7848772406578064,
      "step": 1030
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.562835370152206e-06,
      "logits/chosen": 0.299723744392395,
      "logits/rejected": 0.3728645443916321,
      "logps/chosen": -298.93182373046875,
      "logps/rejected": -308.7339172363281,
      "loss": 0.6124,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.49450236558914185,
      "rewards/margins": 0.2736920416355133,
      "rewards/rejected": -0.7681943774223328,
      "step": 1040
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.54984365705243e-06,
      "logits/chosen": 0.33221280574798584,
      "logits/rejected": 0.3348609209060669,
      "logps/chosen": -304.962646484375,
      "logps/rejected": -305.4454650878906,
      "loss": 0.6168,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.5311123132705688,
      "rewards/margins": 0.2822791039943695,
      "rewards/rejected": -0.8133915066719055,
      "step": 1050
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.536680782597191e-06,
      "logits/chosen": 0.29144400358200073,
      "logits/rejected": 0.3640415370464325,
      "logps/chosen": -345.25677490234375,
      "logps/rejected": -343.65972900390625,
      "loss": 0.6002,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.48992738127708435,
      "rewards/margins": 0.32441776990890503,
      "rewards/rejected": -0.8143451809883118,
      "step": 1060
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.523347845882718e-06,
      "logits/chosen": 0.3392336964607239,
      "logits/rejected": 0.3736265301704407,
      "logps/chosen": -311.3797302246094,
      "logps/rejected": -290.25030517578125,
      "loss": 0.5924,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.560332179069519,
      "rewards/margins": 0.34816139936447144,
      "rewards/rejected": -0.9084935188293457,
      "step": 1070
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.50984596020539e-06,
      "logits/chosen": 0.3102160096168518,
      "logits/rejected": 0.34888216853141785,
      "logps/chosen": -352.9917297363281,
      "logps/rejected": -350.4587707519531,
      "loss": 0.589,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.5566148161888123,
      "rewards/margins": 0.3757871091365814,
      "rewards/rejected": -0.9324019551277161,
      "step": 1080
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.4961762529687745e-06,
      "logits/chosen": 0.27754610776901245,
      "logits/rejected": 0.3419993817806244,
      "logps/chosen": -365.9258117675781,
      "logps/rejected": -324.9228210449219,
      "loss": 0.5993,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.6483023762702942,
      "rewards/margins": 0.34900346398353577,
      "rewards/rejected": -0.9973058700561523,
      "step": 1090
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.482339865589492e-06,
      "logits/chosen": 0.23795409500598907,
      "logits/rejected": 0.339063435792923,
      "logps/chosen": -321.9118957519531,
      "logps/rejected": -328.0565185546875,
      "loss": 0.5626,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.6000961661338806,
      "rewards/margins": 0.43007296323776245,
      "rewards/rejected": -1.030169129371643,
      "step": 1100
    },
    {
      "epoch": 0.29,
      "eval_logits/chosen": 0.28789982199668884,
      "eval_logits/rejected": 0.34160664677619934,
      "eval_logps/chosen": -345.95831298828125,
      "eval_logps/rejected": -346.8067932128906,
      "eval_loss": 0.5970672369003296,
      "eval_rewards/accuracies": 0.6194999814033508,
      "eval_rewards/chosen": -0.6841151714324951,
      "eval_rewards/margins": 0.3457929790019989,
      "eval_rewards/rejected": -1.0299081802368164,
      "eval_runtime": 545.4538,
      "eval_samples_per_second": 3.667,
      "eval_steps_per_second": 0.917,
      "step": 1100
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.468337953401909e-06,
      "logits/chosen": 0.3074113726615906,
      "logits/rejected": 0.34023183584213257,
      "logps/chosen": -362.4942626953125,
      "logps/rejected": -334.08331298828125,
      "loss": 0.6164,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.6692425608634949,
      "rewards/margins": 0.2884816527366638,
      "rewards/rejected": -0.9577242136001587,
      "step": 1110
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.45417168556166e-06,
      "logits/chosen": 0.25964727997779846,
      "logits/rejected": 0.30631059408187866,
      "logps/chosen": -312.24346923828125,
      "logps/rejected": -344.4534606933594,
      "loss": 0.5919,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.5477944612503052,
      "rewards/margins": 0.3570755422115326,
      "rewards/rejected": -0.9048700332641602,
      "step": 1120
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.439842244948036e-06,
      "logits/chosen": 0.23340173065662384,
      "logits/rejected": 0.2836596369743347,
      "logps/chosen": -316.97265625,
      "logps/rejected": -322.099609375,
      "loss": 0.6289,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.5921416878700256,
      "rewards/margins": 0.2623347342014313,
      "rewards/rejected": -0.8544764518737793,
      "step": 1130
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.425350828065204e-06,
      "logits/chosen": 0.24592900276184082,
      "logits/rejected": 0.24909226596355438,
      "logps/chosen": -335.1441345214844,
      "logps/rejected": -347.17657470703125,
      "loss": 0.6054,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.5570545792579651,
      "rewards/margins": 0.3322640657424927,
      "rewards/rejected": -0.889318585395813,
      "step": 1140
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.410698644942303e-06,
      "logits/chosen": 0.27134156227111816,
      "logits/rejected": 0.2923016846179962,
      "logps/chosen": -332.7898254394531,
      "logps/rejected": -365.0975036621094,
      "loss": 0.6223,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.5455362796783447,
      "rewards/margins": 0.32090216875076294,
      "rewards/rejected": -0.8664385080337524,
      "step": 1150
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.395886919032406e-06,
      "logits/chosen": 0.22134526073932648,
      "logits/rejected": 0.2600460648536682,
      "logps/chosen": -330.41302490234375,
      "logps/rejected": -321.27239990234375,
      "loss": 0.5828,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.5916182994842529,
      "rewards/margins": 0.3787124752998352,
      "rewards/rejected": -0.9703305959701538,
      "step": 1160
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.380916887110366e-06,
      "logits/chosen": 0.167360320687294,
      "logits/rejected": 0.23531296849250793,
      "logps/chosen": -317.62847900390625,
      "logps/rejected": -324.3927001953125,
      "loss": 0.5666,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.6931602358818054,
      "rewards/margins": 0.4072093963623047,
      "rewards/rejected": -1.1003696918487549,
      "step": 1170
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.365789799169539e-06,
      "logits/chosen": 0.21306875348091125,
      "logits/rejected": 0.1979639232158661,
      "logps/chosen": -391.1867370605469,
      "logps/rejected": -379.17315673828125,
      "loss": 0.563,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.8303465843200684,
      "rewards/margins": 0.44011443853378296,
      "rewards/rejected": -1.270461082458496,
      "step": 1180
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.350506918317416e-06,
      "logits/chosen": 0.16972529888153076,
      "logits/rejected": 0.1481637805700302,
      "logps/chosen": -377.83966064453125,
      "logps/rejected": -376.23394775390625,
      "loss": 0.6126,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.7504847645759583,
      "rewards/margins": 0.33371812105178833,
      "rewards/rejected": -1.084202766418457,
      "step": 1190
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.335069520670149e-06,
      "logits/chosen": 0.17195920646190643,
      "logits/rejected": 0.2304229438304901,
      "logps/chosen": -354.7908935546875,
      "logps/rejected": -338.830810546875,
      "loss": 0.5319,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.7843652963638306,
      "rewards/margins": 0.5480031967163086,
      "rewards/rejected": -1.3323684930801392,
      "step": 1200
    },
    {
      "epoch": 0.31,
      "eval_logits/chosen": 0.1477048397064209,
      "eval_logits/rejected": 0.19143778085708618,
      "eval_logps/chosen": -366.0687255859375,
      "eval_logps/rejected": -372.77557373046875,
      "eval_loss": 0.5971232652664185,
      "eval_rewards/accuracies": 0.628000020980835,
      "eval_rewards/chosen": -0.8852190971374512,
      "eval_rewards/margins": 0.404377281665802,
      "eval_rewards/rejected": -1.289596438407898,
      "eval_runtime": 545.4758,
      "eval_samples_per_second": 3.667,
      "eval_steps_per_second": 0.917,
      "step": 1200
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.319478895246e-06,
      "logits/chosen": 0.16348786652088165,
      "logits/rejected": 0.13432151079177856,
      "logps/chosen": -354.9012756347656,
      "logps/rejected": -346.83770751953125,
      "loss": 0.6192,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.8076756596565247,
      "rewards/margins": 0.3335443139076233,
      "rewards/rejected": -1.141219973564148,
      "step": 1210
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.303736343857704e-06,
      "logits/chosen": 0.2526926100254059,
      "logits/rejected": 0.21305808424949646,
      "logps/chosen": -342.25537109375,
      "logps/rejected": -363.7655334472656,
      "loss": 0.5549,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.781582236289978,
      "rewards/margins": 0.44490519165992737,
      "rewards/rejected": -1.2264875173568726,
      "step": 1220
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.287843181003772e-06,
      "logits/chosen": 0.21736888587474823,
      "logits/rejected": 0.27333998680114746,
      "logps/chosen": -357.6294860839844,
      "logps/rejected": -358.72308349609375,
      "loss": 0.5833,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.799574077129364,
      "rewards/margins": 0.4974464476108551,
      "rewards/rejected": -1.2970205545425415,
      "step": 1230
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.27180073375873e-06,
      "logits/chosen": 0.19267229735851288,
      "logits/rejected": 0.20701631903648376,
      "logps/chosen": -362.3644104003906,
      "logps/rejected": -376.9123229980469,
      "loss": 0.5756,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.8460612297058105,
      "rewards/margins": 0.47652435302734375,
      "rewards/rejected": -1.3225855827331543,
      "step": 1240
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.255610341662304e-06,
      "logits/chosen": 0.2003142386674881,
      "logits/rejected": 0.24681635200977325,
      "logps/chosen": -385.9356384277344,
      "logps/rejected": -390.82818603515625,
      "loss": 0.5636,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.8146869540214539,
      "rewards/margins": 0.4905151426792145,
      "rewards/rejected": -1.3052021265029907,
      "step": 1250
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.2392733566075764e-06,
      "logits/chosen": 0.21379859745502472,
      "logits/rejected": 0.2299179583787918,
      "logps/chosen": -327.9105529785156,
      "logps/rejected": -345.62481689453125,
      "loss": 0.5906,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8707124590873718,
      "rewards/margins": 0.42304062843322754,
      "rewards/rejected": -1.293752908706665,
      "step": 1260
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.2227911427280975e-06,
      "logits/chosen": 0.2143145352602005,
      "logits/rejected": 0.24731318652629852,
      "logps/chosen": -370.6251525878906,
      "logps/rejected": -387.04022216796875,
      "loss": 0.6156,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.8683725595474243,
      "rewards/margins": 0.36514347791671753,
      "rewards/rejected": -1.233515977859497,
      "step": 1270
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.206165076283983e-06,
      "logits/chosen": 0.15996266901493073,
      "logits/rejected": 0.1850823163986206,
      "logps/chosen": -320.31396484375,
      "logps/rejected": -331.081298828125,
      "loss": 0.636,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.8364042043685913,
      "rewards/margins": 0.2532604932785034,
      "rewards/rejected": -1.0896646976470947,
      "step": 1280
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.189396545546995e-06,
      "logits/chosen": 0.2215954065322876,
      "logits/rejected": 0.27586954832077026,
      "logps/chosen": -358.5014343261719,
      "logps/rejected": -373.06488037109375,
      "loss": 0.6226,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.7066820859909058,
      "rewards/margins": 0.27226629853248596,
      "rewards/rejected": -0.9789482951164246,
      "step": 1290
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.172486950684627e-06,
      "logits/chosen": 0.21022501587867737,
      "logits/rejected": 0.22880622744560242,
      "logps/chosen": -348.2843322753906,
      "logps/rejected": -368.8757629394531,
      "loss": 0.5818,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.7178322076797485,
      "rewards/margins": 0.42600828409194946,
      "rewards/rejected": -1.1438405513763428,
      "step": 1300
    },
    {
      "epoch": 0.34,
      "eval_logits/chosen": 0.16877275705337524,
      "eval_logits/rejected": 0.21647466719150543,
      "eval_logps/chosen": -349.3257141113281,
      "eval_logps/rejected": -354.08599853515625,
      "eval_loss": 0.5948539972305298,
      "eval_rewards/accuracies": 0.6315000057220459,
      "eval_rewards/chosen": -0.7177892327308655,
      "eval_rewards/margins": 0.38491156697273254,
      "eval_rewards/rejected": -1.1027007102966309,
      "eval_runtime": 545.5664,
      "eval_samples_per_second": 3.666,
      "eval_steps_per_second": 0.916,
      "step": 1300
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.155437703643182e-06,
      "logits/chosen": 0.138587087392807,
      "logits/rejected": 0.1511545479297638,
      "logps/chosen": -347.5982971191406,
      "logps/rejected": -357.45428466796875,
      "loss": 0.5593,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.6473184823989868,
      "rewards/margins": 0.49595245718955994,
      "rewards/rejected": -1.1432709693908691,
      "step": 1310
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.138250228029882e-06,
      "logits/chosen": 0.1727471649646759,
      "logits/rejected": 0.22663483023643494,
      "logps/chosen": -302.2596740722656,
      "logps/rejected": -316.713134765625,
      "loss": 0.5862,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.7017611861228943,
      "rewards/margins": 0.384599506855011,
      "rewards/rejected": -1.0863606929779053,
      "step": 1320
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.120925958993994e-06,
      "logits/chosen": 0.18980373442173004,
      "logits/rejected": 0.21815748512744904,
      "logps/chosen": -365.2742614746094,
      "logps/rejected": -347.21844482421875,
      "loss": 0.6083,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.7208053469657898,
      "rewards/margins": 0.3296663165092468,
      "rewards/rejected": -1.0504717826843262,
      "step": 1330
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.103466343106999e-06,
      "logits/chosen": 0.17980042099952698,
      "logits/rejected": 0.18235398828983307,
      "logps/chosen": -345.6518249511719,
      "logps/rejected": -340.5759582519531,
      "loss": 0.6229,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.7314938902854919,
      "rewards/margins": 0.3103274703025818,
      "rewards/rejected": -1.0418212413787842,
      "step": 1340
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.085872838241797e-06,
      "logits/chosen": 0.22088384628295898,
      "logits/rejected": 0.22433125972747803,
      "logps/chosen": -352.87835693359375,
      "logps/rejected": -389.94769287109375,
      "loss": 0.5642,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.6772549152374268,
      "rewards/margins": 0.4428323209285736,
      "rewards/rejected": -1.1200872659683228,
      "step": 1350
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.06814691345098e-06,
      "logits/chosen": 0.20037512481212616,
      "logits/rejected": 0.26389187574386597,
      "logps/chosen": -373.4176940917969,
      "logps/rejected": -363.95367431640625,
      "loss": 0.5832,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7378250360488892,
      "rewards/margins": 0.40974587202072144,
      "rewards/rejected": -1.1475709676742554,
      "step": 1360
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.050290048844171e-06,
      "logits/chosen": 0.216600701212883,
      "logits/rejected": 0.23960819840431213,
      "logps/chosen": -316.2613220214844,
      "logps/rejected": -314.7508850097656,
      "loss": 0.6094,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.6164644360542297,
      "rewards/margins": 0.32712945342063904,
      "rewards/rejected": -0.9435938000679016,
      "step": 1370
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.032303735464422e-06,
      "logits/chosen": 0.17721518874168396,
      "logits/rejected": 0.22735758125782013,
      "logps/chosen": -344.809326171875,
      "logps/rejected": -347.049072265625,
      "loss": 0.5851,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5567644238471985,
      "rewards/margins": 0.39752551913261414,
      "rewards/rejected": -0.9542900323867798,
      "step": 1380
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.014189475163727e-06,
      "logits/chosen": 0.16344419121742249,
      "logits/rejected": 0.21651005744934082,
      "logps/chosen": -322.96533203125,
      "logps/rejected": -321.7102966308594,
      "loss": 0.6153,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.567093551158905,
      "rewards/margins": 0.2981587052345276,
      "rewards/rejected": -0.8652523159980774,
      "step": 1390
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.995948780477605e-06,
      "logits/chosen": 0.19968202710151672,
      "logits/rejected": 0.21880850195884705,
      "logps/chosen": -346.12286376953125,
      "logps/rejected": -336.0102844238281,
      "loss": 0.5981,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.597925066947937,
      "rewards/margins": 0.34026554226875305,
      "rewards/rejected": -0.9381906390190125,
      "step": 1400
    },
    {
      "epoch": 0.37,
      "eval_logits/chosen": 0.14649315178394318,
      "eval_logits/rejected": 0.19736629724502563,
      "eval_logps/chosen": -343.7120361328125,
      "eval_logps/rejected": -346.38848876953125,
      "eval_loss": 0.5935717821121216,
      "eval_rewards/accuracies": 0.6290000081062317,
      "eval_rewards/chosen": -0.6616523265838623,
      "eval_rewards/margins": 0.36407363414764404,
      "eval_rewards/rejected": -1.0257259607315063,
      "eval_runtime": 544.5501,
      "eval_samples_per_second": 3.673,
      "eval_steps_per_second": 0.918,
      "step": 1400
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.977583174498816e-06,
      "logits/chosen": 0.1473083198070526,
      "logits/rejected": 0.16147521138191223,
      "logps/chosen": -336.35028076171875,
      "logps/rejected": -337.4244079589844,
      "loss": 0.5943,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.6331855654716492,
      "rewards/margins": 0.3674471378326416,
      "rewards/rejected": -1.0006327629089355,
      "step": 1410
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.959094190750172e-06,
      "logits/chosen": 0.09590881317853928,
      "logits/rejected": 0.1392073780298233,
      "logps/chosen": -368.92742919921875,
      "logps/rejected": -366.66680908203125,
      "loss": 0.6168,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.8335236310958862,
      "rewards/margins": 0.3043787181377411,
      "rewards/rejected": -1.1379024982452393,
      "step": 1420
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.9404833730564975e-06,
      "logits/chosen": 0.12012658268213272,
      "logits/rejected": 0.13889259099960327,
      "logps/chosen": -338.89166259765625,
      "logps/rejected": -369.4378356933594,
      "loss": 0.6076,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.9257830381393433,
      "rewards/margins": 0.3463601768016815,
      "rewards/rejected": -1.2721431255340576,
      "step": 1430
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.921752275415712e-06,
      "logits/chosen": 0.07003729790449142,
      "logits/rejected": 0.10457529872655869,
      "logps/chosen": -324.9673156738281,
      "logps/rejected": -325.56304931640625,
      "loss": 0.6099,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.8632106781005859,
      "rewards/margins": 0.33071261644363403,
      "rewards/rejected": -1.1939232349395752,
      "step": 1440
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.902902461869079e-06,
      "logits/chosen": 0.04336017742753029,
      "logits/rejected": 0.107862189412117,
      "logps/chosen": -366.1966247558594,
      "logps/rejected": -370.04730224609375,
      "loss": 0.5981,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.9756678342819214,
      "rewards/margins": 0.36889126896858215,
      "rewards/rejected": -1.3445593118667603,
      "step": 1450
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.883935506370605e-06,
      "logits/chosen": 0.03404034674167633,
      "logits/rejected": 0.07542404532432556,
      "logps/chosen": -363.59234619140625,
      "logps/rejected": -358.55291748046875,
      "loss": 0.6025,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.868129551410675,
      "rewards/margins": 0.37522757053375244,
      "rewards/rejected": -1.2433571815490723,
      "step": 1460
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.864852992655617e-06,
      "logits/chosen": 0.029746342450380325,
      "logits/rejected": 0.08432164043188095,
      "logps/chosen": -335.89105224609375,
      "logps/rejected": -358.84063720703125,
      "loss": 0.5915,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.6996029615402222,
      "rewards/margins": 0.38555437326431274,
      "rewards/rejected": -1.0851573944091797,
      "step": 1470
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.845656514108516e-06,
      "logits/chosen": 0.0590134859085083,
      "logits/rejected": 0.08817918598651886,
      "logps/chosen": -366.0428771972656,
      "logps/rejected": -356.31353759765625,
      "loss": 0.5654,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.7267670631408691,
      "rewards/margins": 0.442646324634552,
      "rewards/rejected": -1.1694133281707764,
      "step": 1480
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.826347673629738e-06,
      "logits/chosen": 0.11978931725025177,
      "logits/rejected": 0.11140674352645874,
      "logps/chosen": -309.720947265625,
      "logps/rejected": -339.85491943359375,
      "loss": 0.5517,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.671705961227417,
      "rewards/margins": 0.5157135128974915,
      "rewards/rejected": -1.1874195337295532,
      "step": 1490
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.8069280835019062e-06,
      "logits/chosen": 0.06044895201921463,
      "logits/rejected": 0.091534823179245,
      "logps/chosen": -392.0270080566406,
      "logps/rejected": -360.0969543457031,
      "loss": 0.5843,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.6990892887115479,
      "rewards/margins": 0.4130396246910095,
      "rewards/rejected": -1.1121289730072021,
      "step": 1500
    },
    {
      "epoch": 0.39,
      "eval_logits/chosen": 0.058730266988277435,
      "eval_logits/rejected": 0.10037268698215485,
      "eval_logps/chosen": -366.15447998046875,
      "eval_logps/rejected": -374.1299133300781,
      "eval_loss": 0.5905080437660217,
      "eval_rewards/accuracies": 0.6334999799728394,
      "eval_rewards/chosen": -0.886076807975769,
      "eval_rewards/margins": 0.4170626699924469,
      "eval_rewards/rejected": -1.3031394481658936,
      "eval_runtime": 544.4729,
      "eval_samples_per_second": 3.673,
      "eval_steps_per_second": 0.918,
      "step": 1500
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7873993652552077e-06,
      "logits/chosen": 0.05399291589856148,
      "logits/rejected": 0.11311284452676773,
      "logps/chosen": -355.0113830566406,
      "logps/rejected": -351.7707824707031,
      "loss": 0.6685,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.9721186757087708,
      "rewards/margins": 0.20769429206848145,
      "rewards/rejected": -1.1798131465911865,
      "step": 1510
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7677631495319953e-06,
      "logits/chosen": 0.07790177315473557,
      "logits/rejected": 0.10548149049282074,
      "logps/chosen": -347.7140808105469,
      "logps/rejected": -331.2201843261719,
      "loss": 0.6034,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.8945258855819702,
      "rewards/margins": 0.32894036173820496,
      "rewards/rejected": -1.2234662771224976,
      "step": 1520
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.748021075950633e-06,
      "logits/chosen": 0.17391851544380188,
      "logits/rejected": 0.1568259447813034,
      "logps/chosen": -404.0233459472656,
      "logps/rejected": -412.5658264160156,
      "loss": 0.5985,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.019444465637207,
      "rewards/margins": 0.4078293740749359,
      "rewards/rejected": -1.4272738695144653,
      "step": 1530
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7281747929685824e-06,
      "logits/chosen": 0.1422004997730255,
      "logits/rejected": 0.21070496737957,
      "logps/chosen": -367.45281982421875,
      "logps/rejected": -362.89080810546875,
      "loss": 0.6076,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.793968677520752,
      "rewards/margins": 0.35677820444107056,
      "rewards/rejected": -1.1507470607757568,
      "step": 1540
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.7082259577447604e-06,
      "logits/chosen": 0.14684629440307617,
      "logits/rejected": 0.1937289983034134,
      "logps/chosen": -405.5082702636719,
      "logps/rejected": -375.3370666503906,
      "loss": 0.5569,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.8601733446121216,
      "rewards/margins": 0.4416710436344147,
      "rewards/rejected": -1.3018442392349243,
      "step": 1550
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6881762360011688e-06,
      "logits/chosen": 0.1889687478542328,
      "logits/rejected": 0.2150932103395462,
      "logps/chosen": -373.6668395996094,
      "logps/rejected": -366.2331848144531,
      "loss": 0.6058,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.8771917223930359,
      "rewards/margins": 0.3977741599082947,
      "rewards/rejected": -1.2749658823013306,
      "step": 1560
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.668027301883802e-06,
      "logits/chosen": 0.16446110606193542,
      "logits/rejected": 0.18785087764263153,
      "logps/chosen": -351.145263671875,
      "logps/rejected": -343.36676025390625,
      "loss": 0.5739,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.8971999883651733,
      "rewards/margins": 0.3938083052635193,
      "rewards/rejected": -1.2910082340240479,
      "step": 1570
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.64778083782286e-06,
      "logits/chosen": 0.22796198725700378,
      "logits/rejected": 0.2517542243003845,
      "logps/chosen": -352.5816955566406,
      "logps/rejected": -400.53314208984375,
      "loss": 0.5811,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.8595455884933472,
      "rewards/margins": 0.454528272151947,
      "rewards/rejected": -1.3140738010406494,
      "step": 1580
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.627438534392268e-06,
      "logits/chosen": 0.18804603815078735,
      "logits/rejected": 0.19986502826213837,
      "logps/chosen": -342.622314453125,
      "logps/rejected": -325.24755859375,
      "loss": 0.5872,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.8171426057815552,
      "rewards/margins": 0.37278664112091064,
      "rewards/rejected": -1.1899292469024658,
      "step": 1590
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.607002090168506e-06,
      "logits/chosen": 0.182193785905838,
      "logits/rejected": 0.20623533427715302,
      "logps/chosen": -343.2569274902344,
      "logps/rejected": -345.95123291015625,
      "loss": 0.6283,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.8513103723526001,
      "rewards/margins": 0.2995356619358063,
      "rewards/rejected": -1.1508458852767944,
      "step": 1600
    },
    {
      "epoch": 0.42,
      "eval_logits/chosen": 0.1738397777080536,
      "eval_logits/rejected": 0.22415193915367126,
      "eval_logps/chosen": -356.0013122558594,
      "eval_logps/rejected": -360.87457275390625,
      "eval_loss": 0.5882366895675659,
      "eval_rewards/accuracies": 0.6305000185966492,
      "eval_rewards/chosen": -0.7845450639724731,
      "eval_rewards/margins": 0.3860413432121277,
      "eval_rewards/rejected": -1.1705864667892456,
      "eval_runtime": 544.4759,
      "eval_samples_per_second": 3.673,
      "eval_steps_per_second": 0.918,
      "step": 1600
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.586473211588787e-06,
      "logits/chosen": 0.21881847083568573,
      "logits/rejected": 0.2783322334289551,
      "logps/chosen": -363.8297424316406,
      "logps/rejected": -347.1457214355469,
      "loss": 0.5606,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.6486612558364868,
      "rewards/margins": 0.46156877279281616,
      "rewards/rejected": -1.1102300882339478,
      "step": 1610
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.5658536128085623e-06,
      "logits/chosen": 0.1464618593454361,
      "logits/rejected": 0.18708249926567078,
      "logps/chosen": -318.83502197265625,
      "logps/rejected": -347.23028564453125,
      "loss": 0.5755,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.6937158107757568,
      "rewards/margins": 0.39851704239845276,
      "rewards/rejected": -1.0922327041625977,
      "step": 1620
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.545145015558399e-06,
      "logits/chosen": 0.18751280009746552,
      "logits/rejected": 0.19455750286579132,
      "logps/chosen": -365.55816650390625,
      "logps/rejected": -388.07025146484375,
      "loss": 0.6032,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.6676291823387146,
      "rewards/margins": 0.3806609809398651,
      "rewards/rejected": -1.0482903718948364,
      "step": 1630
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5243491490002056e-06,
      "logits/chosen": 0.1777867078781128,
      "logits/rejected": 0.18567804992198944,
      "logps/chosen": -341.19940185546875,
      "logps/rejected": -340.8212890625,
      "loss": 0.6013,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.6734684109687805,
      "rewards/margins": 0.36503511667251587,
      "rewards/rejected": -1.0385034084320068,
      "step": 1640
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.503467749582857e-06,
      "logits/chosen": 0.15182527899742126,
      "logits/rejected": 0.15495124459266663,
      "logps/chosen": -309.76153564453125,
      "logps/rejected": -296.9736633300781,
      "loss": 0.5905,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.6183659434318542,
      "rewards/margins": 0.3707262873649597,
      "rewards/rejected": -0.989092230796814,
      "step": 1650
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.4825025608971947e-06,
      "logits/chosen": 0.1220487505197525,
      "logits/rejected": 0.18718338012695312,
      "logps/chosen": -390.09881591796875,
      "logps/rejected": -361.5142517089844,
      "loss": 0.5749,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.7525783777236938,
      "rewards/margins": 0.42852458357810974,
      "rewards/rejected": -1.181102991104126,
      "step": 1660
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4614553335304407e-06,
      "logits/chosen": 0.17326000332832336,
      "logits/rejected": 0.2386179268360138,
      "logps/chosen": -360.65924072265625,
      "logps/rejected": -356.8613586425781,
      "loss": 0.5791,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8462890386581421,
      "rewards/margins": 0.41173258423805237,
      "rewards/rejected": -1.258021593093872,
      "step": 1670
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4403278249200222e-06,
      "logits/chosen": 0.1455180048942566,
      "logits/rejected": 0.17783500254154205,
      "logps/chosen": -328.61309814453125,
      "logps/rejected": -342.9682922363281,
      "loss": 0.5814,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.714964747428894,
      "rewards/margins": 0.37197887897491455,
      "rewards/rejected": -1.086943507194519,
      "step": 1680
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4191217992068293e-06,
      "logits/chosen": 0.14492273330688477,
      "logits/rejected": 0.1336786448955536,
      "logps/chosen": -332.1494140625,
      "logps/rejected": -359.5680236816406,
      "loss": 0.5922,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.7651108503341675,
      "rewards/margins": 0.34422460198402405,
      "rewards/rejected": -1.1093354225158691,
      "step": 1690
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.3978390270879056e-06,
      "logits/chosen": 0.1335761845111847,
      "logits/rejected": 0.17906330525875092,
      "logps/chosen": -360.72698974609375,
      "logps/rejected": -350.4703674316406,
      "loss": 0.5892,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.6610103249549866,
      "rewards/margins": 0.3798225224018097,
      "rewards/rejected": -1.040832757949829,
      "step": 1700
    },
    {
      "epoch": 0.44,
      "eval_logits/chosen": 0.12588946521282196,
      "eval_logits/rejected": 0.1717854142189026,
      "eval_logps/chosen": -344.9545593261719,
      "eval_logps/rejected": -349.971923828125,
      "eval_loss": 0.5890761017799377,
      "eval_rewards/accuracies": 0.6309999823570251,
      "eval_rewards/chosen": -0.6740775108337402,
      "eval_rewards/margins": 0.3874828517436981,
      "eval_rewards/rejected": -1.0615602731704712,
      "eval_runtime": 544.4943,
      "eval_samples_per_second": 3.673,
      "eval_steps_per_second": 0.918,
      "step": 1700
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3764812856685995e-06,
      "logits/chosen": 0.14172333478927612,
      "logits/rejected": 0.13182663917541504,
      "logps/chosen": -354.5355529785156,
      "logps/rejected": -362.7613830566406,
      "loss": 0.5575,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.6454203724861145,
      "rewards/margins": 0.4289492070674896,
      "rewards/rejected": -1.0743694305419922,
      "step": 1710
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3550503583141726e-06,
      "logits/chosen": 0.12564226984977722,
      "logits/rejected": 0.13638147711753845,
      "logps/chosen": -399.1639404296875,
      "logps/rejected": -421.84246826171875,
      "loss": 0.547,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.7657305002212524,
      "rewards/margins": 0.4795493185520172,
      "rewards/rejected": -1.2452797889709473,
      "step": 1720
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3335480345008907e-06,
      "logits/chosen": 0.10991547256708145,
      "logits/rejected": 0.14212583005428314,
      "logps/chosen": -335.371337890625,
      "logps/rejected": -338.5823669433594,
      "loss": 0.5657,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.7321099638938904,
      "rewards/margins": 0.46315819025039673,
      "rewards/rejected": -1.1952682733535767,
      "step": 1730
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.3119761096666055e-06,
      "logits/chosen": 0.15486234426498413,
      "logits/rejected": 0.13218258321285248,
      "logps/chosen": -401.4673767089844,
      "logps/rejected": -383.27923583984375,
      "loss": 0.601,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.9068065881729126,
      "rewards/margins": 0.3835510313510895,
      "rewards/rejected": -1.2903575897216797,
      "step": 1740
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.290336385060832e-06,
      "logits/chosen": 0.14742621779441833,
      "logits/rejected": 0.13394896686077118,
      "logps/chosen": -362.3402404785156,
      "logps/rejected": -378.41552734375,
      "loss": 0.5781,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.7834215760231018,
      "rewards/margins": 0.44024190306663513,
      "rewards/rejected": -1.223663568496704,
      "step": 1750
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.268630667594348e-06,
      "logits/chosen": 0.1237776055932045,
      "logits/rejected": 0.17092521488666534,
      "logps/chosen": -345.5756530761719,
      "logps/rejected": -362.49029541015625,
      "loss": 0.5809,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.6653984189033508,
      "rewards/margins": 0.4018256664276123,
      "rewards/rejected": -1.0672239065170288,
      "step": 1760
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2468607696883147e-06,
      "logits/chosen": 0.14814460277557373,
      "logits/rejected": 0.16757544875144958,
      "logps/chosen": -364.9542541503906,
      "logps/rejected": -357.6173400878906,
      "loss": 0.5768,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.6790551543235779,
      "rewards/margins": 0.43311649560928345,
      "rewards/rejected": -1.1121716499328613,
      "step": 1770
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.225028509122944e-06,
      "logits/chosen": 0.1401061713695526,
      "logits/rejected": 0.2017345428466797,
      "logps/chosen": -364.52392578125,
      "logps/rejected": -358.08514404296875,
      "loss": 0.5438,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.6000980734825134,
      "rewards/margins": 0.47187384963035583,
      "rewards/rejected": -1.0719718933105469,
      "step": 1780
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.2031357088857083e-06,
      "logits/chosen": 0.11304491758346558,
      "logits/rejected": 0.11087970435619354,
      "logps/chosen": -351.2516784667969,
      "logps/rejected": -373.64520263671875,
      "loss": 0.5746,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.802719235420227,
      "rewards/margins": 0.42133206129074097,
      "rewards/rejected": -1.2240512371063232,
      "step": 1790
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.181184197019127e-06,
      "logits/chosen": 0.0998636931180954,
      "logits/rejected": 0.07577769458293915,
      "logps/chosen": -364.93353271484375,
      "logps/rejected": -355.5188903808594,
      "loss": 0.5821,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.8589967489242554,
      "rewards/margins": 0.436631977558136,
      "rewards/rejected": -1.2956287860870361,
      "step": 1800
    },
    {
      "epoch": 0.47,
      "eval_logits/chosen": 0.07607755810022354,
      "eval_logits/rejected": 0.11990514397621155,
      "eval_logps/chosen": -367.0340881347656,
      "eval_logps/rejected": -377.3439025878906,
      "eval_loss": 0.5855891704559326,
      "eval_rewards/accuracies": 0.6315000057220459,
      "eval_rewards/chosen": -0.8948729038238525,
      "eval_rewards/margins": 0.44040703773498535,
      "eval_rewards/rejected": -1.335280179977417,
      "eval_runtime": 544.5621,
      "eval_samples_per_second": 3.673,
      "eval_steps_per_second": 0.918,
      "step": 1800
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.159175806468126e-06,
      "logits/chosen": 0.06636621057987213,
      "logits/rejected": 0.11029330641031265,
      "logps/chosen": -359.0235595703125,
      "logps/rejected": -389.5569763183594,
      "loss": 0.5938,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7768658995628357,
      "rewards/margins": 0.4340268075466156,
      "rewards/rejected": -1.2108927965164185,
      "step": 1810
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1371123749269804e-06,
      "logits/chosen": 0.12099988758563995,
      "logits/rejected": 0.1557311713695526,
      "logps/chosen": -347.983154296875,
      "logps/rejected": -352.2116394042969,
      "loss": 0.5816,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.6870421171188354,
      "rewards/margins": 0.41189369559288025,
      "rewards/rejected": -1.0989357233047485,
      "step": 1820
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.114995744685877e-06,
      "logits/chosen": 0.09171854704618454,
      "logits/rejected": 0.06376312673091888,
      "logps/chosen": -322.3338623046875,
      "logps/rejected": -354.04266357421875,
      "loss": 0.5491,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.6575523614883423,
      "rewards/margins": 0.5730739831924438,
      "rewards/rejected": -1.2306262254714966,
      "step": 1830
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.0928277624770743e-06,
      "logits/chosen": 0.08149708807468414,
      "logits/rejected": 0.10335078090429306,
      "logps/chosen": -356.908447265625,
      "logps/rejected": -348.29168701171875,
      "loss": 0.544,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.7228487730026245,
      "rewards/margins": 0.5269914865493774,
      "rewards/rejected": -1.249840259552002,
      "step": 1840
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.070610279320708e-06,
      "logits/chosen": 0.06698820739984512,
      "logits/rejected": 0.12880167365074158,
      "logps/chosen": -370.1624450683594,
      "logps/rejected": -340.4905700683594,
      "loss": 0.6352,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.8653889894485474,
      "rewards/margins": 0.28897637128829956,
      "rewards/rejected": -1.1543653011322021,
      "step": 1850
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0483451503702264e-06,
      "logits/chosen": 0.11194877326488495,
      "logits/rejected": 0.1369941681623459,
      "logps/chosen": -352.0809326171875,
      "logps/rejected": -356.8692321777344,
      "loss": 0.6048,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6633593440055847,
      "rewards/margins": 0.3344351649284363,
      "rewards/rejected": -0.997794508934021,
      "step": 1860
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0260342347574916e-06,
      "logits/chosen": 0.1743788868188858,
      "logits/rejected": 0.20504312217235565,
      "logps/chosen": -359.6429748535156,
      "logps/rejected": -349.24249267578125,
      "loss": 0.5542,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.642277717590332,
      "rewards/margins": 0.46937379240989685,
      "rewards/rejected": -1.1116515398025513,
      "step": 1870
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0036793954375358e-06,
      "logits/chosen": 0.11372777074575424,
      "logits/rejected": 0.14487095177173615,
      "logps/chosen": -341.3155212402344,
      "logps/rejected": -347.41705322265625,
      "loss": 0.5501,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.6311262845993042,
      "rewards/margins": 0.5035245418548584,
      "rewards/rejected": -1.1346508264541626,
      "step": 1880
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.981282499033009e-06,
      "logits/chosen": 0.09161444753408432,
      "logits/rejected": 0.11823789775371552,
      "logps/chosen": -359.84173583984375,
      "logps/rejected": -349.1728820800781,
      "loss": 0.5653,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.682461142539978,
      "rewards/margins": 0.44472140073776245,
      "rewards/rejected": -1.1271824836730957,
      "step": 1890
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9588454156783163e-06,
      "logits/chosen": 0.13893449306488037,
      "logits/rejected": 0.10511553287506104,
      "logps/chosen": -345.8633728027344,
      "logps/rejected": -348.959228515625,
      "loss": 0.6072,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.7028459310531616,
      "rewards/margins": 0.3482803702354431,
      "rewards/rejected": -1.05112624168396,
      "step": 1900
    },
    {
      "epoch": 0.5,
      "eval_logits/chosen": 0.07725726068019867,
      "eval_logits/rejected": 0.12365095317363739,
      "eval_logps/chosen": -349.35150146484375,
      "eval_logps/rejected": -357.206298828125,
      "eval_loss": 0.5861266255378723,
      "eval_rewards/accuracies": 0.6269999742507935,
      "eval_rewards/chosen": -0.7180466055870056,
      "eval_rewards/margins": 0.4158574938774109,
      "eval_rewards/rejected": -1.1339040994644165,
      "eval_runtime": 545.0027,
      "eval_samples_per_second": 3.67,
      "eval_steps_per_second": 0.917,
      "step": 1900
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9363700188634597e-06,
      "logits/chosen": 0.09426288306713104,
      "logits/rejected": 0.12164957821369171,
      "logps/chosen": -340.0146179199219,
      "logps/rejected": -344.6805419921875,
      "loss": 0.6206,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.7131950259208679,
      "rewards/margins": 0.31129521131515503,
      "rewards/rejected": -1.024490237236023,
      "step": 1910
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9138581852776053e-06,
      "logits/chosen": 0.08359435945749283,
      "logits/rejected": 0.13281409442424774,
      "logps/chosen": -344.29339599609375,
      "logps/rejected": -368.0191955566406,
      "loss": 0.5695,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.7011746168136597,
      "rewards/margins": 0.45050668716430664,
      "rewards/rejected": -1.1516811847686768,
      "step": 1920
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8913117946523805e-06,
      "logits/chosen": 0.1351374387741089,
      "logits/rejected": 0.10901328176259995,
      "logps/chosen": -330.74658203125,
      "logps/rejected": -321.38983154296875,
      "loss": 0.5932,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.7365423440933228,
      "rewards/margins": 0.3617197275161743,
      "rewards/rejected": -1.098262071609497,
      "step": 1930
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8687327296049126e-06,
      "logits/chosen": 0.07262937724590302,
      "logits/rejected": 0.10712842643260956,
      "logps/chosen": -345.1435546875,
      "logps/rejected": -357.8013610839844,
      "loss": 0.5871,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.6691123247146606,
      "rewards/margins": 0.3879561722278595,
      "rewards/rejected": -1.0570685863494873,
      "step": 1940
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8461228754806376e-06,
      "logits/chosen": 0.05750247836112976,
      "logits/rejected": 0.10219607502222061,
      "logps/chosen": -311.95819091796875,
      "logps/rejected": -345.6867980957031,
      "loss": 0.5663,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.6487970352172852,
      "rewards/margins": 0.4769372344017029,
      "rewards/rejected": -1.1257343292236328,
      "step": 1950
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.823484120195865e-06,
      "logits/chosen": 0.07319775968790054,
      "logits/rejected": 0.042475730180740356,
      "logps/chosen": -262.2962646484375,
      "logps/rejected": -308.214599609375,
      "loss": 0.5832,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.5616660118103027,
      "rewards/margins": 0.37587177753448486,
      "rewards/rejected": -0.9375377893447876,
      "step": 1960
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.8008183540801486e-06,
      "logits/chosen": -0.014851477928459644,
      "logits/rejected": 0.042431194335222244,
      "logps/chosen": -351.74517822265625,
      "logps/rejected": -362.99530029296875,
      "loss": 0.6036,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.6677108407020569,
      "rewards/margins": 0.3557429313659668,
      "rewards/rejected": -1.023453712463379,
      "step": 1970
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7781274697184353e-06,
      "logits/chosen": 0.008823997341096401,
      "logits/rejected": 0.035457246005535126,
      "logps/chosen": -338.5292053222656,
      "logps/rejected": -352.94232177734375,
      "loss": 0.5809,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.6730680465698242,
      "rewards/margins": 0.4013848304748535,
      "rewards/rejected": -1.0744528770446777,
      "step": 1980
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7554133617930397e-06,
      "logits/chosen": 0.019716525450348854,
      "logits/rejected": 0.012831945903599262,
      "logps/chosen": -341.1331787109375,
      "logps/rejected": -363.53704833984375,
      "loss": 0.6296,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.7185381650924683,
      "rewards/margins": 0.3087863326072693,
      "rewards/rejected": -1.0273244380950928,
      "step": 1990
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7326779269254363e-06,
      "logits/chosen": -0.04277997091412544,
      "logits/rejected": -0.009286623448133469,
      "logps/chosen": -321.3934326171875,
      "logps/rejected": -318.6489562988281,
      "loss": 0.6338,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.6931952834129333,
      "rewards/margins": 0.2543143033981323,
      "rewards/rejected": -0.9475095868110657,
      "step": 2000
    },
    {
      "epoch": 0.52,
      "eval_logits/chosen": -0.030136894434690475,
      "eval_logits/rejected": 0.00869889184832573,
      "eval_logps/chosen": -349.098388671875,
      "eval_logps/rejected": -356.585205078125,
      "eval_loss": 0.5851887464523315,
      "eval_rewards/accuracies": 0.6340000033378601,
      "eval_rewards/chosen": -0.7155155539512634,
      "eval_rewards/margins": 0.41217753291130066,
      "eval_rewards/rejected": -1.1276930570602417,
      "eval_runtime": 545.3798,
      "eval_samples_per_second": 3.667,
      "eval_steps_per_second": 0.917,
      "step": 2000
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.7099230635178954e-06,
      "logits/chosen": -0.03830226510763168,
      "logits/rejected": 0.004055617842823267,
      "logps/chosen": -313.728759765625,
      "logps/rejected": -323.69134521484375,
      "loss": 0.5366,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.6489647030830383,
      "rewards/margins": 0.5445326566696167,
      "rewards/rejected": -1.1934973001480103,
      "step": 2010
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6871506715949608e-06,
      "logits/chosen": 0.018227530643343925,
      "logits/rejected": 0.021982427686452866,
      "logps/chosen": -351.05743408203125,
      "logps/rejected": -378.8858337402344,
      "loss": 0.5654,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.7617127895355225,
      "rewards/margins": 0.49486279487609863,
      "rewards/rejected": -1.256575584411621,
      "step": 2020
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6643626526448063e-06,
      "logits/chosen": -0.030996423214673996,
      "logits/rejected": 0.006755811162292957,
      "logps/chosen": -323.4994201660156,
      "logps/rejected": -362.4649353027344,
      "loss": 0.6141,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8070539236068726,
      "rewards/margins": 0.3914108872413635,
      "rewards/rejected": -1.1984648704528809,
      "step": 2030
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6415609094604562e-06,
      "logits/chosen": -0.05985324829816818,
      "logits/rejected": -0.0018987020011991262,
      "logps/chosen": -371.3141174316406,
      "logps/rejected": -374.3472595214844,
      "loss": 0.5541,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.7072352170944214,
      "rewards/margins": 0.5413237810134888,
      "rewards/rejected": -1.2485589981079102,
      "step": 2040
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.618747345980904e-06,
      "logits/chosen": -0.0632224753499031,
      "logits/rejected": -0.028610538691282272,
      "logps/chosen": -346.6549377441406,
      "logps/rejected": -351.1329345703125,
      "loss": 0.5581,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5960323214530945,
      "rewards/margins": 0.5037523508071899,
      "rewards/rejected": -1.0997846126556396,
      "step": 2050
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.595923867132136e-06,
      "logits/chosen": -0.02200886234641075,
      "logits/rejected": -0.015088886022567749,
      "logps/chosen": -280.63067626953125,
      "logps/rejected": -327.4749755859375,
      "loss": 0.5462,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5528582334518433,
      "rewards/margins": 0.4946451187133789,
      "rewards/rejected": -1.0475032329559326,
      "step": 2060
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5730923786680672e-06,
      "logits/chosen": -0.06903935968875885,
      "logits/rejected": -0.05149111896753311,
      "logps/chosen": -348.12213134765625,
      "logps/rejected": -344.7447509765625,
      "loss": 0.5743,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.7621151208877563,
      "rewards/margins": 0.47058719396591187,
      "rewards/rejected": -1.232702374458313,
      "step": 2070
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5502547870114137e-06,
      "logits/chosen": -0.07594850659370422,
      "logits/rejected": -0.05803506448864937,
      "logps/chosen": -301.5760498046875,
      "logps/rejected": -354.6790771484375,
      "loss": 0.5983,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.7226616740226746,
      "rewards/margins": 0.3710746169090271,
      "rewards/rejected": -1.0937364101409912,
      "step": 2080
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.527412999094507e-06,
      "logits/chosen": -0.06949422508478165,
      "logits/rejected": -0.053729571402072906,
      "logps/chosen": -362.20013427734375,
      "logps/rejected": -371.27325439453125,
      "loss": 0.5922,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.8210228681564331,
      "rewards/margins": 0.4187556207180023,
      "rewards/rejected": -1.2397785186767578,
      "step": 2090
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.504568922200064e-06,
      "logits/chosen": -0.07110200822353363,
      "logits/rejected": -0.049906354397535324,
      "logps/chosen": -334.72540283203125,
      "logps/rejected": -360.26605224609375,
      "loss": 0.5582,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.7283493280410767,
      "rewards/margins": 0.47346624732017517,
      "rewards/rejected": -1.2018156051635742,
      "step": 2100
    },
    {
      "epoch": 0.55,
      "eval_logits/chosen": -0.059490080922842026,
      "eval_logits/rejected": -0.022946665063500404,
      "eval_logps/chosen": -351.37261962890625,
      "eval_logps/rejected": -360.6402282714844,
      "eval_loss": 0.5859557390213013,
      "eval_rewards/accuracies": 0.6340000033378601,
      "eval_rewards/chosen": -0.7382580637931824,
      "eval_rewards/margins": 0.42998480796813965,
      "eval_rewards/rejected": -1.1682429313659668,
      "eval_runtime": 544.2836,
      "eval_samples_per_second": 3.675,
      "eval_steps_per_second": 0.919,
      "step": 2100
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.4817244638019333e-06,
      "logits/chosen": -0.032096896320581436,
      "logits/rejected": -0.0033429942559450865,
      "logps/chosen": -323.3070068359375,
      "logps/rejected": -365.35467529296875,
      "loss": 0.5343,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.6752323508262634,
      "rewards/margins": 0.5560353994369507,
      "rewards/rejected": -1.2312676906585693,
      "step": 2110
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.4588815314058155e-06,
      "logits/chosen": -0.070304274559021,
      "logits/rejected": -0.0624236986041069,
      "logps/chosen": -384.3260803222656,
      "logps/rejected": -400.78741455078125,
      "loss": 0.6034,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.7910431623458862,
      "rewards/margins": 0.3856695294380188,
      "rewards/rejected": -1.1767126321792603,
      "step": 2120
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4360420323899922e-06,
      "logits/chosen": -0.10860241949558258,
      "logits/rejected": -0.08050969243049622,
      "logps/chosen": -387.05120849609375,
      "logps/rejected": -377.1029357910156,
      "loss": 0.5609,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9089959263801575,
      "rewards/margins": 0.5126160979270935,
      "rewards/rejected": -1.4216121435165405,
      "step": 2130
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4132078738460585e-06,
      "logits/chosen": -0.06523410975933075,
      "logits/rejected": -0.08063043653964996,
      "logps/chosen": -368.7019348144531,
      "logps/rejected": -400.15985107421875,
      "loss": 0.5792,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9961389303207397,
      "rewards/margins": 0.48970723152160645,
      "rewards/rejected": -1.4858464002609253,
      "step": 2140
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.3903809624196826e-06,
      "logits/chosen": -0.06891070306301117,
      "logits/rejected": -0.04549848660826683,
      "logps/chosen": -364.5621032714844,
      "logps/rejected": -373.35076904296875,
      "loss": 0.574,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.0895036458969116,
      "rewards/margins": 0.43499454855918884,
      "rewards/rejected": -1.5244982242584229,
      "step": 2150
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3675632041513978e-06,
      "logits/chosen": -0.052164457738399506,
      "logits/rejected": -0.03239618241786957,
      "logps/chosen": -338.9453430175781,
      "logps/rejected": -358.5426025390625,
      "loss": 0.5825,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.007673740386963,
      "rewards/margins": 0.40731167793273926,
      "rewards/rejected": -1.4149854183197021,
      "step": 2160
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3447565043174533e-06,
      "logits/chosen": -0.09212299436330795,
      "logits/rejected": -0.09307185560464859,
      "logps/chosen": -365.0643005371094,
      "logps/rejected": -378.61541748046875,
      "loss": 0.5813,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.9552974700927734,
      "rewards/margins": 0.4227164387702942,
      "rewards/rejected": -1.3780138492584229,
      "step": 2170
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.321962767270724e-06,
      "logits/chosen": -0.03847888484597206,
      "logits/rejected": -0.023943722248077393,
      "logps/chosen": -340.03472900390625,
      "logps/rejected": -382.23504638671875,
      "loss": 0.537,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.8605219721794128,
      "rewards/margins": 0.5241655707359314,
      "rewards/rejected": -1.3846876621246338,
      "step": 2180
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.299183896281692e-06,
      "logits/chosen": -0.10258199274539948,
      "logits/rejected": -0.038494259119033813,
      "logps/chosen": -355.46978759765625,
      "logps/rejected": -364.3291320800781,
      "loss": 0.5412,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.8702327609062195,
      "rewards/margins": 0.5633158087730408,
      "rewards/rejected": -1.4335486888885498,
      "step": 2190
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2764217933795297e-06,
      "logits/chosen": -0.09681537002325058,
      "logits/rejected": -0.14781834185123444,
      "logps/chosen": -382.71356201171875,
      "logps/rejected": -385.5505676269531,
      "loss": 0.6103,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.9750157594680786,
      "rewards/margins": 0.3789269030094147,
      "rewards/rejected": -1.353942632675171,
      "step": 2200
    },
    {
      "epoch": 0.58,
      "eval_logits/chosen": -0.10652041435241699,
      "eval_logits/rejected": -0.07140377908945084,
      "eval_logps/chosen": -369.8921203613281,
      "eval_logps/rejected": -382.3634948730469,
      "eval_loss": 0.5820576548576355,
      "eval_rewards/accuracies": 0.6345000267028809,
      "eval_rewards/chosen": -0.9234529733657837,
      "eval_rewards/margins": 0.46202272176742554,
      "eval_rewards/rejected": -1.385475754737854,
      "eval_runtime": 545.4224,
      "eval_samples_per_second": 3.667,
      "eval_steps_per_second": 0.917,
      "step": 2200
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2536783591932786e-06,
      "logits/chosen": -0.019483990967273712,
      "logits/rejected": -0.0737195685505867,
      "logps/chosen": -357.1891174316406,
      "logps/rejected": -374.19061279296875,
      "loss": 0.5947,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.8676769137382507,
      "rewards/margins": 0.434825599193573,
      "rewards/rejected": -1.3025026321411133,
      "step": 2210
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.230955492793149e-06,
      "logits/chosen": -0.07425358891487122,
      "logits/rejected": -0.042141932994127274,
      "logps/chosen": -356.19927978515625,
      "logps/rejected": -359.3014221191406,
      "loss": 0.5553,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.8453773260116577,
      "rewards/margins": 0.47125759720802307,
      "rewards/rejected": -1.3166348934173584,
      "step": 2220
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.208255091531947e-06,
      "logits/chosen": -0.0675973892211914,
      "logits/rejected": -0.07175878435373306,
      "logps/chosen": -349.75653076171875,
      "logps/rejected": -339.6773986816406,
      "loss": 0.5812,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.8538468480110168,
      "rewards/margins": 0.4233093857765198,
      "rewards/rejected": -1.277156114578247,
      "step": 2230
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1855790508866435e-06,
      "logits/chosen": -0.10004396736621857,
      "logits/rejected": -0.0791083425283432,
      "logps/chosen": -374.1147766113281,
      "logps/rejected": -397.65362548828125,
      "loss": 0.5657,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.8433374166488647,
      "rewards/margins": 0.45402494072914124,
      "rewards/rejected": -1.2973623275756836,
      "step": 2240
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.162929264300107e-06,
      "logits/chosen": -0.07542629539966583,
      "logits/rejected": -0.037315886467695236,
      "logps/chosen": -337.01116943359375,
      "logps/rejected": -340.6423034667969,
      "loss": 0.5868,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.8542795181274414,
      "rewards/margins": 0.3838973939418793,
      "rewards/rejected": -1.238176941871643,
      "step": 2250
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1403076230230006e-06,
      "logits/chosen": -0.04517320543527603,
      "logits/rejected": -0.029649287462234497,
      "logps/chosen": -352.4161376953125,
      "logps/rejected": -330.69232177734375,
      "loss": 0.5847,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.8612442016601562,
      "rewards/margins": 0.444766104221344,
      "rewards/rejected": -1.306010365486145,
      "step": 2260
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.11771601595586e-06,
      "logits/chosen": -0.05350247770547867,
      "logits/rejected": -0.04654960706830025,
      "logps/chosen": -330.7650451660156,
      "logps/rejected": -376.1535339355469,
      "loss": 0.5635,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.7328948974609375,
      "rewards/margins": 0.4829980731010437,
      "rewards/rejected": -1.215893030166626,
      "step": 2270
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0951563294913737e-06,
      "logits/chosen": -0.14542168378829956,
      "logits/rejected": -0.09369027614593506,
      "logps/chosen": -351.79052734375,
      "logps/rejected": -338.8112487792969,
      "loss": 0.6275,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.9116989374160767,
      "rewards/margins": 0.3388912081718445,
      "rewards/rejected": -1.250590205192566,
      "step": 2280
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0726304473568693e-06,
      "logits/chosen": -0.08726407587528229,
      "logits/rejected": -0.07750742137432098,
      "logps/chosen": -325.08526611328125,
      "logps/rejected": -386.10626220703125,
      "loss": 0.5793,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7758111357688904,
      "rewards/margins": 0.4725108742713928,
      "rewards/rejected": -1.2483221292495728,
      "step": 2290
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.050140250457023e-06,
      "logits/chosen": -0.07832643389701843,
      "logits/rejected": -0.0965295284986496,
      "logps/chosen": -324.20269775390625,
      "logps/rejected": -357.1085510253906,
      "loss": 0.5636,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.7607049942016602,
      "rewards/margins": 0.48484665155410767,
      "rewards/rejected": -1.245551586151123,
      "step": 2300
    },
    {
      "epoch": 0.6,
      "eval_logits/chosen": -0.08411271125078201,
      "eval_logits/rejected": -0.048063673079013824,
      "eval_logps/chosen": -354.1103820800781,
      "eval_logps/rejected": -364.19696044921875,
      "eval_loss": 0.5836405158042908,
      "eval_rewards/accuracies": 0.6334999799728394,
      "eval_rewards/chosen": -0.76563560962677,
      "eval_rewards/margins": 0.438174843788147,
      "eval_rewards/rejected": -1.203810453414917,
      "eval_runtime": 544.4264,
      "eval_samples_per_second": 3.674,
      "eval_steps_per_second": 0.918,
      "step": 2300
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0276876167168042e-06,
      "logits/chosen": -0.0398612916469574,
      "logits/rejected": 0.02817544713616371,
      "logps/chosen": -361.60784912109375,
      "logps/rejected": -378.54473876953125,
      "loss": 0.5713,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.7841297388076782,
      "rewards/margins": 0.4361855089664459,
      "rewards/rejected": -1.2203152179718018,
      "step": 2310
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.0052744209246682e-06,
      "logits/chosen": -0.11950035393238068,
      "logits/rejected": -0.09671922028064728,
      "logps/chosen": -326.01458740234375,
      "logps/rejected": -356.91595458984375,
      "loss": 0.5813,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.7009235620498657,
      "rewards/margins": 0.41417956352233887,
      "rewards/rejected": -1.1151031255722046,
      "step": 2320
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.9829025345760127e-06,
      "logits/chosen": -0.10013041645288467,
      "logits/rejected": -0.09448160976171494,
      "logps/chosen": -394.4470520019531,
      "logps/rejected": -384.83270263671875,
      "loss": 0.5991,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.8150526285171509,
      "rewards/margins": 0.39240118861198425,
      "rewards/rejected": -1.2074538469314575,
      "step": 2330
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.9605738257169115e-06,
      "logits/chosen": -0.02237033285200596,
      "logits/rejected": -0.04650190472602844,
      "logps/chosen": -323.2276916503906,
      "logps/rejected": -357.5064697265625,
      "loss": 0.6222,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.7723985910415649,
      "rewards/margins": 0.3768603801727295,
      "rewards/rejected": -1.1492589712142944,
      "step": 2340
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.9382901587881275e-06,
      "logits/chosen": -0.08437281101942062,
      "logits/rejected": -0.03884163498878479,
      "logps/chosen": -325.86907958984375,
      "logps/rejected": -347.9014892578125,
      "loss": 0.5916,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.6954203844070435,
      "rewards/margins": 0.40348443388938904,
      "rewards/rejected": -1.0989047288894653,
      "step": 2350
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.916053394469437e-06,
      "logits/chosen": -0.058065224438905716,
      "logits/rejected": -0.028360579162836075,
      "logps/chosen": -287.13189697265625,
      "logps/rejected": -332.1973571777344,
      "loss": 0.5753,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.6241050958633423,
      "rewards/margins": 0.4590820372104645,
      "rewards/rejected": -1.083187222480774,
      "step": 2360
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8938653895242604e-06,
      "logits/chosen": -0.03688746690750122,
      "logits/rejected": -0.03670436143875122,
      "logps/chosen": -300.11126708984375,
      "logps/rejected": -324.38092041015625,
      "loss": 0.5589,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.5876749753952026,
      "rewards/margins": 0.47541046142578125,
      "rewards/rejected": -1.0630855560302734,
      "step": 2370
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8717279966446267e-06,
      "logits/chosen": -0.09554643929004669,
      "logits/rejected": -0.03961080312728882,
      "logps/chosen": -335.5707702636719,
      "logps/rejected": -373.040283203125,
      "loss": 0.5263,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.6644365787506104,
      "rewards/margins": 0.5852737426757812,
      "rewards/rejected": -1.2497103214263916,
      "step": 2380
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8496430642964698e-06,
      "logits/chosen": -0.1166432723402977,
      "logits/rejected": -0.0716785341501236,
      "logps/chosen": -326.2940979003906,
      "logps/rejected": -342.4819030761719,
      "loss": 0.5844,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.7313794493675232,
      "rewards/margins": 0.3761158585548401,
      "rewards/rejected": -1.1074954271316528,
      "step": 2390
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.827612436565286e-06,
      "logits/chosen": -0.11490534245967865,
      "logits/rejected": -0.09440793097019196,
      "logps/chosen": -334.36724853515625,
      "logps/rejected": -351.0407409667969,
      "loss": 0.5846,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.8265805244445801,
      "rewards/margins": 0.4359659254550934,
      "rewards/rejected": -1.262546420097351,
      "step": 2400
    },
    {
      "epoch": 0.63,
      "eval_logits/chosen": -0.11999206244945526,
      "eval_logits/rejected": -0.0871233344078064,
      "eval_logps/chosen": -365.27813720703125,
      "eval_logps/rejected": -377.25079345703125,
      "eval_loss": 0.5803542733192444,
      "eval_rewards/accuracies": 0.6334999799728394,
      "eval_rewards/chosen": -0.8773132562637329,
      "eval_rewards/margins": 0.45703527331352234,
      "eval_rewards/rejected": -1.3343485593795776,
      "eval_runtime": 545.1487,
      "eval_samples_per_second": 3.669,
      "eval_steps_per_second": 0.917,
      "step": 2400
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8056379530021492e-06,
      "logits/chosen": -0.13810701668262482,
      "logits/rejected": -0.10457410663366318,
      "logps/chosen": -339.19525146484375,
      "logps/rejected": -346.99029541015625,
      "loss": 0.6025,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.8567565679550171,
      "rewards/margins": 0.4513315260410309,
      "rewards/rejected": -1.3080880641937256,
      "step": 2410
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.7837214484701154e-06,
      "logits/chosen": -0.08872968703508377,
      "logits/rejected": -0.06984793394804001,
      "logps/chosen": -364.3599548339844,
      "logps/rejected": -402.4684753417969,
      "loss": 0.538,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.9384418725967407,
      "rewards/margins": 0.5457102060317993,
      "rewards/rejected": -1.48415207862854,
      "step": 2420
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7618647529910043e-06,
      "logits/chosen": -0.11679291725158691,
      "logits/rejected": -0.02643129602074623,
      "logps/chosen": -353.79022216796875,
      "logps/rejected": -382.530517578125,
      "loss": 0.5517,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.9540901184082031,
      "rewards/margins": 0.555814802646637,
      "rewards/rejected": -1.5099048614501953,
      "step": 2430
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7400696915925996e-06,
      "logits/chosen": -0.09259249269962311,
      "logits/rejected": -0.04453275352716446,
      "logps/chosen": -364.08837890625,
      "logps/rejected": -407.524169921875,
      "loss": 0.5457,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.8758049011230469,
      "rewards/margins": 0.5904634594917297,
      "rewards/rejected": -1.4662683010101318,
      "step": 2440
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.718338084156254e-06,
      "logits/chosen": -0.08697889745235443,
      "logits/rejected": -0.06519980728626251,
      "logps/chosen": -361.45709228515625,
      "logps/rejected": -372.6293029785156,
      "loss": 0.5659,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.866511344909668,
      "rewards/margins": 0.4529503881931305,
      "rewards/rejected": -1.3194618225097656,
      "step": 2450
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.6966717452649372e-06,
      "logits/chosen": -0.1362774819135666,
      "logits/rejected": -0.11426180601119995,
      "logps/chosen": -371.2001037597656,
      "logps/rejected": -412.7559509277344,
      "loss": 0.5511,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.9300951957702637,
      "rewards/margins": 0.5544837713241577,
      "rewards/rejected": -1.4845788478851318,
      "step": 2460
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6750724840517103e-06,
      "logits/chosen": -0.08611822873353958,
      "logits/rejected": -0.05831156298518181,
      "logps/chosen": -380.17535400390625,
      "logps/rejected": -386.8459777832031,
      "loss": 0.5452,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.8702682256698608,
      "rewards/margins": 0.552179217338562,
      "rewards/rejected": -1.4224474430084229,
      "step": 2470
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6535421040486686e-06,
      "logits/chosen": -0.08759529888629913,
      "logits/rejected": -0.07618779689073563,
      "logps/chosen": -335.5084228515625,
      "logps/rejected": -350.1391296386719,
      "loss": 0.56,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.8075240850448608,
      "rewards/margins": 0.4414435029029846,
      "rewards/rejected": -1.2489675283432007,
      "step": 2480
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6320824030363458e-06,
      "logits/chosen": -0.10661186277866364,
      "logits/rejected": -0.008265363052487373,
      "logps/chosen": -349.26861572265625,
      "logps/rejected": -366.3367614746094,
      "loss": 0.5722,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.8367792963981628,
      "rewards/margins": 0.4592077136039734,
      "rewards/rejected": -1.2959868907928467,
      "step": 2490
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6106951728936028e-06,
      "logits/chosen": -0.0859353244304657,
      "logits/rejected": -0.04287604242563248,
      "logps/chosen": -309.8092346191406,
      "logps/rejected": -339.9555358886719,
      "loss": 0.5799,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.8133733868598938,
      "rewards/margins": 0.4632648527622223,
      "rewards/rejected": -1.276638150215149,
      "step": 2500
    },
    {
      "epoch": 0.65,
      "eval_logits/chosen": -0.09223704785108566,
      "eval_logits/rejected": -0.0575764924287796,
      "eval_logps/chosen": -361.7434997558594,
      "eval_logps/rejected": -374.26409912109375,
      "eval_loss": 0.5833784341812134,
      "eval_rewards/accuracies": 0.6340000033378601,
      "eval_rewards/chosen": -0.8419671058654785,
      "eval_rewards/margins": 0.4625144600868225,
      "eval_rewards/rejected": -1.3044817447662354,
      "eval_runtime": 544.9858,
      "eval_samples_per_second": 3.67,
      "eval_steps_per_second": 0.917,
      "step": 2500
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5893821994479996e-06,
      "logits/chosen": -0.038878120481967926,
      "logits/rejected": -0.01672372780740261,
      "logps/chosen": -355.44061279296875,
      "logps/rejected": -390.06854248046875,
      "loss": 0.5178,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.7850661277770996,
      "rewards/margins": 0.603364109992981,
      "rewards/rejected": -1.3884302377700806,
      "step": 2510
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5681452623266868e-06,
      "logits/chosen": -0.06229633837938309,
      "logits/rejected": -0.0467236191034317,
      "logps/chosen": -382.23602294921875,
      "logps/rejected": -387.97332763671875,
      "loss": 0.5756,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.8509229421615601,
      "rewards/margins": 0.5015177726745605,
      "rewards/rejected": -1.3524408340454102,
      "step": 2520
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5469861348078014e-06,
      "logits/chosen": -0.06945382058620453,
      "logits/rejected": -0.06613973528146744,
      "logps/chosen": -349.61444091796875,
      "logps/rejected": -358.06915283203125,
      "loss": 0.6046,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.8453807830810547,
      "rewards/margins": 0.39862456917762756,
      "rewards/rejected": -1.2440054416656494,
      "step": 2530
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5259065836724035e-06,
      "logits/chosen": -0.08767147362232208,
      "logits/rejected": -0.06391812860965729,
      "logps/chosen": -341.8178405761719,
      "logps/rejected": -372.5906677246094,
      "loss": 0.5727,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.7495141625404358,
      "rewards/margins": 0.47481536865234375,
      "rewards/rejected": -1.2243295907974243,
      "step": 2540
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.5049083690569456e-06,
      "logits/chosen": -0.10644855350255966,
      "logits/rejected": -0.0691339522600174,
      "logps/chosen": -362.25970458984375,
      "logps/rejected": -358.62139892578125,
      "loss": 0.591,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.8960472941398621,
      "rewards/margins": 0.43379873037338257,
      "rewards/rejected": -1.3298461437225342,
      "step": 2550
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4839932443063057e-06,
      "logits/chosen": -0.02317485585808754,
      "logits/rejected": -0.03716563433408737,
      "logps/chosen": -316.1383056640625,
      "logps/rejected": -324.560546875,
      "loss": 0.5806,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.7911745309829712,
      "rewards/margins": 0.4005607068538666,
      "rewards/rejected": -1.1917351484298706,
      "step": 2560
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4631629558273803e-06,
      "logits/chosen": -0.044547874480485916,
      "logits/rejected": -0.05655717849731445,
      "logps/chosen": -363.1641845703125,
      "logps/rejected": -377.47503662109375,
      "loss": 0.5422,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.744333803653717,
      "rewards/margins": 0.5686662793159485,
      "rewards/rejected": -1.3130000829696655,
      "step": 2570
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4424192429432657e-06,
      "logits/chosen": 0.012710513547062874,
      "logits/rejected": -0.007722015026956797,
      "logps/chosen": -382.29779052734375,
      "logps/rejected": -335.0196838378906,
      "loss": 0.5918,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.7723060846328735,
      "rewards/margins": 0.41917362809181213,
      "rewards/rejected": -1.1914796829223633,
      "step": 2580
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.421763837748016e-06,
      "logits/chosen": -0.06088981777429581,
      "logits/rejected": -0.05146254226565361,
      "logps/chosen": -351.4329528808594,
      "logps/rejected": -388.3905944824219,
      "loss": 0.6294,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.8842247724533081,
      "rewards/margins": 0.37267476320266724,
      "rewards/rejected": -1.2568994760513306,
      "step": 2590
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.401198464962021e-06,
      "logits/chosen": -0.03741256147623062,
      "logits/rejected": -0.06474824994802475,
      "logps/chosen": -347.2456970214844,
      "logps/rejected": -356.04510498046875,
      "loss": 0.5565,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.775657057762146,
      "rewards/margins": 0.47477155923843384,
      "rewards/rejected": -1.250428557395935,
      "step": 2600
    },
    {
      "epoch": 0.68,
      "eval_logits/chosen": -0.06430140882730484,
      "eval_logits/rejected": -0.028463589027523994,
      "eval_logps/chosen": -357.6354675292969,
      "eval_logps/rejected": -369.3044128417969,
      "eval_loss": 0.5810229778289795,
      "eval_rewards/accuracies": 0.6345000267028809,
      "eval_rewards/chosen": -0.8008865714073181,
      "eval_rewards/margins": 0.45399823784828186,
      "eval_rewards/rejected": -1.2548847198486328,
      "eval_runtime": 545.1722,
      "eval_samples_per_second": 3.669,
      "eval_steps_per_second": 0.917,
      "step": 2600
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.3807248417879896e-06,
      "logits/chosen": -0.04454026371240616,
      "logits/rejected": -0.05448485538363457,
      "logps/chosen": -329.34661865234375,
      "logps/rejected": -327.6650695800781,
      "loss": 0.573,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.7315627336502075,
      "rewards/margins": 0.4313267767429352,
      "rewards/rejected": -1.1628895998001099,
      "step": 2610
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3603446777675665e-06,
      "logits/chosen": -0.06595277786254883,
      "logits/rejected": -0.06403062492609024,
      "logps/chosen": -338.12213134765625,
      "logps/rejected": -354.1922607421875,
      "loss": 0.5974,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.8146448135375977,
      "rewards/margins": 0.40636712312698364,
      "rewards/rejected": -1.2210118770599365,
      "step": 2620
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3400596746385817e-06,
      "logits/chosen": -0.04939567297697067,
      "logits/rejected": -0.0366520918905735,
      "logps/chosen": -371.56658935546875,
      "logps/rejected": -373.9921569824219,
      "loss": 0.588,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.8429405093193054,
      "rewards/margins": 0.44567233324050903,
      "rewards/rejected": -1.2886126041412354,
      "step": 2630
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3198715261929587e-06,
      "logits/chosen": -0.04225140064954758,
      "logits/rejected": -0.0014917313819751143,
      "logps/chosen": -399.6859436035156,
      "logps/rejected": -362.3399658203125,
      "loss": 0.5701,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.8035332560539246,
      "rewards/margins": 0.4362492561340332,
      "rewards/rejected": -1.2397825717926025,
      "step": 2640
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.2997819181352823e-06,
      "logits/chosen": -0.0454365499317646,
      "logits/rejected": -0.055598776787519455,
      "logps/chosen": -379.8087463378906,
      "logps/rejected": -388.2283630371094,
      "loss": 0.5703,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.8369846343994141,
      "rewards/margins": 0.4632073938846588,
      "rewards/rejected": -1.30019211769104,
      "step": 2650
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2797925279420454e-06,
      "logits/chosen": -0.0032060600351542234,
      "logits/rejected": 0.0273395087569952,
      "logps/chosen": -372.40802001953125,
      "logps/rejected": -371.7253112792969,
      "loss": 0.62,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.8528251647949219,
      "rewards/margins": 0.36443841457366943,
      "rewards/rejected": -1.2172635793685913,
      "step": 2660
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2599050247215764e-06,
      "logits/chosen": -0.01836007833480835,
      "logits/rejected": -0.006750327534973621,
      "logps/chosen": -360.90728759765625,
      "logps/rejected": -392.369140625,
      "loss": 0.5012,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.7728686332702637,
      "rewards/margins": 0.625950813293457,
      "rewards/rejected": -1.3988194465637207,
      "step": 2670
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2401210690746705e-06,
      "logits/chosen": -0.01354785542935133,
      "logits/rejected": 0.06222701072692871,
      "logps/chosen": -352.80438232421875,
      "logps/rejected": -359.9801330566406,
      "loss": 0.6309,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.9136732220649719,
      "rewards/margins": 0.35234978795051575,
      "rewards/rejected": -1.2660231590270996,
      "step": 2680
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2204423129559306e-06,
      "logits/chosen": -0.11001361906528473,
      "logits/rejected": -0.03707585483789444,
      "logps/chosen": -392.40118408203125,
      "logps/rejected": -391.91204833984375,
      "loss": 0.6266,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.9878837466239929,
      "rewards/margins": 0.30070778727531433,
      "rewards/rejected": -1.2885915040969849,
      "step": 2690
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.20087039953583e-06,
      "logits/chosen": -0.03697098046541214,
      "logits/rejected": -0.031819600611925125,
      "logps/chosen": -363.90045166015625,
      "logps/rejected": -389.9029541015625,
      "loss": 0.5614,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.8472734689712524,
      "rewards/margins": 0.538908839225769,
      "rewards/rejected": -1.386182188987732,
      "step": 2700
    },
    {
      "epoch": 0.71,
      "eval_logits/chosen": -0.06983576714992523,
      "eval_logits/rejected": -0.035794876515865326,
      "eval_logps/chosen": -372.7677307128906,
      "eval_logps/rejected": -385.643310546875,
      "eval_loss": 0.5781762599945068,
      "eval_rewards/accuracies": 0.6324999928474426,
      "eval_rewards/chosen": -0.9522089958190918,
      "eval_rewards/margins": 0.46606478095054626,
      "eval_rewards/rejected": -1.4182738065719604,
      "eval_runtime": 544.3902,
      "eval_samples_per_second": 3.674,
      "eval_steps_per_second": 0.918,
      "step": 2700
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.181406963063507e-06,
      "logits/chosen": -0.035129472613334656,
      "logits/rejected": -0.05755387619137764,
      "logps/chosen": -368.37615966796875,
      "logps/rejected": -361.5987243652344,
      "loss": 0.5847,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.9345572590827942,
      "rewards/margins": 0.43491941690444946,
      "rewards/rejected": -1.369476556777954,
      "step": 2710
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1620536287303052e-06,
      "logits/chosen": -0.03167080506682396,
      "logits/rejected": -0.009920826181769371,
      "logps/chosen": -349.4122314453125,
      "logps/rejected": -378.13641357421875,
      "loss": 0.5641,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.9168907403945923,
      "rewards/margins": 0.5256475210189819,
      "rewards/rejected": -1.4425382614135742,
      "step": 2720
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1428120125340717e-06,
      "logits/chosen": -0.0319071002304554,
      "logits/rejected": -0.09542439877986908,
      "logps/chosen": -364.5460510253906,
      "logps/rejected": -374.586669921875,
      "loss": 0.5987,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.9462617039680481,
      "rewards/margins": 0.45422202348709106,
      "rewards/rejected": -1.4004836082458496,
      "step": 2730
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.123683721144223e-06,
      "logits/chosen": -0.06852748990058899,
      "logits/rejected": -0.08457393199205399,
      "logps/chosen": -362.50433349609375,
      "logps/rejected": -380.47039794921875,
      "loss": 0.5369,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.930228590965271,
      "rewards/margins": 0.5651718378067017,
      "rewards/rejected": -1.4954004287719727,
      "step": 2740
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.1046703517675848e-06,
      "logits/chosen": -0.049139805138111115,
      "logits/rejected": -0.047232285141944885,
      "logps/chosen": -340.8487243652344,
      "logps/rejected": -377.1474304199219,
      "loss": 0.6067,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.9621003270149231,
      "rewards/margins": 0.389037549495697,
      "rewards/rejected": -1.3511378765106201,
      "step": 2750
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.085773492015028e-06,
      "logits/chosen": -0.07854189723730087,
      "logits/rejected": -0.054329145699739456,
      "logps/chosen": -389.5224914550781,
      "logps/rejected": -377.756103515625,
      "loss": 0.5664,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.0421987771987915,
      "rewards/margins": 0.48210200667381287,
      "rewards/rejected": -1.5243008136749268,
      "step": 2760
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.0669947197689034e-06,
      "logits/chosen": -0.06451062858104706,
      "logits/rejected": -0.047071583569049835,
      "logps/chosen": -326.9189758300781,
      "logps/rejected": -366.94000244140625,
      "loss": 0.641,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.016249656677246,
      "rewards/margins": 0.30962103605270386,
      "rewards/rejected": -1.3258706331253052,
      "step": 2770
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.048335603051291e-06,
      "logits/chosen": -0.09542266279459,
      "logits/rejected": -0.05664687231183052,
      "logps/chosen": -363.3104248046875,
      "logps/rejected": -388.03424072265625,
      "loss": 0.5604,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.9354135394096375,
      "rewards/margins": 0.4914797842502594,
      "rewards/rejected": -1.4268933534622192,
      "step": 2780
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0297976998930665e-06,
      "logits/chosen": -0.07029617577791214,
      "logits/rejected": -0.07720007747411728,
      "logps/chosen": -372.33538818359375,
      "logps/rejected": -371.91656494140625,
      "loss": 0.6013,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.960688591003418,
      "rewards/margins": 0.40848368406295776,
      "rewards/rejected": -1.3691723346710205,
      "step": 2790
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0113825582038078e-06,
      "logits/chosen": -0.051131028681993484,
      "logits/rejected": -0.03366447612643242,
      "logps/chosen": -368.6939392089844,
      "logps/rejected": -360.71551513671875,
      "loss": 0.608,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.0129467248916626,
      "rewards/margins": 0.43145838379859924,
      "rewards/rejected": -1.4444050788879395,
      "step": 2800
    },
    {
      "epoch": 0.73,
      "eval_logits/chosen": -0.05713397264480591,
      "eval_logits/rejected": -0.022933386266231537,
      "eval_logps/chosen": -371.3293151855469,
      "eval_logps/rejected": -383.7585144042969,
      "eval_loss": 0.5776425004005432,
      "eval_rewards/accuracies": 0.6359999775886536,
      "eval_rewards/chosen": -0.9378249645233154,
      "eval_rewards/margins": 0.46160098910331726,
      "eval_rewards/rejected": -1.399425983428955,
      "eval_runtime": 545.1619,
      "eval_samples_per_second": 3.669,
      "eval_steps_per_second": 0.917,
      "step": 2800
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.930917156425477e-07,
      "logits/chosen": -0.035455573350191116,
      "logits/rejected": 0.024086173623800278,
      "logps/chosen": -332.34405517578125,
      "logps/rejected": -361.7284240722656,
      "loss": 0.5317,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.8735530972480774,
      "rewards/margins": 0.5774091482162476,
      "rewards/rejected": -1.4509623050689697,
      "step": 2810
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.749266994893756e-07,
      "logits/chosen": -0.05711311846971512,
      "logits/rejected": -0.02643768861889839,
      "logps/chosen": -336.5233459472656,
      "logps/rejected": -384.072265625,
      "loss": 0.6026,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.9693512916564941,
      "rewards/margins": 0.39003175497055054,
      "rewards/rejected": -1.359383225440979,
      "step": 2820
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.56889026517913e-07,
      "logits/chosen": -0.03704976290464401,
      "logits/rejected": -0.0004162020923104137,
      "logps/chosen": -362.77703857421875,
      "logps/rejected": -375.52349853515625,
      "loss": 0.5474,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.9288007616996765,
      "rewards/margins": 0.5011378526687622,
      "rewards/rejected": -1.429938554763794,
      "step": 2830
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.389802028686617e-07,
      "logits/chosen": -0.03835465759038925,
      "logits/rejected": -0.016497110947966576,
      "logps/chosen": -301.2439270019531,
      "logps/rejected": -307.9148864746094,
      "loss": 0.607,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.851264476776123,
      "rewards/margins": 0.3882144093513489,
      "rewards/rejected": -1.2394790649414062,
      "step": 2840
    },
    {
      "epoch": 0.75,
      "learning_rate": 9.212017239232427e-07,
      "logits/chosen": -0.021128101274371147,
      "logits/rejected": 0.029263263568282127,
      "logps/chosen": -350.95611572265625,
      "logps/rejected": -408.35638427734375,
      "loss": 0.528,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.8077753782272339,
      "rewards/margins": 0.570160448551178,
      "rewards/rejected": -1.3779360055923462,
      "step": 2850
    },
    {
      "epoch": 0.75,
      "learning_rate": 9.03555074179533e-07,
      "logits/chosen": -0.06519512832164764,
      "logits/rejected": -0.003372351173311472,
      "logps/chosen": -351.7237548828125,
      "logps/rejected": -381.6541442871094,
      "loss": 0.5374,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.8637690544128418,
      "rewards/margins": 0.5793260335922241,
      "rewards/rejected": -1.443095088005066,
      "step": 2860
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.860417271277067e-07,
      "logits/chosen": -0.02651647850871086,
      "logits/rejected": 4.415661169332452e-05,
      "logps/chosen": -319.85113525390625,
      "logps/rejected": -349.617919921875,
      "loss": 0.613,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.7974709868431091,
      "rewards/margins": 0.3686184287071228,
      "rewards/rejected": -1.1660895347595215,
      "step": 2870
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.686631451272029e-07,
      "logits/chosen": -0.06533274054527283,
      "logits/rejected": -0.06797342002391815,
      "logps/chosen": -354.38055419921875,
      "logps/rejected": -345.04345703125,
      "loss": 0.5937,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.8840041160583496,
      "rewards/margins": 0.4283463954925537,
      "rewards/rejected": -1.3123505115509033,
      "step": 2880
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.514207792846168e-07,
      "logits/chosen": -0.019940800964832306,
      "logits/rejected": -0.029427438974380493,
      "logps/chosen": -332.21832275390625,
      "logps/rejected": -374.78057861328125,
      "loss": 0.5771,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.8241235613822937,
      "rewards/margins": 0.44140610098838806,
      "rewards/rejected": -1.2655296325683594,
      "step": 2890
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.343160693325356e-07,
      "logits/chosen": -0.06264803558588028,
      "logits/rejected": -0.056431032717227936,
      "logps/chosen": -308.91790771484375,
      "logps/rejected": -341.4127502441406,
      "loss": 0.588,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.7939392924308777,
      "rewards/margins": 0.4327456057071686,
      "rewards/rejected": -1.226684808731079,
      "step": 2900
    },
    {
      "epoch": 0.76,
      "eval_logits/chosen": -0.07915809750556946,
      "eval_logits/rejected": -0.04424045607447624,
      "eval_logps/chosen": -360.8502502441406,
      "eval_logps/rejected": -372.722412109375,
      "eval_loss": 0.5795174241065979,
      "eval_rewards/accuracies": 0.6345000267028809,
      "eval_rewards/chosen": -0.8330343961715698,
      "eval_rewards/margins": 0.4560302197933197,
      "eval_rewards/rejected": -1.2890645265579224,
      "eval_runtime": 545.3094,
      "eval_samples_per_second": 3.668,
      "eval_steps_per_second": 0.917,
      "step": 2900
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.173504435093174e-07,
      "logits/chosen": -0.07090556621551514,
      "logits/rejected": -0.05571649596095085,
      "logps/chosen": -374.98504638671875,
      "logps/rejected": -389.0013732910156,
      "loss": 0.6176,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.9230650067329407,
      "rewards/margins": 0.3573823869228363,
      "rewards/rejected": -1.2804473638534546,
      "step": 2910
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.00525318439836e-07,
      "logits/chosen": -0.05975785106420517,
      "logits/rejected": -0.047933854162693024,
      "logps/chosen": -353.83294677734375,
      "logps/rejected": -353.1571350097656,
      "loss": 0.5915,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.8541162610054016,
      "rewards/margins": 0.4437043070793152,
      "rewards/rejected": -1.2978206872940063,
      "step": 2920
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.838420990171927e-07,
      "logits/chosen": -0.07875867187976837,
      "logits/rejected": -0.03914656117558479,
      "logps/chosen": -388.00396728515625,
      "logps/rejected": -384.4292907714844,
      "loss": 0.5651,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.7708128690719604,
      "rewards/margins": 0.4774767756462097,
      "rewards/rejected": -1.2482898235321045,
      "step": 2930
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.673021782854084e-07,
      "logits/chosen": -0.03692690655589104,
      "logits/rejected": -0.04455095902085304,
      "logps/chosen": -327.183837890625,
      "logps/rejected": -382.13055419921875,
      "loss": 0.5735,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.7954224348068237,
      "rewards/margins": 0.46773552894592285,
      "rewards/rejected": -1.2631580829620361,
      "step": 2940
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.509069373231039e-07,
      "logits/chosen": -0.05857111141085625,
      "logits/rejected": -0.06518695503473282,
      "logps/chosen": -374.9933166503906,
      "logps/rejected": -397.5784912109375,
      "loss": 0.5719,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.8247407674789429,
      "rewards/margins": 0.5263202786445618,
      "rewards/rejected": -1.3510611057281494,
      "step": 2950
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.346577451281822e-07,
      "logits/chosen": -0.07813980430364609,
      "logits/rejected": -0.044347889721393585,
      "logps/chosen": -332.4440612792969,
      "logps/rejected": -346.2649230957031,
      "loss": 0.6059,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.8520699739456177,
      "rewards/margins": 0.38116154074668884,
      "rewards/rejected": -1.2332316637039185,
      "step": 2960
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.185559585035138e-07,
      "logits/chosen": -0.06604515016078949,
      "logits/rejected": -0.038625769317150116,
      "logps/chosen": -364.8755798339844,
      "logps/rejected": -371.142822265625,
      "loss": 0.5878,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.8049036264419556,
      "rewards/margins": 0.382902055978775,
      "rewards/rejected": -1.1878057718276978,
      "step": 2970
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.026029219436504e-07,
      "logits/chosen": -0.05492454767227173,
      "logits/rejected": -0.057348012924194336,
      "logps/chosen": -383.4015197753906,
      "logps/rejected": -387.12176513671875,
      "loss": 0.587,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.7947750091552734,
      "rewards/margins": 0.4462098181247711,
      "rewards/rejected": -1.2409846782684326,
      "step": 2980
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.867999675225523e-07,
      "logits/chosen": -0.03865230828523636,
      "logits/rejected": -0.05615551024675369,
      "logps/chosen": -369.0574645996094,
      "logps/rejected": -353.56817626953125,
      "loss": 0.6035,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7356120944023132,
      "rewards/margins": 0.3736583888530731,
      "rewards/rejected": -1.109270691871643,
      "step": 2990
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.711484147823663e-07,
      "logits/chosen": -0.06074659898877144,
      "logits/rejected": -0.058938294649124146,
      "logps/chosen": -339.2735595703125,
      "logps/rejected": -357.4136047363281,
      "loss": 0.5324,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.7124096751213074,
      "rewards/margins": 0.5349728465080261,
      "rewards/rejected": -1.247382402420044,
      "step": 3000
    },
    {
      "epoch": 0.79,
      "eval_logits/chosen": -0.064789317548275,
      "eval_logits/rejected": -0.029839608818292618,
      "eval_logps/chosen": -354.6904296875,
      "eval_logps/rejected": -365.1566162109375,
      "eval_loss": 0.5806678533554077,
      "eval_rewards/accuracies": 0.6340000033378601,
      "eval_rewards/chosen": -0.7714364528656006,
      "eval_rewards/margins": 0.4419707953929901,
      "eval_rewards/rejected": -1.213407278060913,
      "eval_runtime": 544.3782,
      "eval_samples_per_second": 3.674,
      "eval_steps_per_second": 0.918,
      "step": 3000
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.556495706232413e-07,
      "logits/chosen": -0.04762962460517883,
      "logits/rejected": -0.026001859456300735,
      "logps/chosen": -345.75860595703125,
      "logps/rejected": -352.12677001953125,
      "loss": 0.5552,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.7305065393447876,
      "rewards/margins": 0.48486828804016113,
      "rewards/rejected": -1.2153748273849487,
      "step": 3010
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.403047291942057e-07,
      "logits/chosen": -0.06475412100553513,
      "logits/rejected": -0.04636804386973381,
      "logps/chosen": -351.92230224609375,
      "logps/rejected": -373.903564453125,
      "loss": 0.585,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.7460962533950806,
      "rewards/margins": 0.4199795126914978,
      "rewards/rejected": -1.1660758256912231,
      "step": 3020
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.251151717851023e-07,
      "logits/chosen": -0.084681436419487,
      "logits/rejected": -0.07976353913545609,
      "logps/chosen": -377.3009033203125,
      "logps/rejected": -403.86688232421875,
      "loss": 0.603,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.8924350738525391,
      "rewards/margins": 0.5002483129501343,
      "rewards/rejected": -1.3926832675933838,
      "step": 3030
    },
    {
      "epoch": 0.8,
      "learning_rate": 6.100821667196041e-07,
      "logits/chosen": -0.023926103487610817,
      "logits/rejected": -0.0013149696169421077,
      "logps/chosen": -373.28125,
      "logps/rejected": -354.6680908203125,
      "loss": 0.5745,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7040396332740784,
      "rewards/margins": 0.43626755475997925,
      "rewards/rejected": -1.1403071880340576,
      "step": 3040
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.952069692493062e-07,
      "logits/chosen": -0.0607212670147419,
      "logits/rejected": -0.030897587537765503,
      "logps/chosen": -368.138916015625,
      "logps/rejected": -378.4091796875,
      "loss": 0.526,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.6818228960037231,
      "rewards/margins": 0.5211673974990845,
      "rewards/rejected": -1.2029902935028076,
      "step": 3050
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.80490821448918e-07,
      "logits/chosen": -0.03866980969905853,
      "logits/rejected": -0.04572879895567894,
      "logps/chosen": -291.18768310546875,
      "logps/rejected": -308.963134765625,
      "loss": 0.6146,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7203935980796814,
      "rewards/margins": 0.39186891913414,
      "rewards/rejected": -1.112262487411499,
      "step": 3060
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.659349521125459e-07,
      "logits/chosen": -0.08902426809072495,
      "logits/rejected": -0.05652226135134697,
      "logps/chosen": -306.17620849609375,
      "logps/rejected": -353.67633056640625,
      "loss": 0.5513,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.7082656025886536,
      "rewards/margins": 0.4930747151374817,
      "rewards/rejected": -1.2013403177261353,
      "step": 3070
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.5154057665109e-07,
      "logits/chosen": -0.055385757237672806,
      "logits/rejected": -0.04034740477800369,
      "logps/chosen": -354.96563720703125,
      "logps/rejected": -360.63677978515625,
      "loss": 0.5646,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.7717067003250122,
      "rewards/margins": 0.48541682958602905,
      "rewards/rejected": -1.257123589515686,
      "step": 3080
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.373088969907586e-07,
      "logits/chosen": -0.09343139082193375,
      "logits/rejected": -0.045906342566013336,
      "logps/chosen": -329.02557373046875,
      "logps/rejected": -373.9976501464844,
      "loss": 0.5557,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.7075044512748718,
      "rewards/margins": 0.5255839228630066,
      "rewards/rejected": -1.2330883741378784,
      "step": 3090
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.23241101472709e-07,
      "logits/chosen": -0.03690626472234726,
      "logits/rejected": -0.054735153913497925,
      "logps/chosen": -324.6033935546875,
      "logps/rejected": -346.44830322265625,
      "loss": 0.6036,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.7127725481987,
      "rewards/margins": 0.39219218492507935,
      "rewards/rejected": -1.1049648523330688,
      "step": 3100
    },
    {
      "epoch": 0.81,
      "eval_logits/chosen": -0.07098916918039322,
      "eval_logits/rejected": -0.03594445437192917,
      "eval_logps/chosen": -352.0881042480469,
      "eval_logps/rejected": -362.20758056640625,
      "eval_loss": 0.5816840529441833,
      "eval_rewards/accuracies": 0.6359999775886536,
      "eval_rewards/chosen": -0.7454131245613098,
      "eval_rewards/margins": 0.4385035037994385,
      "eval_rewards/rejected": -1.1839165687561035,
      "eval_runtime": 544.6044,
      "eval_samples_per_second": 3.672,
      "eval_steps_per_second": 0.918,
      "step": 3100
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.09338364753818e-07,
      "logits/chosen": -0.09596423804759979,
      "logits/rejected": -0.0356723852455616,
      "logps/chosen": -363.4578857421875,
      "logps/rejected": -385.2988586425781,
      "loss": 0.5387,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.724037230014801,
      "rewards/margins": 0.5893635153770447,
      "rewards/rejected": -1.3134007453918457,
      "step": 3110
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.956018477086005e-07,
      "logits/chosen": -0.023332688957452774,
      "logits/rejected": -0.05660749599337578,
      "logps/chosen": -351.01885986328125,
      "logps/rejected": -340.53472900390625,
      "loss": 0.5815,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.7153294682502747,
      "rewards/margins": 0.46078458428382874,
      "rewards/rejected": -1.1761140823364258,
      "step": 3120
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.820326973322764e-07,
      "logits/chosen": -0.028302233666181564,
      "logits/rejected": -0.03387041389942169,
      "logps/chosen": -360.9173889160156,
      "logps/rejected": -340.7845458984375,
      "loss": 0.5284,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.664130687713623,
      "rewards/margins": 0.609849750995636,
      "rewards/rejected": -1.2739803791046143,
      "step": 3130
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.686320466449981e-07,
      "logits/chosen": -0.05822853371500969,
      "logits/rejected": -0.05845784395933151,
      "logps/chosen": -303.1192321777344,
      "logps/rejected": -334.73553466796875,
      "loss": 0.5321,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.6799777746200562,
      "rewards/margins": 0.5427129864692688,
      "rewards/rejected": -1.2226907014846802,
      "step": 3140
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.554010145972418e-07,
      "logits/chosen": -0.06433075666427612,
      "logits/rejected": -0.04098791256546974,
      "logps/chosen": -356.281982421875,
      "logps/rejected": -342.19537353515625,
      "loss": 0.581,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.7194234132766724,
      "rewards/margins": 0.4737864136695862,
      "rewards/rejected": -1.1932098865509033,
      "step": 3150
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.4234070597637455e-07,
      "logits/chosen": -0.07401960343122482,
      "logits/rejected": -0.006474087946116924,
      "logps/chosen": -350.6880798339844,
      "logps/rejected": -360.044677734375,
      "loss": 0.6235,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.8287215232849121,
      "rewards/margins": 0.3001879155635834,
      "rewards/rejected": -1.1289094686508179,
      "step": 3160
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.2945221131440783e-07,
      "logits/chosen": -0.05917246267199516,
      "logits/rejected": -0.042995236814022064,
      "logps/chosen": -318.0530700683594,
      "logps/rejected": -330.96832275390625,
      "loss": 0.5841,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.7468527555465698,
      "rewards/margins": 0.4306907057762146,
      "rewards/rejected": -1.1775435209274292,
      "step": 3170
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.167366067969381e-07,
      "logits/chosen": -0.09087510406970978,
      "logits/rejected": -0.07458924502134323,
      "logps/chosen": -342.1099548339844,
      "logps/rejected": -375.07635498046875,
      "loss": 0.5468,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.7130734324455261,
      "rewards/margins": 0.5242506861686707,
      "rewards/rejected": -1.2373241186141968,
      "step": 3180
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.041949541732826e-07,
      "logits/chosen": -0.01006515882909298,
      "logits/rejected": 0.011426875367760658,
      "logps/chosen": -374.0743103027344,
      "logps/rejected": -352.95697021484375,
      "loss": 0.6166,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.8070684671401978,
      "rewards/margins": 0.3434571623802185,
      "rewards/rejected": -1.150525450706482,
      "step": 3190
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.9182830066782614e-07,
      "logits/chosen": -0.06810037791728973,
      "logits/rejected": -0.07136163860559464,
      "logps/chosen": -360.11090087890625,
      "logps/rejected": -351.59515380859375,
      "loss": 0.615,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.8139978647232056,
      "rewards/margins": 0.3238958418369293,
      "rewards/rejected": -1.1378936767578125,
      "step": 3200
    },
    {
      "epoch": 0.84,
      "eval_logits/chosen": -0.06448013335466385,
      "eval_logits/rejected": -0.029476074501872063,
      "eval_logps/chosen": -353.8468933105469,
      "eval_logps/rejected": -364.4670104980469,
      "eval_loss": 0.5805965662002563,
      "eval_rewards/accuracies": 0.6330000162124634,
      "eval_rewards/chosen": -0.7630012035369873,
      "eval_rewards/margins": 0.443509578704834,
      "eval_rewards/rejected": -1.2065109014511108,
      "eval_runtime": 545.1592,
      "eval_samples_per_second": 3.669,
      "eval_steps_per_second": 0.917,
      "step": 3200
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.796376788925771e-07,
      "logits/chosen": -0.07999769598245621,
      "logits/rejected": -0.016174430027604103,
      "logps/chosen": -317.23468017578125,
      "logps/rejected": -333.4161682128906,
      "loss": 0.5777,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.7274065017700195,
      "rewards/margins": 0.44895920157432556,
      "rewards/rejected": -1.1763657331466675,
      "step": 3210
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.676241067609465e-07,
      "logits/chosen": -0.065385602414608,
      "logits/rejected": -0.08215048164129257,
      "logps/chosen": -313.9781799316406,
      "logps/rejected": -344.7863464355469,
      "loss": 0.5884,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.7430699467658997,
      "rewards/margins": 0.42152637243270874,
      "rewards/rejected": -1.1645963191986084,
      "step": 3220
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.5578858740274976e-07,
      "logits/chosen": -0.049977466464042664,
      "logits/rejected": -0.022836443036794662,
      "logps/chosen": -371.64337158203125,
      "logps/rejected": -372.68505859375,
      "loss": 0.5669,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.7283893823623657,
      "rewards/margins": 0.4756964147090912,
      "rewards/rejected": -1.2040858268737793,
      "step": 3230
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.44132109080447e-07,
      "logits/chosen": -0.07746875286102295,
      "logits/rejected": -0.013276422396302223,
      "logps/chosen": -342.2890625,
      "logps/rejected": -340.8277282714844,
      "loss": 0.6697,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.8580360412597656,
      "rewards/margins": 0.2627985179424286,
      "rewards/rejected": -1.1208345890045166,
      "step": 3240
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.3265564510662344e-07,
      "logits/chosen": -0.04408205673098564,
      "logits/rejected": -0.006346794776618481,
      "logps/chosen": -323.6293640136719,
      "logps/rejected": -328.1676025390625,
      "loss": 0.5655,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.7188665866851807,
      "rewards/margins": 0.4509055018424988,
      "rewards/rejected": -1.1697720289230347,
      "step": 3250
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.213601537627195e-07,
      "logits/chosen": -0.02172664925456047,
      "logits/rejected": -0.02120097354054451,
      "logps/chosen": -335.9650573730469,
      "logps/rejected": -357.8267517089844,
      "loss": 0.6194,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.7779746055603027,
      "rewards/margins": 0.3631519675254822,
      "rewards/rejected": -1.1411265134811401,
      "step": 3260
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.1024657821901063e-07,
      "logits/chosen": -0.013195675797760487,
      "logits/rejected": -0.027406075969338417,
      "logps/chosen": -323.6843566894531,
      "logps/rejected": -349.3821105957031,
      "loss": 0.5386,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.7054836750030518,
      "rewards/margins": 0.501741886138916,
      "rewards/rejected": -1.2072255611419678,
      "step": 3270
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.9931584645585654e-07,
      "logits/chosen": 0.0001564420817885548,
      "logits/rejected": -0.0028263225685805082,
      "logps/chosen": -366.2295837402344,
      "logps/rejected": -400.4193115234375,
      "loss": 0.5951,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.8342908024787903,
      "rewards/margins": 0.4561484754085541,
      "rewards/rejected": -1.2904393672943115,
      "step": 3280
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.885688711862136e-07,
      "logits/chosen": -0.040136925876140594,
      "logits/rejected": -0.030243951827287674,
      "logps/chosen": -333.1867980957031,
      "logps/rejected": -329.26239013671875,
      "loss": 0.6174,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.7828749418258667,
      "rewards/margins": 0.3514179587364197,
      "rewards/rejected": -1.1342929601669312,
      "step": 3290
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.7800654977942486e-07,
      "logits/chosen": -0.05975431948900223,
      "logits/rejected": -0.024867946282029152,
      "logps/chosen": -372.686767578125,
      "logps/rejected": -331.25628662109375,
      "loss": 0.6211,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.8066738843917847,
      "rewards/margins": 0.33745184540748596,
      "rewards/rejected": -1.1441256999969482,
      "step": 3300
    },
    {
      "epoch": 0.86,
      "eval_logits/chosen": -0.0585113987326622,
      "eval_logits/rejected": -0.0239940844476223,
      "eval_logps/chosen": -355.21856689453125,
      "eval_logps/rejected": -365.8819885253906,
      "eval_loss": 0.5793652534484863,
      "eval_rewards/accuracies": 0.6334999799728394,
      "eval_rewards/chosen": -0.7767176032066345,
      "eval_rewards/margins": 0.4439431130886078,
      "eval_rewards/rejected": -1.2206608057022095,
      "eval_runtime": 544.4721,
      "eval_samples_per_second": 3.673,
      "eval_steps_per_second": 0.918,
      "step": 3300
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.6762976418628797e-07,
      "logits/chosen": -0.03742988407611847,
      "logits/rejected": -0.038391679525375366,
      "logps/chosen": -328.2310791015625,
      "logps/rejected": -339.87530517578125,
      "loss": 0.5982,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.777148425579071,
      "rewards/margins": 0.3832804560661316,
      "rewards/rejected": -1.1604288816452026,
      "step": 3310
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.5743938086541354e-07,
      "logits/chosen": -0.038022901862859726,
      "logits/rejected": -0.04044175148010254,
      "logps/chosen": -369.03594970703125,
      "logps/rejected": -378.79937744140625,
      "loss": 0.5056,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.7325864434242249,
      "rewards/margins": 0.6389061212539673,
      "rewards/rejected": -1.3714925050735474,
      "step": 3320
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.4743625071087574e-07,
      "logits/chosen": -0.08106885850429535,
      "logits/rejected": -0.0331001877784729,
      "logps/chosen": -350.80322265625,
      "logps/rejected": -370.1235656738281,
      "loss": 0.5724,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.8414134979248047,
      "rewards/margins": 0.4927336573600769,
      "rewards/rejected": -1.3341472148895264,
      "step": 3330
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.3762120898116498e-07,
      "logits/chosen": -0.045328982174396515,
      "logits/rejected": -0.03163378685712814,
      "logps/chosen": -341.0201416015625,
      "logps/rejected": -346.0947265625,
      "loss": 0.616,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.7547915577888489,
      "rewards/margins": 0.3216248154640198,
      "rewards/rejected": -1.0764163732528687,
      "step": 3340
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.2799507522944048e-07,
      "logits/chosen": 0.0050236075185239315,
      "logits/rejected": -0.00028944091172888875,
      "logps/chosen": -366.76171875,
      "logps/rejected": -338.931640625,
      "loss": 0.6071,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.729737401008606,
      "rewards/margins": 0.3199540972709656,
      "rewards/rejected": -1.0496914386749268,
      "step": 3350
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.1855865323510056e-07,
      "logits/chosen": -0.01202615536749363,
      "logits/rejected": 0.009629396721720695,
      "logps/chosen": -367.4909973144531,
      "logps/rejected": -385.5251159667969,
      "loss": 0.5707,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.7918294668197632,
      "rewards/margins": 0.4775357246398926,
      "rewards/rejected": -1.2693650722503662,
      "step": 3360
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.0931273093666575e-07,
      "logits/chosen": 0.0005378912901505828,
      "logits/rejected": 0.012183221988379955,
      "logps/chosen": -339.87249755859375,
      "logps/rejected": -380.7689208984375,
      "loss": 0.5681,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.7448574304580688,
      "rewards/margins": 0.48650699853897095,
      "rewards/rejected": -1.2313644886016846,
      "step": 3370
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.002580803659873e-07,
      "logits/chosen": -0.02887069061398506,
      "logits/rejected": -0.045493822544813156,
      "logps/chosen": -373.1261901855469,
      "logps/rejected": -381.5206604003906,
      "loss": 0.5724,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.831575870513916,
      "rewards/margins": 0.43848705291748047,
      "rewards/rejected": -1.2700629234313965,
      "step": 3380
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.913954575837826e-07,
      "logits/chosen": 0.019021127372980118,
      "logits/rejected": 0.05180368572473526,
      "logps/chosen": -373.37060546875,
      "logps/rejected": -386.58941650390625,
      "loss": 0.5659,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.8404852151870728,
      "rewards/margins": 0.47979259490966797,
      "rewards/rejected": -1.3202778100967407,
      "step": 3390
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.827256026165028e-07,
      "logits/chosen": -0.036004990339279175,
      "logits/rejected": -0.0617971308529377,
      "logps/chosen": -350.956298828125,
      "logps/rejected": -374.48272705078125,
      "loss": 0.535,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.8171570897102356,
      "rewards/margins": 0.6308014988899231,
      "rewards/rejected": -1.4479585886001587,
      "step": 3400
    },
    {
      "epoch": 0.89,
      "eval_logits/chosen": -0.05579707399010658,
      "eval_logits/rejected": -0.022538870573043823,
      "eval_logps/chosen": -361.53656005859375,
      "eval_logps/rejected": -373.102783203125,
      "eval_loss": 0.5777334570884705,
      "eval_rewards/accuracies": 0.6320000290870667,
      "eval_rewards/chosen": -0.8398973941802979,
      "eval_rewards/margins": 0.45297136902809143,
      "eval_rewards/rejected": -1.292868733406067,
      "eval_runtime": 545.3246,
      "eval_samples_per_second": 3.668,
      "eval_steps_per_second": 0.917,
      "step": 3400
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.7424923939454274e-07,
      "logits/chosen": -0.002502134535461664,
      "logits/rejected": -0.018689513206481934,
      "logps/chosen": -365.55181884765625,
      "logps/rejected": -376.9024963378906,
      "loss": 0.5815,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.8611559867858887,
      "rewards/margins": 0.43401390314102173,
      "rewards/rejected": -1.2951698303222656,
      "step": 3410
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.6596707569179304e-07,
      "logits/chosen": -0.0391705147922039,
      "logits/rejected": -0.029395347461104393,
      "logps/chosen": -344.93084716796875,
      "logps/rejected": -354.69110107421875,
      "loss": 0.5448,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.7967727780342102,
      "rewards/margins": 0.5383495092391968,
      "rewards/rejected": -1.3351223468780518,
      "step": 3420
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.578798030665385e-07,
      "logits/chosen": -0.02771691046655178,
      "logits/rejected": -0.03689311072230339,
      "logps/chosen": -369.16424560546875,
      "logps/rejected": -351.5723571777344,
      "loss": 0.577,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.8476957082748413,
      "rewards/margins": 0.4203530251979828,
      "rewards/rejected": -1.268048882484436,
      "step": 3430
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.499880968037165e-07,
      "logits/chosen": -0.05807967856526375,
      "logits/rejected": 0.006117827724665403,
      "logps/chosen": -368.61480712890625,
      "logps/rejected": -385.88482666015625,
      "loss": 0.5313,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.8740746378898621,
      "rewards/margins": 0.5491721034049988,
      "rewards/rejected": -1.4232467412948608,
      "step": 3440
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.4229261585852805e-07,
      "logits/chosen": -0.08000718057155609,
      "logits/rejected": -0.013188359327614307,
      "logps/chosen": -344.84844970703125,
      "logps/rejected": -366.2679138183594,
      "loss": 0.5822,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.8913000822067261,
      "rewards/margins": 0.4382581114768982,
      "rewards/rejected": -1.3295581340789795,
      "step": 3450
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.3479400280141886e-07,
      "logits/chosen": -0.05246477201581001,
      "logits/rejected": -0.064031220972538,
      "logps/chosen": -311.46075439453125,
      "logps/rejected": -326.723876953125,
      "loss": 0.5766,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.7297900915145874,
      "rewards/margins": 0.3982476592063904,
      "rewards/rejected": -1.128037691116333,
      "step": 3460
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.2749288376442044e-07,
      "logits/chosen": -0.013275760225951672,
      "logits/rejected": -0.015950758010149002,
      "logps/chosen": -327.622802734375,
      "logps/rejected": -358.119873046875,
      "loss": 0.5896,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.814193606376648,
      "rewards/margins": 0.4229184091091156,
      "rewards/rejected": -1.2371121644973755,
      "step": 3470
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.203898683888713e-07,
      "logits/chosen": -0.010835160501301289,
      "logits/rejected": -0.0052233547903597355,
      "logps/chosen": -340.2196350097656,
      "logps/rejected": -364.1300048828125,
      "loss": 0.5993,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.8792353868484497,
      "rewards/margins": 0.44398754835128784,
      "rewards/rejected": -1.3232228755950928,
      "step": 3480
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1348554977451132e-07,
      "logits/chosen": -0.02504626475274563,
      "logits/rejected": -0.034282900393009186,
      "logps/chosen": -324.88525390625,
      "logps/rejected": -377.1581726074219,
      "loss": 0.5687,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.8490117788314819,
      "rewards/margins": 0.49014702439308167,
      "rewards/rejected": -1.3391587734222412,
      "step": 3490
    },
    {
      "epoch": 0.92,
      "learning_rate": 1.0678050442995802e-07,
      "logits/chosen": -0.06707514822483063,
      "logits/rejected": -0.05451773479580879,
      "logps/chosen": -336.59759521484375,
      "logps/rejected": -344.66265869140625,
      "loss": 0.5322,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.7733240723609924,
      "rewards/margins": 0.5705543160438538,
      "rewards/rejected": -1.3438783884048462,
      "step": 3500
    },
    {
      "epoch": 0.92,
      "eval_logits/chosen": -0.05459979921579361,
      "eval_logits/rejected": -0.020987825468182564,
      "eval_logps/chosen": -360.141845703125,
      "eval_logps/rejected": -371.6272277832031,
      "eval_loss": 0.5779351592063904,
      "eval_rewards/accuracies": 0.6334999799728394,
      "eval_rewards/chosen": -0.825950562953949,
      "eval_rewards/margins": 0.4521624445915222,
      "eval_rewards/rejected": -1.2781130075454712,
      "eval_runtime": 544.5596,
      "eval_samples_per_second": 3.673,
      "eval_steps_per_second": 0.918,
      "step": 3500
    },
    {
      "epoch": 0.92,
      "learning_rate": 1.0027529222456755e-07,
      "logits/chosen": -0.02216022089123726,
      "logits/rejected": -0.042939335107803345,
      "logps/chosen": -348.16436767578125,
      "logps/rejected": -375.9942321777344,
      "loss": 0.5861,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.8684269189834595,
      "rewards/margins": 0.43816670775413513,
      "rewards/rejected": -1.306593656539917,
      "step": 3510
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.397045634168766e-08,
      "logits/chosen": -0.06902258098125458,
      "logits/rejected": 0.005351958330720663,
      "logps/chosen": -349.08087158203125,
      "logps/rejected": -350.078125,
      "loss": 0.5637,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.7523408532142639,
      "rewards/margins": 0.476735919713974,
      "rewards/rejected": -1.2290767431259155,
      "step": 3520
    },
    {
      "epoch": 0.92,
      "learning_rate": 8.78665232332998e-08,
      "logits/chosen": -0.04204776883125305,
      "logits/rejected": -0.0012271578889340162,
      "logps/chosen": -342.7869567871094,
      "logps/rejected": -374.901611328125,
      "loss": 0.5951,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.8473308682441711,
      "rewards/margins": 0.3966173529624939,
      "rewards/rejected": -1.243948221206665,
      "step": 3530
    },
    {
      "epoch": 0.93,
      "learning_rate": 8.196400257606208e-08,
      "logits/chosen": -0.07394047826528549,
      "logits/rejected": -0.030526086688041687,
      "logps/chosen": -361.0905456542969,
      "logps/rejected": -363.9695739746094,
      "loss": 0.605,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.8390079736709595,
      "rewards/margins": 0.3738323748111725,
      "rewards/rejected": -1.21284019947052,
      "step": 3540
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.626338722875076e-08,
      "logits/chosen": -0.025840681046247482,
      "logits/rejected": -0.05815352872014046,
      "logps/chosen": -363.9366149902344,
      "logps/rejected": -375.3655090332031,
      "loss": 0.5786,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8692588806152344,
      "rewards/margins": 0.4229603707790375,
      "rewards/rejected": -1.2922192811965942,
      "step": 3550
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.076515319110688e-08,
      "logits/chosen": 0.011594533920288086,
      "logits/rejected": -0.023100445047020912,
      "logps/chosen": -372.5281066894531,
      "logps/rejected": -373.2308654785156,
      "loss": 0.5711,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.8273743391036987,
      "rewards/margins": 0.46333497762680054,
      "rewards/rejected": -1.290709376335144,
      "step": 3560
    },
    {
      "epoch": 0.93,
      "learning_rate": 6.54697595640899e-08,
      "logits/chosen": 0.024370530620217323,
      "logits/rejected": -0.025257308036088943,
      "logps/chosen": -357.7359924316406,
      "logps/rejected": -373.8916015625,
      "loss": 0.5944,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.8015440702438354,
      "rewards/margins": 0.399541437625885,
      "rewards/rejected": -1.2010856866836548,
      "step": 3570
    },
    {
      "epoch": 0.94,
      "learning_rate": 6.037764851154426e-08,
      "logits/chosen": -0.061722129583358765,
      "logits/rejected": -0.017051083967089653,
      "logps/chosen": -358.57586669921875,
      "logps/rejected": -358.6295471191406,
      "loss": 0.5696,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.8154916763305664,
      "rewards/margins": 0.4519859254360199,
      "rewards/rejected": -1.2674776315689087,
      "step": 3580
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.548924522327748e-08,
      "logits/chosen": -0.08785736560821533,
      "logits/rejected": -0.004190725274384022,
      "logps/chosen": -355.2710876464844,
      "logps/rejected": -382.51019287109375,
      "loss": 0.5576,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.8198660016059875,
      "rewards/margins": 0.5074083805084229,
      "rewards/rejected": -1.3272743225097656,
      "step": 3590
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.0804957879556915e-08,
      "logits/chosen": -0.0530325286090374,
      "logits/rejected": -0.013538384810090065,
      "logps/chosen": -400.12078857421875,
      "logps/rejected": -389.3551330566406,
      "loss": 0.5527,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.8526579737663269,
      "rewards/margins": 0.523281455039978,
      "rewards/rejected": -1.3759396076202393,
      "step": 3600
    },
    {
      "epoch": 0.94,
      "eval_logits/chosen": -0.056537650525569916,
      "eval_logits/rejected": -0.022854406386613846,
      "eval_logps/chosen": -360.0846862792969,
      "eval_logps/rejected": -371.6083068847656,
      "eval_loss": 0.5779842734336853,
      "eval_rewards/accuracies": 0.6315000057220459,
      "eval_rewards/chosen": -0.8253786563873291,
      "eval_rewards/margins": 0.4525452256202698,
      "eval_rewards/rejected": -1.277923822402954,
      "eval_runtime": 544.3878,
      "eval_samples_per_second": 3.674,
      "eval_steps_per_second": 0.918,
      "step": 3600
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.632517761702815e-08,
      "logits/chosen": -0.02192116342484951,
      "logits/rejected": -0.0027442649006843567,
      "logps/chosen": -359.52862548828125,
      "logps/rejected": -368.43524169921875,
      "loss": 0.5519,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.7623158097267151,
      "rewards/margins": 0.5025709271430969,
      "rewards/rejected": -1.2648866176605225,
      "step": 3610
    },
    {
      "epoch": 0.95,
      "learning_rate": 4.205027849605359e-08,
      "logits/chosen": -0.03419053927063942,
      "logits/rejected": -0.0502307191491127,
      "logps/chosen": -303.33673095703125,
      "logps/rejected": -344.5325012207031,
      "loss": 0.5772,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.7383955717086792,
      "rewards/margins": 0.42240723967552185,
      "rewards/rejected": -1.1608028411865234,
      "step": 3620
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.798061746947995e-08,
      "logits/chosen": -0.033175647258758545,
      "logits/rejected": -0.056212086230516434,
      "logps/chosen": -424.09918212890625,
      "logps/rejected": -388.13372802734375,
      "loss": 0.5523,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.8043220639228821,
      "rewards/margins": 0.5199757814407349,
      "rewards/rejected": -1.3242979049682617,
      "step": 3630
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.411653435283158e-08,
      "logits/chosen": -0.045365557074546814,
      "logits/rejected": -0.025770818814635277,
      "logps/chosen": -379.8509826660156,
      "logps/rejected": -381.9056091308594,
      "loss": 0.6058,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8896015286445618,
      "rewards/margins": 0.4383910596370697,
      "rewards/rejected": -1.3279926776885986,
      "step": 3640
    },
    {
      "epoch": 0.96,
      "learning_rate": 3.04583517959367e-08,
      "logits/chosen": -0.045900799334049225,
      "logits/rejected": -0.05127967521548271,
      "logps/chosen": -372.5838623046875,
      "logps/rejected": -371.11956787109375,
      "loss": 0.5314,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.7395545840263367,
      "rewards/margins": 0.5564114451408386,
      "rewards/rejected": -1.2959661483764648,
      "step": 3650
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.7006375255985984e-08,
      "logits/chosen": -0.05672993138432503,
      "logits/rejected": -0.0464431568980217,
      "logps/chosen": -315.34393310546875,
      "logps/rejected": -327.28131103515625,
      "loss": 0.5784,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.7688011527061462,
      "rewards/margins": 0.4246784746646881,
      "rewards/rejected": -1.1934795379638672,
      "step": 3660
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.3760892972027328e-08,
      "logits/chosen": -0.030247915536165237,
      "logits/rejected": 0.009180205874145031,
      "logps/chosen": -366.7953186035156,
      "logps/rejected": -355.39910888671875,
      "loss": 0.5729,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.7872218489646912,
      "rewards/margins": 0.47359123826026917,
      "rewards/rejected": -1.2608129978179932,
      "step": 3670
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.072217594089765e-08,
      "logits/chosen": 0.01734892651438713,
      "logits/rejected": -0.01647915318608284,
      "logps/chosen": -321.6292419433594,
      "logps/rejected": -336.16650390625,
      "loss": 0.6141,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.7641597390174866,
      "rewards/margins": 0.34002798795700073,
      "rewards/rejected": -1.1041877269744873,
      "step": 3680
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.789047789459375e-08,
      "logits/chosen": -0.058907002210617065,
      "logits/rejected": -0.03177405148744583,
      "logps/chosen": -361.2591857910156,
      "logps/rejected": -376.9844665527344,
      "loss": 0.5805,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.8447058796882629,
      "rewards/margins": 0.43825072050094604,
      "rewards/rejected": -1.282956600189209,
      "step": 3690
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.5266035279088708e-08,
      "logits/chosen": -0.05076723173260689,
      "logits/rejected": -0.03689832612872124,
      "logps/chosen": -401.41070556640625,
      "logps/rejected": -366.7544860839844,
      "loss": 0.5769,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.8378704786300659,
      "rewards/margins": 0.45128026604652405,
      "rewards/rejected": -1.2891508340835571,
      "step": 3700
    },
    {
      "epoch": 0.97,
      "eval_logits/chosen": -0.0561523512005806,
      "eval_logits/rejected": -0.022463036701083183,
      "eval_logps/chosen": -360.40625,
      "eval_logps/rejected": -371.9745178222656,
      "eval_loss": 0.5780424475669861,
      "eval_rewards/accuracies": 0.6315000057220459,
      "eval_rewards/chosen": -0.8285941481590271,
      "eval_rewards/margins": 0.45299187302589417,
      "eval_rewards/rejected": -1.2815860509872437,
      "eval_runtime": 544.8127,
      "eval_samples_per_second": 3.671,
      "eval_steps_per_second": 0.918,
      "step": 3700
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.2849067234584623e-08,
      "logits/chosen": -0.06095626950263977,
      "logits/rejected": -0.026778647676110268,
      "logps/chosen": -335.8456115722656,
      "logps/rejected": -357.4090881347656,
      "loss": 0.5608,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.7960391640663147,
      "rewards/margins": 0.452970415353775,
      "rewards/rejected": -1.2490094900131226,
      "step": 3710
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.0639775577218625e-08,
      "logits/chosen": -0.028050154447555542,
      "logits/rejected": -0.012364340014755726,
      "logps/chosen": -363.0377197265625,
      "logps/rejected": -385.4869384765625,
      "loss": 0.5457,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.7599583268165588,
      "rewards/margins": 0.5159587860107422,
      "rewards/rejected": -1.2759170532226562,
      "step": 3720
    },
    {
      "epoch": 0.98,
      "learning_rate": 8.638344782207486e-09,
      "logits/chosen": -0.06382229179143906,
      "logits/rejected": -0.024947451427578926,
      "logps/chosen": -392.31329345703125,
      "logps/rejected": -396.88507080078125,
      "loss": 0.6442,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.9264042973518372,
      "rewards/margins": 0.30078691244125366,
      "rewards/rejected": -1.2271912097930908,
      "step": 3730
    },
    {
      "epoch": 0.98,
      "learning_rate": 6.84494196844715e-09,
      "logits/chosen": -0.022581690922379494,
      "logits/rejected": 0.011578726582229137,
      "logps/chosen": -341.0782775878906,
      "logps/rejected": -361.1309509277344,
      "loss": 0.5693,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.7737487554550171,
      "rewards/margins": 0.4638180136680603,
      "rewards/rejected": -1.2375667095184326,
      "step": 3740
    },
    {
      "epoch": 0.98,
      "learning_rate": 5.259716884556121e-09,
      "logits/chosen": -0.06770457327365875,
      "logits/rejected": -0.046060215681791306,
      "logps/chosen": -390.3734130859375,
      "logps/rejected": -387.06060791015625,
      "loss": 0.538,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.8117982745170593,
      "rewards/margins": 0.5823758840560913,
      "rewards/rejected": -1.3941742181777954,
      "step": 3750
    },
    {
      "epoch": 0.98,
      "learning_rate": 3.882801896372967e-09,
      "logits/chosen": -0.09617499262094498,
      "logits/rejected": -0.03756389021873474,
      "logps/chosen": -361.5705871582031,
      "logps/rejected": -379.3876037597656,
      "loss": 0.5924,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.812038242816925,
      "rewards/margins": 0.4387938976287842,
      "rewards/rejected": -1.2508320808410645,
      "step": 3760
    },
    {
      "epoch": 0.99,
      "learning_rate": 2.7143119759026614e-09,
      "logits/chosen": -0.05360639840364456,
      "logits/rejected": 0.002357491757720709,
      "logps/chosen": -346.79510498046875,
      "logps/rejected": -385.0536804199219,
      "loss": 0.5966,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.7745226621627808,
      "rewards/margins": 0.40841707587242126,
      "rewards/rejected": -1.1829397678375244,
      "step": 3770
    },
    {
      "epoch": 0.99,
      "learning_rate": 1.754344691717591e-09,
      "logits/chosen": -0.05662069469690323,
      "logits/rejected": -0.04745618626475334,
      "logps/chosen": -336.276611328125,
      "logps/rejected": -331.7563171386719,
      "loss": 0.5534,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.7269638776779175,
      "rewards/margins": 0.4797068238258362,
      "rewards/rejected": -1.206670880317688,
      "step": 3780
    },
    {
      "epoch": 0.99,
      "learning_rate": 1.0029802008096335e-09,
      "logits/chosen": 0.008154665119946003,
      "logits/rejected": -0.02306550182402134,
      "logps/chosen": -347.02252197265625,
      "logps/rejected": -350.6134948730469,
      "loss": 0.5594,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.8176683187484741,
      "rewards/margins": 0.4948461055755615,
      "rewards/rejected": -1.3125144243240356,
      "step": 3790
    },
    {
      "epoch": 0.99,
      "learning_rate": 4.602812418974534e-10,
      "logits/chosen": -0.06735100597143173,
      "logits/rejected": -0.036688387393951416,
      "logps/chosen": -344.30023193359375,
      "logps/rejected": -356.15130615234375,
      "loss": 0.635,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.9314397573471069,
      "rewards/margins": 0.3096088171005249,
      "rewards/rejected": -1.2410485744476318,
      "step": 3800
    },
    {
      "epoch": 0.99,
      "eval_logits/chosen": -0.05727638304233551,
      "eval_logits/rejected": -0.02366885170340538,
      "eval_logps/chosen": -360.228759765625,
      "eval_logps/rejected": -371.7966613769531,
      "eval_loss": 0.5779924392700195,
      "eval_rewards/accuracies": 0.6299999952316284,
      "eval_rewards/chosen": -0.8268191814422607,
      "eval_rewards/margins": 0.4529884457588196,
      "eval_rewards/rejected": -1.2798075675964355,
      "eval_runtime": 544.7616,
      "eval_samples_per_second": 3.671,
      "eval_steps_per_second": 0.918,
      "step": 3800
    },
    {
      "epoch": 1.0,
      "learning_rate": 1.2629313018819312e-10,
      "logits/chosen": -0.05566060543060303,
      "logits/rejected": -0.01925462670624256,
      "logps/chosen": -380.08441162109375,
      "logps/rejected": -407.0417175292969,
      "loss": 0.5533,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.7933879494667053,
      "rewards/margins": 0.494006872177124,
      "rewards/rejected": -1.2873947620391846,
      "step": 3810
    },
    {
      "epoch": 1.0,
      "learning_rate": 1.0437535929996855e-12,
      "logits/chosen": -0.04778105020523071,
      "logits/rejected": -0.05873732641339302,
      "logps/chosen": -362.91461181640625,
      "logps/rejected": -384.0444641113281,
      "loss": 0.5796,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.8017057180404663,
      "rewards/margins": 0.4591229557991028,
      "rewards/rejected": -1.2608287334442139,
      "step": 3820
    },
    {
      "epoch": 1.0,
      "step": 3821,
      "total_flos": 0.0,
      "train_loss": 0.5976003442126336,
      "train_runtime": 56244.0781,
      "train_samples_per_second": 1.087,
      "train_steps_per_second": 0.068
    }
  ],
  "logging_steps": 10,
  "max_steps": 3821,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}