{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.997676219984508,
  "eval_steps": 100,
  "global_step": 1935,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 2.5773195876288657e-09,
      "logits/chosen": -1.2259318828582764,
      "logits/rejected": -1.2518267631530762,
      "logps/chosen": -300.0604248046875,
      "logps/rejected": -269.6971130371094,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.02,
      "learning_rate": 2.5773195876288656e-08,
      "logits/chosen": -1.137500286102295,
      "logits/rejected": -1.2303526401519775,
      "logps/chosen": -301.8792724609375,
      "logps/rejected": -238.36328125,
      "loss": 0.6942,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0015259875217452645,
      "rewards/margins": -0.003614760935306549,
      "rewards/rejected": 0.005140749271959066,
      "step": 10
    },
    {
      "epoch": 0.03,
      "learning_rate": 5.154639175257731e-08,
      "logits/chosen": -1.1808393001556396,
      "logits/rejected": -1.2743797302246094,
      "logps/chosen": -314.679931640625,
      "logps/rejected": -244.7173614501953,
      "loss": 0.6962,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": 0.004601674620062113,
      "rewards/margins": -0.0029529244638979435,
      "rewards/rejected": 0.007554598152637482,
      "step": 20
    },
    {
      "epoch": 0.05,
      "learning_rate": 7.731958762886598e-08,
      "logits/chosen": -1.069253921508789,
      "logits/rejected": -1.2187018394470215,
      "logps/chosen": -323.1502380371094,
      "logps/rejected": -268.97393798828125,
      "loss": 0.6933,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.005449424032121897,
      "rewards/margins": -0.0008543418953195214,
      "rewards/rejected": -0.004595082253217697,
      "step": 30
    },
    {
      "epoch": 0.06,
      "learning_rate": 1.0309278350515462e-07,
      "logits/chosen": -1.1713712215423584,
      "logits/rejected": -1.2351148128509521,
      "logps/chosen": -255.2301025390625,
      "logps/rejected": -231.7521514892578,
      "loss": 0.6938,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.004361768253147602,
      "rewards/margins": 0.002766171470284462,
      "rewards/rejected": -0.007127939257770777,
      "step": 40
    },
    {
      "epoch": 0.08,
      "learning_rate": 1.2886597938144328e-07,
      "logits/chosen": -1.14518404006958,
      "logits/rejected": -1.2944872379302979,
      "logps/chosen": -297.48272705078125,
      "logps/rejected": -256.091796875,
      "loss": 0.6953,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.005987298674881458,
      "rewards/margins": 0.006507495883852243,
      "rewards/rejected": -0.0005201972089707851,
      "step": 50
    },
    {
      "epoch": 0.09,
      "learning_rate": 1.5463917525773197e-07,
      "logits/chosen": -1.1941990852355957,
      "logits/rejected": -1.266606092453003,
      "logps/chosen": -308.13873291015625,
      "logps/rejected": -251.240966796875,
      "loss": 0.6919,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.0025153695605695248,
      "rewards/margins": 0.005558790173381567,
      "rewards/rejected": -0.008074158802628517,
      "step": 60
    },
    {
      "epoch": 0.11,
      "learning_rate": 1.804123711340206e-07,
      "logits/chosen": -1.214543104171753,
      "logits/rejected": -1.2368158102035522,
      "logps/chosen": -308.1925048828125,
      "logps/rejected": -258.2166748046875,
      "loss": 0.6931,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.001422025030478835,
      "rewards/margins": 0.00037188082933425903,
      "rewards/rejected": -0.0017939055105671287,
      "step": 70
    },
    {
      "epoch": 0.12,
      "learning_rate": 2.0618556701030925e-07,
      "logits/chosen": -1.1733652353286743,
      "logits/rejected": -1.2068002223968506,
      "logps/chosen": -339.65484619140625,
      "logps/rejected": -251.48757934570312,
      "loss": 0.6954,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.000313587486743927,
      "rewards/margins": -0.0003553667920641601,
      "rewards/rejected": 0.0006689537549391389,
      "step": 80
    },
    {
      "epoch": 0.14,
      "learning_rate": 2.3195876288659794e-07,
      "logits/chosen": -1.1699542999267578,
      "logits/rejected": -1.2276877164840698,
      "logps/chosen": -309.55816650390625,
      "logps/rejected": -253.14212036132812,
      "loss": 0.6937,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.008602892979979515,
      "rewards/margins": -0.002404768718406558,
      "rewards/rejected": -0.006198124028742313,
      "step": 90
    },
    {
      "epoch": 0.15,
      "learning_rate": 2.5773195876288655e-07,
      "logits/chosen": -1.2193819284439087,
      "logits/rejected": -1.2733923196792603,
      "logps/chosen": -284.50494384765625,
      "logps/rejected": -257.36614990234375,
      "loss": 0.6944,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.008577315136790276,
      "rewards/margins": -0.007729054894298315,
      "rewards/rejected": -0.0008482601260766387,
      "step": 100
    },
    {
      "epoch": 0.17,
      "learning_rate": 2.835051546391752e-07,
      "logits/chosen": -1.1101040840148926,
      "logits/rejected": -1.2103769779205322,
      "logps/chosen": -272.78472900390625,
      "logps/rejected": -236.69888305664062,
      "loss": 0.6924,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.008216220885515213,
      "rewards/margins": -0.006476385984569788,
      "rewards/rejected": -0.001739834202453494,
      "step": 110
    },
    {
      "epoch": 0.19,
      "learning_rate": 3.0927835051546394e-07,
      "logits/chosen": -1.1474252939224243,
      "logits/rejected": -1.2477691173553467,
      "logps/chosen": -313.0333557128906,
      "logps/rejected": -276.30560302734375,
      "loss": 0.6926,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.0040068114176392555,
      "rewards/margins": -0.0006697230273857713,
      "rewards/rejected": -0.0033370896708220243,
      "step": 120
    },
    {
      "epoch": 0.2,
      "learning_rate": 3.3505154639175255e-07,
      "logits/chosen": -1.1452791690826416,
      "logits/rejected": -1.2140785455703735,
      "logps/chosen": -306.3499755859375,
      "logps/rejected": -241.6109161376953,
      "loss": 0.6923,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.002921496983617544,
      "rewards/margins": -0.0032920476514846087,
      "rewards/rejected": 0.0003705514536704868,
      "step": 130
    },
    {
      "epoch": 0.22,
      "learning_rate": 3.608247422680412e-07,
      "logits/chosen": -1.2596957683563232,
      "logits/rejected": -1.3245229721069336,
      "logps/chosen": -281.8450012207031,
      "logps/rejected": -239.2882537841797,
      "loss": 0.6932,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -0.006516546942293644,
      "rewards/margins": 0.0012826811289414763,
      "rewards/rejected": -0.007799227721989155,
      "step": 140
    },
    {
      "epoch": 0.23,
      "learning_rate": 3.865979381443299e-07,
      "logits/chosen": -1.1537089347839355,
      "logits/rejected": -1.1991795301437378,
      "logps/chosen": -282.4929504394531,
      "logps/rejected": -248.6052703857422,
      "loss": 0.6922,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.002510285470634699,
      "rewards/margins": 0.007546475622802973,
      "rewards/rejected": -0.005036188755184412,
      "step": 150
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.123711340206185e-07,
      "logits/chosen": -1.0950748920440674,
      "logits/rejected": -1.2696372270584106,
      "logps/chosen": -284.0737609863281,
      "logps/rejected": -226.47195434570312,
      "loss": 0.6936,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.0014800997450947762,
      "rewards/margins": 0.0013229111209511757,
      "rewards/rejected": -0.002803010866045952,
      "step": 160
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.381443298969072e-07,
      "logits/chosen": -1.231684923171997,
      "logits/rejected": -1.1929103136062622,
      "logps/chosen": -278.35870361328125,
      "logps/rejected": -222.3286590576172,
      "loss": 0.6909,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.004064156208187342,
      "rewards/margins": -0.0029725716449320316,
      "rewards/rejected": -0.0010915847960859537,
      "step": 170
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.639175257731959e-07,
      "logits/chosen": -1.1976374387741089,
      "logits/rejected": -1.228857398033142,
      "logps/chosen": -254.65087890625,
      "logps/rejected": -227.2077178955078,
      "loss": 0.6928,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.004570499062538147,
      "rewards/margins": -0.005624748300760984,
      "rewards/rejected": 0.0010542500531300902,
      "step": 180
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.896907216494845e-07,
      "logits/chosen": -1.1410865783691406,
      "logits/rejected": -1.2138147354125977,
      "logps/chosen": -312.42230224609375,
      "logps/rejected": -247.4111785888672,
      "loss": 0.6933,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": 0.0012876918772235513,
      "rewards/margins": 0.005938920192420483,
      "rewards/rejected": -0.0046512288972735405,
      "step": 190
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.982768523836875e-07,
      "logits/chosen": -1.1175594329833984,
      "logits/rejected": -1.2174949645996094,
      "logps/chosen": -300.48736572265625,
      "logps/rejected": -253.6021728515625,
      "loss": 0.6933,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.010855994187295437,
      "rewards/margins": -0.0076787760481238365,
      "rewards/rejected": -0.0031772174406796694,
      "step": 200
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.954049396898335e-07,
      "logits/chosen": -1.1029257774353027,
      "logits/rejected": -1.266903042793274,
      "logps/chosen": -292.01513671875,
      "logps/rejected": -248.255615234375,
      "loss": 0.6933,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.003372871782630682,
      "rewards/margins": 0.01588493585586548,
      "rewards/rejected": -0.012512065470218658,
      "step": 210
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.925330269959793e-07,
      "logits/chosen": -1.1873070001602173,
      "logits/rejected": -1.2105329036712646,
      "logps/chosen": -294.33184814453125,
      "logps/rejected": -248.6623077392578,
      "loss": 0.6925,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.0029001368675380945,
      "rewards/margins": 0.0060717323794960976,
      "rewards/rejected": -0.008971868082880974,
      "step": 220
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.896611143021252e-07,
      "logits/chosen": -1.126260757446289,
      "logits/rejected": -1.1767059564590454,
      "logps/chosen": -283.59649658203125,
      "logps/rejected": -265.82476806640625,
      "loss": 0.6927,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": 0.0003154706209897995,
      "rewards/margins": 0.011949801817536354,
      "rewards/rejected": -0.01163433212786913,
      "step": 230
    },
    {
      "epoch": 0.37,
      "learning_rate": 4.867892016082711e-07,
      "logits/chosen": -1.0998083353042603,
      "logits/rejected": -1.197954773902893,
      "logps/chosen": -297.81396484375,
      "logps/rejected": -251.5466766357422,
      "loss": 0.6936,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.001978257205337286,
      "rewards/margins": 0.006527910940349102,
      "rewards/rejected": -0.008506168611347675,
      "step": 240
    },
    {
      "epoch": 0.39,
      "learning_rate": 4.839172889144169e-07,
      "logits/chosen": -1.1897470951080322,
      "logits/rejected": -1.2483431100845337,
      "logps/chosen": -278.9944763183594,
      "logps/rejected": -237.2654571533203,
      "loss": 0.693,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.013788111507892609,
      "rewards/margins": -0.006659683771431446,
      "rewards/rejected": -0.007128427270799875,
      "step": 250
    },
    {
      "epoch": 0.4,
      "learning_rate": 4.810453762205628e-07,
      "logits/chosen": -1.1891684532165527,
      "logits/rejected": -1.2836068868637085,
      "logps/chosen": -286.28570556640625,
      "logps/rejected": -255.8869171142578,
      "loss": 0.6915,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.0036178447771817446,
      "rewards/margins": 0.003644808428362012,
      "rewards/rejected": -0.0072626532055437565,
      "step": 260
    },
    {
      "epoch": 0.42,
      "learning_rate": 4.781734635267088e-07,
      "logits/chosen": -1.1823965311050415,
      "logits/rejected": -1.2820310592651367,
      "logps/chosen": -270.109375,
      "logps/rejected": -234.67568969726562,
      "loss": 0.6928,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.007160546723753214,
      "rewards/margins": -0.003257888136431575,
      "rewards/rejected": -0.0039026569575071335,
      "step": 270
    },
    {
      "epoch": 0.43,
      "learning_rate": 4.7530155083285466e-07,
      "logits/chosen": -1.1655725240707397,
      "logits/rejected": -1.2631051540374756,
      "logps/chosen": -290.2110290527344,
      "logps/rejected": -247.9181671142578,
      "loss": 0.6945,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.005572388414293528,
      "rewards/margins": 0.005412190221250057,
      "rewards/rejected": -0.010984579101204872,
      "step": 280
    },
    {
      "epoch": 0.45,
      "learning_rate": 4.7242963813900055e-07,
      "logits/chosen": -1.1850521564483643,
      "logits/rejected": -1.2781732082366943,
      "logps/chosen": -305.34063720703125,
      "logps/rejected": -231.7410430908203,
      "loss": 0.6908,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.008810663595795631,
      "rewards/margins": 0.002493415493518114,
      "rewards/rejected": -0.011304078623652458,
      "step": 290
    },
    {
      "epoch": 0.46,
      "learning_rate": 4.6955772544514645e-07,
      "logits/chosen": -1.2146244049072266,
      "logits/rejected": -1.3335716724395752,
      "logps/chosen": -260.8929748535156,
      "logps/rejected": -225.0554656982422,
      "loss": 0.6915,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.0020951449405401945,
      "rewards/margins": 0.0035616860259324312,
      "rewards/rejected": -0.005656830966472626,
      "step": 300
    },
    {
      "epoch": 0.48,
      "learning_rate": 4.6668581275129234e-07,
      "logits/chosen": -1.174975872039795,
      "logits/rejected": -1.2588573694229126,
      "logps/chosen": -304.65008544921875,
      "logps/rejected": -261.6839599609375,
      "loss": 0.6929,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.012646270915865898,
      "rewards/margins": -0.005786383058875799,
      "rewards/rejected": -0.006859886460006237,
      "step": 310
    },
    {
      "epoch": 0.5,
      "learning_rate": 4.6381390005743824e-07,
      "logits/chosen": -1.1501063108444214,
      "logits/rejected": -1.244741678237915,
      "logps/chosen": -287.64208984375,
      "logps/rejected": -252.3353729248047,
      "loss": 0.693,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.006176070310175419,
      "rewards/margins": 0.004587090574204922,
      "rewards/rejected": -0.01076316274702549,
      "step": 320
    },
    {
      "epoch": 0.51,
      "learning_rate": 4.6094198736358413e-07,
      "logits/chosen": -1.2028628587722778,
      "logits/rejected": -1.2731959819793701,
      "logps/chosen": -297.4624938964844,
      "logps/rejected": -271.55548095703125,
      "loss": 0.6926,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.006290009710937738,
      "rewards/margins": 0.0007669615442864597,
      "rewards/rejected": -0.007056971546262503,
      "step": 330
    },
    {
      "epoch": 0.53,
      "learning_rate": 4.5807007466973003e-07,
      "logits/chosen": -1.1701081991195679,
      "logits/rejected": -1.245665431022644,
      "logps/chosen": -285.584716796875,
      "logps/rejected": -249.4613494873047,
      "loss": 0.6893,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.011785658076405525,
      "rewards/margins": 0.0033443807624280453,
      "rewards/rejected": -0.015130040235817432,
      "step": 340
    },
    {
      "epoch": 0.54,
      "learning_rate": 4.551981619758759e-07,
      "logits/chosen": -1.0303632020950317,
      "logits/rejected": -1.1990408897399902,
      "logps/chosen": -314.17156982421875,
      "logps/rejected": -267.7991638183594,
      "loss": 0.6916,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0035480200313031673,
      "rewards/margins": 0.006092972122132778,
      "rewards/rejected": -0.009640992619097233,
      "step": 350
    },
    {
      "epoch": 0.56,
      "learning_rate": 4.5232624928202176e-07,
      "logits/chosen": -1.1599308252334595,
      "logits/rejected": -1.2161757946014404,
      "logps/chosen": -302.0650634765625,
      "logps/rejected": -254.35116577148438,
      "loss": 0.6888,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.006021351087838411,
      "rewards/margins": 0.019538475200533867,
      "rewards/rejected": -0.02555982768535614,
      "step": 360
    },
    {
      "epoch": 0.57,
      "learning_rate": 4.494543365881677e-07,
      "logits/chosen": -1.1341716051101685,
      "logits/rejected": -1.2106345891952515,
      "logps/chosen": -268.8562927246094,
      "logps/rejected": -254.6302490234375,
      "loss": 0.6908,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.01141696609556675,
      "rewards/margins": 0.0022176806814968586,
      "rewards/rejected": -0.013634647242724895,
      "step": 370
    },
    {
      "epoch": 0.59,
      "learning_rate": 4.465824238943136e-07,
      "logits/chosen": -1.1235488653182983,
      "logits/rejected": -1.2134369611740112,
      "logps/chosen": -294.3124084472656,
      "logps/rejected": -249.931884765625,
      "loss": 0.6896,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.0007959249196574092,
      "rewards/margins": 0.014344396069645882,
      "rewards/rejected": -0.013548469170928001,
      "step": 380
    },
    {
      "epoch": 0.6,
      "learning_rate": 4.437105112004595e-07,
      "logits/chosen": -1.174169659614563,
      "logits/rejected": -1.2081773281097412,
      "logps/chosen": -268.9616394042969,
      "logps/rejected": -272.663818359375,
      "loss": 0.6914,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -8.791415166342631e-05,
      "rewards/margins": 0.008916514925658703,
      "rewards/rejected": -0.009004429914057255,
      "step": 390
    },
    {
      "epoch": 0.62,
      "learning_rate": 4.408385985066054e-07,
      "logits/chosen": -1.2326834201812744,
      "logits/rejected": -1.2733819484710693,
      "logps/chosen": -279.9195251464844,
      "logps/rejected": -262.4093933105469,
      "loss": 0.6896,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.00013022082566749305,
      "rewards/margins": 0.019076677039265633,
      "rewards/rejected": -0.019206896424293518,
      "step": 400
    },
    {
      "epoch": 0.64,
      "learning_rate": 4.3796668581275124e-07,
      "logits/chosen": -1.0889310836791992,
      "logits/rejected": -1.1624057292938232,
      "logps/chosen": -342.1612243652344,
      "logps/rejected": -292.48516845703125,
      "loss": 0.6883,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.00807907897979021,
      "rewards/margins": 0.017683448269963264,
      "rewards/rejected": -0.02576252445578575,
      "step": 410
    },
    {
      "epoch": 0.65,
      "learning_rate": 4.3509477311889713e-07,
      "logits/chosen": -1.1696580648422241,
      "logits/rejected": -1.2402842044830322,
      "logps/chosen": -318.536376953125,
      "logps/rejected": -249.26010131835938,
      "loss": 0.6889,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.011863854713737965,
      "rewards/margins": 0.004245345946401358,
      "rewards/rejected": -0.016109202057123184,
      "step": 420
    },
    {
      "epoch": 0.67,
      "learning_rate": 4.322228604250431e-07,
      "logits/chosen": -1.1121430397033691,
      "logits/rejected": -1.1772434711456299,
      "logps/chosen": -316.11346435546875,
      "logps/rejected": -265.7353210449219,
      "loss": 0.688,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.00796548929065466,
      "rewards/margins": 0.01683664321899414,
      "rewards/rejected": -0.024802135303616524,
      "step": 430
    },
    {
      "epoch": 0.68,
      "learning_rate": 4.2935094773118897e-07,
      "logits/chosen": -1.0895384550094604,
      "logits/rejected": -1.235265851020813,
      "logps/chosen": -299.3586120605469,
      "logps/rejected": -271.97650146484375,
      "loss": 0.6891,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.013709778897464275,
      "rewards/margins": 0.007505470421165228,
      "rewards/rejected": -0.021215252578258514,
      "step": 440
    },
    {
      "epoch": 0.7,
      "learning_rate": 4.2647903503733487e-07,
      "logits/chosen": -1.1224366426467896,
      "logits/rejected": -1.21951425075531,
      "logps/chosen": -290.92523193359375,
      "logps/rejected": -250.52157592773438,
      "loss": 0.6883,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.009392457082867622,
      "rewards/margins": 0.012002045288681984,
      "rewards/rejected": -0.021394502371549606,
      "step": 450
    },
    {
      "epoch": 0.71,
      "learning_rate": 4.236071223434807e-07,
      "logits/chosen": -1.1139185428619385,
      "logits/rejected": -1.230420708656311,
      "logps/chosen": -293.9082336425781,
      "logps/rejected": -263.75726318359375,
      "loss": 0.6893,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.015224998816847801,
      "rewards/margins": 0.006627705879509449,
      "rewards/rejected": -0.021852705627679825,
      "step": 460
    },
    {
      "epoch": 0.73,
      "learning_rate": 4.207352096496266e-07,
      "logits/chosen": -1.2127584218978882,
      "logits/rejected": -1.2580344676971436,
      "logps/chosen": -308.8389892578125,
      "logps/rejected": -267.56787109375,
      "loss": 0.6878,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.010395748540759087,
      "rewards/margins": 0.010451807640492916,
      "rewards/rejected": -0.020847555249929428,
      "step": 470
    },
    {
      "epoch": 0.74,
      "learning_rate": 4.1786329695577255e-07,
      "logits/chosen": -1.151767373085022,
      "logits/rejected": -1.2307831048965454,
      "logps/chosen": -312.295166015625,
      "logps/rejected": -272.08050537109375,
      "loss": 0.6898,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.015018184669315815,
      "rewards/margins": 0.0127108721062541,
      "rewards/rejected": -0.027729053050279617,
      "step": 480
    },
    {
      "epoch": 0.76,
      "learning_rate": 4.1499138426191845e-07,
      "logits/chosen": -1.128575086593628,
      "logits/rejected": -1.1787781715393066,
      "logps/chosen": -291.2005310058594,
      "logps/rejected": -258.6954040527344,
      "loss": 0.688,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.006343941204249859,
      "rewards/margins": 0.01303582638502121,
      "rewards/rejected": -0.019379768520593643,
      "step": 490
    },
    {
      "epoch": 0.77,
      "learning_rate": 4.121194715680643e-07,
      "logits/chosen": -1.1914628744125366,
      "logits/rejected": -1.2668449878692627,
      "logps/chosen": -261.34039306640625,
      "logps/rejected": -213.05917358398438,
      "loss": 0.6873,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.013653025031089783,
      "rewards/margins": 0.012117515318095684,
      "rewards/rejected": -0.02577054128050804,
      "step": 500
    },
    {
      "epoch": 0.79,
      "learning_rate": 4.092475588742102e-07,
      "logits/chosen": -1.1350027322769165,
      "logits/rejected": -1.2238458395004272,
      "logps/chosen": -315.3155517578125,
      "logps/rejected": -300.2586975097656,
      "loss": 0.6863,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.009474663995206356,
      "rewards/margins": 0.014838702976703644,
      "rewards/rejected": -0.024313367903232574,
      "step": 510
    },
    {
      "epoch": 0.81,
      "learning_rate": 4.063756461803561e-07,
      "logits/chosen": -1.137836217880249,
      "logits/rejected": -1.269636631011963,
      "logps/chosen": -295.51776123046875,
      "logps/rejected": -245.64749145507812,
      "loss": 0.6872,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.01145818829536438,
      "rewards/margins": 0.014280671253800392,
      "rewards/rejected": -0.025738855823874474,
      "step": 520
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.03503733486502e-07,
      "logits/chosen": -1.0953731536865234,
      "logits/rejected": -1.1840622425079346,
      "logps/chosen": -302.6564025878906,
      "logps/rejected": -265.1612854003906,
      "loss": 0.6842,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.005975248292088509,
      "rewards/margins": 0.02241487428545952,
      "rewards/rejected": -0.028390124440193176,
      "step": 530
    },
    {
      "epoch": 0.84,
      "learning_rate": 4.006318207926479e-07,
      "logits/chosen": -1.1384605169296265,
      "logits/rejected": -1.2369248867034912,
      "logps/chosen": -302.2364196777344,
      "logps/rejected": -282.00164794921875,
      "loss": 0.6864,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.021240079775452614,
      "rewards/margins": 0.015024731867015362,
      "rewards/rejected": -0.0362648107111454,
      "step": 540
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.9775990809879376e-07,
      "logits/chosen": -1.1440156698226929,
      "logits/rejected": -1.2083277702331543,
      "logps/chosen": -297.77606201171875,
      "logps/rejected": -257.05352783203125,
      "loss": 0.6852,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.00584394671022892,
      "rewards/margins": 0.024538297206163406,
      "rewards/rejected": -0.030382240191102028,
      "step": 550
    },
    {
      "epoch": 0.87,
      "learning_rate": 3.9488799540493966e-07,
      "logits/chosen": -1.1610389947891235,
      "logits/rejected": -1.2373404502868652,
      "logps/chosen": -299.90802001953125,
      "logps/rejected": -255.9143524169922,
      "loss": 0.6855,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.008234420791268349,
      "rewards/margins": 0.02389439567923546,
      "rewards/rejected": -0.03212881460785866,
      "step": 560
    },
    {
      "epoch": 0.88,
      "learning_rate": 3.9201608271108555e-07,
      "logits/chosen": -1.1905468702316284,
      "logits/rejected": -1.2244651317596436,
      "logps/chosen": -341.0932922363281,
      "logps/rejected": -253.48916625976562,
      "loss": 0.6872,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.007996906526386738,
      "rewards/margins": 0.019794629886746407,
      "rewards/rejected": -0.02779153548181057,
      "step": 570
    },
    {
      "epoch": 0.9,
      "learning_rate": 3.8914417001723144e-07,
      "logits/chosen": -1.1463348865509033,
      "logits/rejected": -1.2082992792129517,
      "logps/chosen": -278.6533203125,
      "logps/rejected": -247.3466796875,
      "loss": 0.6869,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.013167768716812134,
      "rewards/margins": 0.013775348663330078,
      "rewards/rejected": -0.026943117380142212,
      "step": 580
    },
    {
      "epoch": 0.91,
      "learning_rate": 3.862722573233774e-07,
      "logits/chosen": -1.1172596216201782,
      "logits/rejected": -1.2871202230453491,
      "logps/chosen": -268.90435791015625,
      "logps/rejected": -233.5662384033203,
      "loss": 0.6835,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.017810124903917313,
      "rewards/margins": 0.01590743288397789,
      "rewards/rejected": -0.0337175577878952,
      "step": 590
    },
    {
      "epoch": 0.93,
      "learning_rate": 3.8340034462952323e-07,
      "logits/chosen": -1.1374852657318115,
      "logits/rejected": -1.2383670806884766,
      "logps/chosen": -309.21331787109375,
      "logps/rejected": -242.3319091796875,
      "loss": 0.6843,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.02574210986495018,
      "rewards/margins": 0.018136167898774147,
      "rewards/rejected": -0.043878279626369476,
      "step": 600
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.8052843193566913e-07,
      "logits/chosen": -1.1285089254379272,
      "logits/rejected": -1.1931793689727783,
      "logps/chosen": -278.48101806640625,
      "logps/rejected": -242.2903289794922,
      "loss": 0.6851,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.017196612432599068,
      "rewards/margins": 0.032739974558353424,
      "rewards/rejected": -0.04993658885359764,
      "step": 610
    },
    {
      "epoch": 0.96,
      "learning_rate": 3.77656519241815e-07,
      "logits/chosen": -1.1930526494979858,
      "logits/rejected": -1.3126742839813232,
      "logps/chosen": -286.9619140625,
      "logps/rejected": -246.289794921875,
      "loss": 0.6857,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.014147510752081871,
      "rewards/margins": 0.016736771911382675,
      "rewards/rejected": -0.030884284526109695,
      "step": 620
    },
    {
      "epoch": 0.98,
      "learning_rate": 3.747846065479609e-07,
      "logits/chosen": -1.2015870809555054,
      "logits/rejected": -1.2615153789520264,
      "logps/chosen": -286.06475830078125,
      "logps/rejected": -245.6126708984375,
      "loss": 0.6837,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.00835242960602045,
      "rewards/margins": 0.02643100544810295,
      "rewards/rejected": -0.03478343412280083,
      "step": 630
    },
    {
      "epoch": 0.99,
      "learning_rate": 3.7191269385410686e-07,
      "logits/chosen": -1.092500925064087,
      "logits/rejected": -1.2248862981796265,
      "logps/chosen": -280.76324462890625,
      "logps/rejected": -244.59375,
      "loss": 0.6812,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.004551100078970194,
      "rewards/margins": 0.026827871799468994,
      "rewards/rejected": -0.03137897327542305,
      "step": 640
    },
    {
      "epoch": 1.0,
      "eval_logits/chosen": -1.2853448390960693,
      "eval_logits/rejected": -1.455635905265808,
      "eval_logps/chosen": -271.04241943359375,
      "eval_logps/rejected": -240.60862731933594,
      "eval_loss": 0.6831629276275635,
      "eval_rewards/accuracies": 0.6190476417541504,
      "eval_rewards/chosen": -0.015324260108172894,
      "eval_rewards/margins": 0.027137557044625282,
      "eval_rewards/rejected": -0.0424618162214756,
      "eval_runtime": 336.2787,
      "eval_samples_per_second": 5.947,
      "eval_steps_per_second": 0.25,
      "step": 645
    },
    {
      "epoch": 1.01,
      "learning_rate": 3.690407811602527e-07,
      "logits/chosen": -1.1837308406829834,
      "logits/rejected": -1.2207152843475342,
      "logps/chosen": -269.86431884765625,
      "logps/rejected": -258.81109619140625,
      "loss": 0.6826,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.02447674050927162,
      "rewards/margins": 0.018924027681350708,
      "rewards/rejected": -0.04340077191591263,
      "step": 650
    },
    {
      "epoch": 1.02,
      "learning_rate": 3.661688684663986e-07,
      "logits/chosen": -1.0795660018920898,
      "logits/rejected": -1.1991580724716187,
      "logps/chosen": -280.2117919921875,
      "logps/rejected": -253.22000122070312,
      "loss": 0.6848,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.018620852380990982,
      "rewards/margins": 0.016325894743204117,
      "rewards/rejected": -0.0349467471241951,
      "step": 660
    },
    {
      "epoch": 1.04,
      "learning_rate": 3.632969557725445e-07,
      "logits/chosen": -1.1959303617477417,
      "logits/rejected": -1.268479824066162,
      "logps/chosen": -282.8076477050781,
      "logps/rejected": -270.8659973144531,
      "loss": 0.6801,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.014843074604868889,
      "rewards/margins": 0.031116703525185585,
      "rewards/rejected": -0.045959778130054474,
      "step": 670
    },
    {
      "epoch": 1.05,
      "learning_rate": 3.604250430786904e-07,
      "logits/chosen": -1.131414532661438,
      "logits/rejected": -1.2395284175872803,
      "logps/chosen": -268.8525085449219,
      "logps/rejected": -249.85302734375,
      "loss": 0.6822,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.019633300602436066,
      "rewards/margins": 0.028578415513038635,
      "rewards/rejected": -0.0482117161154747,
      "step": 680
    },
    {
      "epoch": 1.07,
      "learning_rate": 3.575531303848363e-07,
      "logits/chosen": -1.1759953498840332,
      "logits/rejected": -1.2496672868728638,
      "logps/chosen": -283.6187438964844,
      "logps/rejected": -251.77706909179688,
      "loss": 0.6822,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.02356325089931488,
      "rewards/margins": 0.026736527681350708,
      "rewards/rejected": -0.05029977485537529,
      "step": 690
    },
    {
      "epoch": 1.08,
      "learning_rate": 3.546812176909822e-07,
      "logits/chosen": -1.2295715808868408,
      "logits/rejected": -1.2652504444122314,
      "logps/chosen": -297.22796630859375,
      "logps/rejected": -288.7464294433594,
      "loss": 0.6792,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.021731413900852203,
      "rewards/margins": 0.02142687328159809,
      "rewards/rejected": -0.043158285319805145,
      "step": 700
    },
    {
      "epoch": 1.1,
      "learning_rate": 3.518093049971281e-07,
      "logits/chosen": -1.1974341869354248,
      "logits/rejected": -1.2550041675567627,
      "logps/chosen": -307.19482421875,
      "logps/rejected": -276.5005798339844,
      "loss": 0.6802,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.021447179839015007,
      "rewards/margins": 0.026276415213942528,
      "rewards/rejected": -0.04772359877824783,
      "step": 710
    },
    {
      "epoch": 1.12,
      "learning_rate": 3.4893739230327397e-07,
      "logits/chosen": -1.2148594856262207,
      "logits/rejected": -1.2141330242156982,
      "logps/chosen": -309.69671630859375,
      "logps/rejected": -253.8720245361328,
      "loss": 0.6816,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.015615013428032398,
      "rewards/margins": 0.021955113857984543,
      "rewards/rejected": -0.037570130079984665,
      "step": 720
    },
    {
      "epoch": 1.13,
      "learning_rate": 3.4606547960941986e-07,
      "logits/chosen": -1.1154258251190186,
      "logits/rejected": -1.2231298685073853,
      "logps/chosen": -280.8895263671875,
      "logps/rejected": -236.9070281982422,
      "loss": 0.6805,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.018392009660601616,
      "rewards/margins": 0.02249971777200699,
      "rewards/rejected": -0.040891729295253754,
      "step": 730
    },
    {
      "epoch": 1.15,
      "learning_rate": 3.431935669155657e-07,
      "logits/chosen": -1.1089565753936768,
      "logits/rejected": -1.2940270900726318,
      "logps/chosen": -317.8031005859375,
      "logps/rejected": -251.05783081054688,
      "loss": 0.6814,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.02725188434123993,
      "rewards/margins": 0.014852823689579964,
      "rewards/rejected": -0.042104706168174744,
      "step": 740
    },
    {
      "epoch": 1.16,
      "learning_rate": 3.4032165422171165e-07,
      "logits/chosen": -1.1635140180587769,
      "logits/rejected": -1.239103078842163,
      "logps/chosen": -273.4166564941406,
      "logps/rejected": -257.73419189453125,
      "loss": 0.6793,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.019765909761190414,
      "rewards/margins": 0.0273622814565897,
      "rewards/rejected": -0.04712819308042526,
      "step": 750
    },
    {
      "epoch": 1.18,
      "learning_rate": 3.3744974152785755e-07,
      "logits/chosen": -1.1813712120056152,
      "logits/rejected": -1.2533613443374634,
      "logps/chosen": -277.0120849609375,
      "logps/rejected": -244.0693817138672,
      "loss": 0.6782,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.027658691629767418,
      "rewards/margins": 0.027155902236700058,
      "rewards/rejected": -0.05481458827853203,
      "step": 760
    },
    {
      "epoch": 1.19,
      "learning_rate": 3.3457782883400344e-07,
      "logits/chosen": -1.121671199798584,
      "logits/rejected": -1.147890329360962,
      "logps/chosen": -311.65179443359375,
      "logps/rejected": -263.4295349121094,
      "loss": 0.68,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.02874746546149254,
      "rewards/margins": 0.03633704036474228,
      "rewards/rejected": -0.06508450210094452,
      "step": 770
    },
    {
      "epoch": 1.21,
      "learning_rate": 3.3170591614014934e-07,
      "logits/chosen": -1.1686742305755615,
      "logits/rejected": -1.274806261062622,
      "logps/chosen": -330.0002746582031,
      "logps/rejected": -265.2279357910156,
      "loss": 0.6788,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.012181037105619907,
      "rewards/margins": 0.05000365525484085,
      "rewards/rejected": -0.062184691429138184,
      "step": 780
    },
    {
      "epoch": 1.22,
      "learning_rate": 3.288340034462952e-07,
      "logits/chosen": -1.1289006471633911,
      "logits/rejected": -1.2150051593780518,
      "logps/chosen": -328.3635559082031,
      "logps/rejected": -280.61029052734375,
      "loss": 0.6799,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.018197020515799522,
      "rewards/margins": 0.041884154081344604,
      "rewards/rejected": -0.06008117273449898,
      "step": 790
    },
    {
      "epoch": 1.24,
      "learning_rate": 3.259620907524411e-07,
      "logits/chosen": -1.0758039951324463,
      "logits/rejected": -1.1864715814590454,
      "logps/chosen": -317.11444091796875,
      "logps/rejected": -275.755615234375,
      "loss": 0.6732,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.023402996361255646,
      "rewards/margins": 0.04022226482629776,
      "rewards/rejected": -0.0636252611875534,
      "step": 800
    },
    {
      "epoch": 1.25,
      "learning_rate": 3.23090178058587e-07,
      "logits/chosen": -1.160382628440857,
      "logits/rejected": -1.2629868984222412,
      "logps/chosen": -266.081298828125,
      "logps/rejected": -271.7577209472656,
      "loss": 0.6808,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.032355837523937225,
      "rewards/margins": 0.018326448276638985,
      "rewards/rejected": -0.05068228393793106,
      "step": 810
    },
    {
      "epoch": 1.27,
      "learning_rate": 3.202182653647329e-07,
      "logits/chosen": -1.0827566385269165,
      "logits/rejected": -1.2318055629730225,
      "logps/chosen": -285.2849426269531,
      "logps/rejected": -233.0825958251953,
      "loss": 0.6791,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.04408087208867073,
      "rewards/margins": 0.018497029319405556,
      "rewards/rejected": -0.06257789582014084,
      "step": 820
    },
    {
      "epoch": 1.29,
      "learning_rate": 3.1734635267087876e-07,
      "logits/chosen": -1.1579523086547852,
      "logits/rejected": -1.2240597009658813,
      "logps/chosen": -298.1856384277344,
      "logps/rejected": -275.46832275390625,
      "loss": 0.6758,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.02176501601934433,
      "rewards/margins": 0.053393684327602386,
      "rewards/rejected": -0.07515870034694672,
      "step": 830
    },
    {
      "epoch": 1.3,
      "learning_rate": 3.1447443997702465e-07,
      "logits/chosen": -1.1462314128875732,
      "logits/rejected": -1.2096561193466187,
      "logps/chosen": -304.68487548828125,
      "logps/rejected": -271.87030029296875,
      "loss": 0.6747,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.029958650469779968,
      "rewards/margins": 0.03902842849493027,
      "rewards/rejected": -0.06898707151412964,
      "step": 840
    },
    {
      "epoch": 1.32,
      "learning_rate": 3.116025272831706e-07,
      "logits/chosen": -1.1924817562103271,
      "logits/rejected": -1.1946598291397095,
      "logps/chosen": -276.5715637207031,
      "logps/rejected": -265.0469970703125,
      "loss": 0.6773,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.03456965461373329,
      "rewards/margins": 0.012200916185975075,
      "rewards/rejected": -0.04677056521177292,
      "step": 850
    },
    {
      "epoch": 1.33,
      "learning_rate": 3.087306145893165e-07,
      "logits/chosen": -1.1429754495620728,
      "logits/rejected": -1.2813076972961426,
      "logps/chosen": -265.50518798828125,
      "logps/rejected": -230.6487579345703,
      "loss": 0.6763,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.02857414446771145,
      "rewards/margins": 0.04311809688806534,
      "rewards/rejected": -0.07169224321842194,
      "step": 860
    },
    {
      "epoch": 1.35,
      "learning_rate": 3.058587018954624e-07,
      "logits/chosen": -1.2023561000823975,
      "logits/rejected": -1.266048550605774,
      "logps/chosen": -274.9041442871094,
      "logps/rejected": -257.80706787109375,
      "loss": 0.678,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.023446839302778244,
      "rewards/margins": 0.02842586673796177,
      "rewards/rejected": -0.05187270790338516,
      "step": 870
    },
    {
      "epoch": 1.36,
      "learning_rate": 3.0298678920160823e-07,
      "logits/chosen": -1.1376311779022217,
      "logits/rejected": -1.2214622497558594,
      "logps/chosen": -261.1297302246094,
      "logps/rejected": -218.02294921875,
      "loss": 0.6756,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.03140454739332199,
      "rewards/margins": 0.039807092398405075,
      "rewards/rejected": -0.07121165096759796,
      "step": 880
    },
    {
      "epoch": 1.38,
      "learning_rate": 3.001148765077541e-07,
      "logits/chosen": -1.1937434673309326,
      "logits/rejected": -1.2949789762496948,
      "logps/chosen": -286.6931457519531,
      "logps/rejected": -233.8115997314453,
      "loss": 0.6759,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.02188955806195736,
      "rewards/margins": 0.04148869216442108,
      "rewards/rejected": -0.06337825208902359,
      "step": 890
    },
    {
      "epoch": 1.39,
      "learning_rate": 2.9724296381390007e-07,
      "logits/chosen": -1.146175742149353,
      "logits/rejected": -1.2258495092391968,
      "logps/chosen": -292.33880615234375,
      "logps/rejected": -259.52349853515625,
      "loss": 0.6728,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.029968639835715294,
      "rewards/margins": 0.03762819617986679,
      "rewards/rejected": -0.06759683787822723,
      "step": 900
    },
    {
      "epoch": 1.41,
      "learning_rate": 2.9437105112004597e-07,
      "logits/chosen": -1.175890564918518,
      "logits/rejected": -1.204742193222046,
      "logps/chosen": -331.7793884277344,
      "logps/rejected": -295.1964416503906,
      "loss": 0.6725,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.03465835377573967,
      "rewards/margins": 0.038827430456876755,
      "rewards/rejected": -0.07348578423261642,
      "step": 910
    },
    {
      "epoch": 1.43,
      "learning_rate": 2.9149913842619186e-07,
      "logits/chosen": -1.1797144412994385,
      "logits/rejected": -1.273648738861084,
      "logps/chosen": -303.84906005859375,
      "logps/rejected": -226.57861328125,
      "loss": 0.674,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.018653806298971176,
      "rewards/margins": 0.048885177820920944,
      "rewards/rejected": -0.06753899157047272,
      "step": 920
    },
    {
      "epoch": 1.44,
      "learning_rate": 2.886272257323377e-07,
      "logits/chosen": -1.1142618656158447,
      "logits/rejected": -1.2289409637451172,
      "logps/chosen": -284.84564208984375,
      "logps/rejected": -266.482421875,
      "loss": 0.6732,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.03997833654284477,
      "rewards/margins": 0.0469382144510746,
      "rewards/rejected": -0.08691655099391937,
      "step": 930
    },
    {
      "epoch": 1.46,
      "learning_rate": 2.857553130384836e-07,
      "logits/chosen": -1.1320688724517822,
      "logits/rejected": -1.2028567790985107,
      "logps/chosen": -295.7919921875,
      "logps/rejected": -275.61029052734375,
      "loss": 0.6751,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.031079787760972977,
      "rewards/margins": 0.04319722205400467,
      "rewards/rejected": -0.07427700608968735,
      "step": 940
    },
    {
      "epoch": 1.47,
      "learning_rate": 2.828834003446295e-07,
      "logits/chosen": -1.2210772037506104,
      "logits/rejected": -1.2374837398529053,
      "logps/chosen": -223.2447052001953,
      "logps/rejected": -223.93301391601562,
      "loss": 0.6746,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.025652194395661354,
      "rewards/margins": 0.030472612008452415,
      "rewards/rejected": -0.05612480640411377,
      "step": 950
    },
    {
      "epoch": 1.49,
      "learning_rate": 2.8001148765077544e-07,
      "logits/chosen": -1.1035312414169312,
      "logits/rejected": -1.2297589778900146,
      "logps/chosen": -272.3734436035156,
      "logps/rejected": -229.17208862304688,
      "loss": 0.6726,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.02614757977426052,
      "rewards/margins": 0.05098317191004753,
      "rewards/rejected": -0.0771307498216629,
      "step": 960
    },
    {
      "epoch": 1.5,
      "learning_rate": 2.7713957495692133e-07,
      "logits/chosen": -1.1308553218841553,
      "logits/rejected": -1.1985657215118408,
      "logps/chosen": -279.56158447265625,
      "logps/rejected": -252.9062957763672,
      "loss": 0.6744,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.025463193655014038,
      "rewards/margins": 0.03752460703253746,
      "rewards/rejected": -0.0629878044128418,
      "step": 970
    },
    {
      "epoch": 1.52,
      "learning_rate": 2.742676622630672e-07,
      "logits/chosen": -1.1320959329605103,
      "logits/rejected": -1.2268012762069702,
      "logps/chosen": -269.58233642578125,
      "logps/rejected": -259.0909423828125,
      "loss": 0.6763,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.03594038635492325,
      "rewards/margins": 0.038814641535282135,
      "rewards/rejected": -0.07475502789020538,
      "step": 980
    },
    {
      "epoch": 1.53,
      "learning_rate": 2.7139574956921307e-07,
      "logits/chosen": -1.1909582614898682,
      "logits/rejected": -1.2770507335662842,
      "logps/chosen": -306.2381286621094,
      "logps/rejected": -258.2312316894531,
      "loss": 0.6694,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.013350972905755043,
      "rewards/margins": 0.0710732489824295,
      "rewards/rejected": -0.08442423492670059,
      "step": 990
    },
    {
      "epoch": 1.55,
      "learning_rate": 2.6852383687535896e-07,
      "logits/chosen": -1.1517307758331299,
      "logits/rejected": -1.1953328847885132,
      "logps/chosen": -309.81707763671875,
      "logps/rejected": -255.09542846679688,
      "loss": 0.673,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.02377471514046192,
      "rewards/margins": 0.05378960445523262,
      "rewards/rejected": -0.0775643140077591,
      "step": 1000
    },
    {
      "epoch": 1.56,
      "learning_rate": 2.656519241815049e-07,
      "logits/chosen": -1.172380805015564,
      "logits/rejected": -1.2242120504379272,
      "logps/chosen": -289.8918762207031,
      "logps/rejected": -240.6064453125,
      "loss": 0.6691,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.022976800799369812,
      "rewards/margins": 0.06258256733417511,
      "rewards/rejected": -0.08555936068296432,
      "step": 1010
    },
    {
      "epoch": 1.58,
      "learning_rate": 2.6278001148765075e-07,
      "logits/chosen": -1.209279179573059,
      "logits/rejected": -1.2857582569122314,
      "logps/chosen": -306.1195373535156,
      "logps/rejected": -252.3911590576172,
      "loss": 0.6716,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.04376746341586113,
      "rewards/margins": 0.04451705887913704,
      "rewards/rejected": -0.08828451484441757,
      "step": 1020
    },
    {
      "epoch": 1.6,
      "learning_rate": 2.5990809879379665e-07,
      "logits/chosen": -1.119346261024475,
      "logits/rejected": -1.1804701089859009,
      "logps/chosen": -279.9603576660156,
      "logps/rejected": -255.67166137695312,
      "loss": 0.6713,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.032426051795482635,
      "rewards/margins": 0.04803738743066788,
      "rewards/rejected": -0.08046343177556992,
      "step": 1030
    },
    {
      "epoch": 1.61,
      "learning_rate": 2.5703618609994254e-07,
      "logits/chosen": -1.232366681098938,
      "logits/rejected": -1.2660514116287231,
      "logps/chosen": -267.54205322265625,
      "logps/rejected": -217.8297882080078,
      "loss": 0.6692,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0329388864338398,
      "rewards/margins": 0.04563206806778908,
      "rewards/rejected": -0.07857094705104828,
      "step": 1040
    },
    {
      "epoch": 1.63,
      "learning_rate": 2.5416427340608844e-07,
      "logits/chosen": -1.1432456970214844,
      "logits/rejected": -1.2851306200027466,
      "logps/chosen": -325.90716552734375,
      "logps/rejected": -264.51043701171875,
      "loss": 0.6699,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.035186149179935455,
      "rewards/margins": 0.059220410883426666,
      "rewards/rejected": -0.09440656006336212,
      "step": 1050
    },
    {
      "epoch": 1.64,
      "learning_rate": 2.512923607122344e-07,
      "logits/chosen": -1.1394827365875244,
      "logits/rejected": -1.208301305770874,
      "logps/chosen": -295.6949157714844,
      "logps/rejected": -240.1582794189453,
      "loss": 0.6693,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.025663334876298904,
      "rewards/margins": 0.05790363624691963,
      "rewards/rejected": -0.08356697857379913,
      "step": 1060
    },
    {
      "epoch": 1.66,
      "learning_rate": 2.484204480183802e-07,
      "logits/chosen": -1.1204135417938232,
      "logits/rejected": -1.295609712600708,
      "logps/chosen": -274.9275817871094,
      "logps/rejected": -255.2529754638672,
      "loss": 0.6687,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.029915103688836098,
      "rewards/margins": 0.06398225575685501,
      "rewards/rejected": -0.09389735758304596,
      "step": 1070
    },
    {
      "epoch": 1.67,
      "learning_rate": 2.455485353245261e-07,
      "logits/chosen": -1.1964619159698486,
      "logits/rejected": -1.3005297183990479,
      "logps/chosen": -285.21209716796875,
      "logps/rejected": -260.6269226074219,
      "loss": 0.671,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.03515591099858284,
      "rewards/margins": 0.04965870454907417,
      "rewards/rejected": -0.08481462299823761,
      "step": 1080
    },
    {
      "epoch": 1.69,
      "learning_rate": 2.42676622630672e-07,
      "logits/chosen": -1.1853787899017334,
      "logits/rejected": -1.2281911373138428,
      "logps/chosen": -270.4370422363281,
      "logps/rejected": -214.59933471679688,
      "loss": 0.6685,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.03246784955263138,
      "rewards/margins": 0.046676285564899445,
      "rewards/rejected": -0.07914413511753082,
      "step": 1090
    },
    {
      "epoch": 1.7,
      "learning_rate": 2.398047099368179e-07,
      "logits/chosen": -1.2114546298980713,
      "logits/rejected": -1.253631830215454,
      "logps/chosen": -294.1374206542969,
      "logps/rejected": -228.50796508789062,
      "loss": 0.6681,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.027642633765935898,
      "rewards/margins": 0.08140935003757477,
      "rewards/rejected": -0.10905198752880096,
      "step": 1100
    },
    {
      "epoch": 1.72,
      "learning_rate": 2.369327972429638e-07,
      "logits/chosen": -1.1516766548156738,
      "logits/rejected": -1.2245798110961914,
      "logps/chosen": -278.65423583984375,
      "logps/rejected": -243.95529174804688,
      "loss": 0.6674,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.029988175258040428,
      "rewards/margins": 0.060140348970890045,
      "rewards/rejected": -0.09012852609157562,
      "step": 1110
    },
    {
      "epoch": 1.74,
      "learning_rate": 2.340608845491097e-07,
      "logits/chosen": -1.1279699802398682,
      "logits/rejected": -1.162719488143921,
      "logps/chosen": -331.1317443847656,
      "logps/rejected": -264.7960510253906,
      "loss": 0.6691,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.04450936242938042,
      "rewards/margins": 0.06046723201870918,
      "rewards/rejected": -0.1049765944480896,
      "step": 1120
    },
    {
      "epoch": 1.75,
      "learning_rate": 2.311889718552556e-07,
      "logits/chosen": -1.152203917503357,
      "logits/rejected": -1.2137216329574585,
      "logps/chosen": -289.91473388671875,
      "logps/rejected": -252.63174438476562,
      "loss": 0.6689,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.0418432392179966,
      "rewards/margins": 0.05185099318623543,
      "rewards/rejected": -0.09369422495365143,
      "step": 1130
    },
    {
      "epoch": 1.77,
      "learning_rate": 2.283170591614015e-07,
      "logits/chosen": -1.2258096933364868,
      "logits/rejected": -1.271815299987793,
      "logps/chosen": -271.7290344238281,
      "logps/rejected": -253.15481567382812,
      "loss": 0.6652,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.028518235310912132,
      "rewards/margins": 0.08087646216154099,
      "rewards/rejected": -0.10939470678567886,
      "step": 1140
    },
    {
      "epoch": 1.78,
      "learning_rate": 2.2544514646754736e-07,
      "logits/chosen": -1.1422202587127686,
      "logits/rejected": -1.2273261547088623,
      "logps/chosen": -291.89166259765625,
      "logps/rejected": -254.87002563476562,
      "loss": 0.6662,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.04026695340871811,
      "rewards/margins": 0.055162213742733,
      "rewards/rejected": -0.09542916715145111,
      "step": 1150
    },
    {
      "epoch": 1.8,
      "learning_rate": 2.2257323377369328e-07,
      "logits/chosen": -1.1333295106887817,
      "logits/rejected": -1.2422702312469482,
      "logps/chosen": -289.364990234375,
      "logps/rejected": -232.06692504882812,
      "loss": 0.6603,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.021913789212703705,
      "rewards/margins": 0.06650007516145706,
      "rewards/rejected": -0.08841386437416077,
      "step": 1160
    },
    {
      "epoch": 1.81,
      "learning_rate": 2.1970132107983917e-07,
      "logits/chosen": -1.2066811323165894,
      "logits/rejected": -1.2231658697128296,
      "logps/chosen": -275.4366149902344,
      "logps/rejected": -224.98867797851562,
      "loss": 0.6672,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.038331288844347,
      "rewards/margins": 0.05360294505953789,
      "rewards/rejected": -0.09193422645330429,
      "step": 1170
    },
    {
      "epoch": 1.83,
      "learning_rate": 2.1682940838598507e-07,
      "logits/chosen": -1.1569344997406006,
      "logits/rejected": -1.2456449270248413,
      "logps/chosen": -274.32159423828125,
      "logps/rejected": -252.0222625732422,
      "loss": 0.6709,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.052247773855924606,
      "rewards/margins": 0.022707702592015266,
      "rewards/rejected": -0.07495548576116562,
      "step": 1180
    },
    {
      "epoch": 1.84,
      "learning_rate": 2.1395749569213096e-07,
      "logits/chosen": -1.123389720916748,
      "logits/rejected": -1.251558542251587,
      "logps/chosen": -276.007568359375,
      "logps/rejected": -235.1134033203125,
      "loss": 0.6671,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.04328937083482742,
      "rewards/margins": 0.03924347832798958,
      "rewards/rejected": -0.0825328528881073,
      "step": 1190
    },
    {
      "epoch": 1.86,
      "learning_rate": 2.1108558299827683e-07,
      "logits/chosen": -1.143604040145874,
      "logits/rejected": -1.2688237428665161,
      "logps/chosen": -278.1545104980469,
      "logps/rejected": -237.94912719726562,
      "loss": 0.6629,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.042831793427467346,
      "rewards/margins": 0.06205050274729729,
      "rewards/rejected": -0.10488228499889374,
      "step": 1200
    },
    {
      "epoch": 1.87,
      "learning_rate": 2.0821367030442275e-07,
      "logits/chosen": -1.1230822801589966,
      "logits/rejected": -1.2531689405441284,
      "logps/chosen": -297.42108154296875,
      "logps/rejected": -255.71334838867188,
      "loss": 0.67,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.0464252233505249,
      "rewards/margins": 0.07465530931949615,
      "rewards/rejected": -0.12108053267002106,
      "step": 1210
    },
    {
      "epoch": 1.89,
      "learning_rate": 2.0534175761056862e-07,
      "logits/chosen": -1.1438612937927246,
      "logits/rejected": -1.1976368427276611,
      "logps/chosen": -325.1570739746094,
      "logps/rejected": -293.0627136230469,
      "loss": 0.6618,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.029308268800377846,
      "rewards/margins": 0.08643002808094025,
      "rewards/rejected": -0.11573830991983414,
      "step": 1220
    },
    {
      "epoch": 1.91,
      "learning_rate": 2.024698449167145e-07,
      "logits/chosen": -1.1537196636199951,
      "logits/rejected": -1.2745457887649536,
      "logps/chosen": -302.37042236328125,
      "logps/rejected": -251.1383514404297,
      "loss": 0.6664,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.038567863404750824,
      "rewards/margins": 0.06020348519086838,
      "rewards/rejected": -0.0987713485956192,
      "step": 1230
    },
    {
      "epoch": 1.92,
      "learning_rate": 1.9959793222286043e-07,
      "logits/chosen": -1.1626816987991333,
      "logits/rejected": -1.2281776666641235,
      "logps/chosen": -264.9757995605469,
      "logps/rejected": -235.031982421875,
      "loss": 0.6628,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.04553893953561783,
      "rewards/margins": 0.0635872557759285,
      "rewards/rejected": -0.10912618786096573,
      "step": 1240
    },
    {
      "epoch": 1.94,
      "learning_rate": 1.967260195290063e-07,
      "logits/chosen": -1.1658340692520142,
      "logits/rejected": -1.210275411605835,
      "logps/chosen": -296.4925231933594,
      "logps/rejected": -259.5403747558594,
      "loss": 0.6633,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.04651462659239769,
      "rewards/margins": 0.060394324362277985,
      "rewards/rejected": -0.10690895467996597,
      "step": 1250
    },
    {
      "epoch": 1.95,
      "learning_rate": 1.9385410683515222e-07,
      "logits/chosen": -1.1199092864990234,
      "logits/rejected": -1.1857876777648926,
      "logps/chosen": -309.8304748535156,
      "logps/rejected": -252.6132354736328,
      "loss": 0.6665,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.02937604859471321,
      "rewards/margins": 0.06977292895317078,
      "rewards/rejected": -0.09914897382259369,
      "step": 1260
    },
    {
      "epoch": 1.97,
      "learning_rate": 1.909821941412981e-07,
      "logits/chosen": -1.098323106765747,
      "logits/rejected": -1.2124502658843994,
      "logps/chosen": -309.689453125,
      "logps/rejected": -257.48760986328125,
      "loss": 0.6663,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.03487952798604965,
      "rewards/margins": 0.06294190138578415,
      "rewards/rejected": -0.0978214293718338,
      "step": 1270
    },
    {
      "epoch": 1.98,
      "learning_rate": 1.8811028144744398e-07,
      "logits/chosen": -1.1761606931686401,
      "logits/rejected": -1.239966869354248,
      "logps/chosen": -341.273681640625,
      "logps/rejected": -273.9056701660156,
      "loss": 0.6608,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.031055117025971413,
      "rewards/margins": 0.08567170798778534,
      "rewards/rejected": -0.1167268380522728,
      "step": 1280
    },
    {
      "epoch": 2.0,
      "learning_rate": 1.8523836875358988e-07,
      "logits/chosen": -1.2572320699691772,
      "logits/rejected": -1.2493976354599,
      "logps/chosen": -280.1720886230469,
      "logps/rejected": -230.7694854736328,
      "loss": 0.667,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.05663619190454483,
      "rewards/margins": 0.0724005326628685,
      "rewards/rejected": -0.12903672456741333,
      "step": 1290
    },
    {
      "epoch": 2.0,
      "eval_logits/chosen": -1.2786041498184204,
      "eval_logits/rejected": -1.4456814527511597,
      "eval_logps/chosen": -271.3624572753906,
      "eval_logps/rejected": -241.35267639160156,
      "eval_loss": 0.6636343002319336,
      "eval_rewards/accuracies": 0.6517857313156128,
      "eval_rewards/chosen": -0.047330476343631744,
      "eval_rewards/margins": 0.06953465938568115,
      "eval_rewards/rejected": -0.1168651357293129,
      "eval_runtime": 335.8327,
      "eval_samples_per_second": 5.955,
      "eval_steps_per_second": 0.25,
      "step": 1291
    },
    {
      "epoch": 2.01,
      "learning_rate": 1.8236645605973577e-07,
      "logits/chosen": -1.2015001773834229,
      "logits/rejected": -1.2663769721984863,
      "logps/chosen": -270.6482238769531,
      "logps/rejected": -263.001708984375,
      "loss": 0.6659,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.04612879827618599,
      "rewards/margins": 0.05501723289489746,
      "rewards/rejected": -0.10114602744579315,
      "step": 1300
    },
    {
      "epoch": 2.03,
      "learning_rate": 1.7949454336588167e-07,
      "logits/chosen": -1.1482102870941162,
      "logits/rejected": -1.2331697940826416,
      "logps/chosen": -276.347412109375,
      "logps/rejected": -251.0341796875,
      "loss": 0.6646,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.05418051406741142,
      "rewards/margins": 0.06352085620164871,
      "rewards/rejected": -0.11770136654376984,
      "step": 1310
    },
    {
      "epoch": 2.04,
      "learning_rate": 1.7662263067202756e-07,
      "logits/chosen": -1.1770139932632446,
      "logits/rejected": -1.1769516468048096,
      "logps/chosen": -285.74200439453125,
      "logps/rejected": -251.66641235351562,
      "loss": 0.6635,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.056278862059116364,
      "rewards/margins": 0.05115005373954773,
      "rewards/rejected": -0.1074289083480835,
      "step": 1320
    },
    {
      "epoch": 2.06,
      "learning_rate": 1.7375071797817346e-07,
      "logits/chosen": -1.1344884634017944,
      "logits/rejected": -1.1952565908432007,
      "logps/chosen": -281.35772705078125,
      "logps/rejected": -241.61953735351562,
      "loss": 0.662,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.05323750525712967,
      "rewards/margins": 0.04273111745715141,
      "rewards/rejected": -0.09596861898899078,
      "step": 1330
    },
    {
      "epoch": 2.08,
      "learning_rate": 1.7087880528431935e-07,
      "logits/chosen": -1.1021802425384521,
      "logits/rejected": -1.1934154033660889,
      "logps/chosen": -306.3484802246094,
      "logps/rejected": -239.8199005126953,
      "loss": 0.6682,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.03922051936388016,
      "rewards/margins": 0.07538510859012604,
      "rewards/rejected": -0.1146056279540062,
      "step": 1340
    },
    {
      "epoch": 2.09,
      "learning_rate": 1.6800689259046525e-07,
      "logits/chosen": -1.2181620597839355,
      "logits/rejected": -1.3131446838378906,
      "logps/chosen": -274.2059326171875,
      "logps/rejected": -230.08395385742188,
      "loss": 0.6647,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.045338355004787445,
      "rewards/margins": 0.06275372207164764,
      "rewards/rejected": -0.10809208452701569,
      "step": 1350
    },
    {
      "epoch": 2.11,
      "learning_rate": 1.6513497989661112e-07,
      "logits/chosen": -1.1318702697753906,
      "logits/rejected": -1.26137113571167,
      "logps/chosen": -282.0677795410156,
      "logps/rejected": -236.37948608398438,
      "loss": 0.6592,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.05192691087722778,
      "rewards/margins": 0.07602426409721375,
      "rewards/rejected": -0.12795117497444153,
      "step": 1360
    },
    {
      "epoch": 2.12,
      "learning_rate": 1.6226306720275704e-07,
      "logits/chosen": -1.1841795444488525,
      "logits/rejected": -1.297102451324463,
      "logps/chosen": -305.37451171875,
      "logps/rejected": -253.83737182617188,
      "loss": 0.6605,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.03761546313762665,
      "rewards/margins": 0.06815005838871002,
      "rewards/rejected": -0.10576552152633667,
      "step": 1370
    },
    {
      "epoch": 2.14,
      "learning_rate": 1.5939115450890293e-07,
      "logits/chosen": -1.2069101333618164,
      "logits/rejected": -1.291855812072754,
      "logps/chosen": -324.9100341796875,
      "logps/rejected": -252.5089111328125,
      "loss": 0.6648,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04376627877354622,
      "rewards/margins": 0.05180872231721878,
      "rewards/rejected": -0.0955749899148941,
      "step": 1380
    },
    {
      "epoch": 2.15,
      "learning_rate": 1.565192418150488e-07,
      "logits/chosen": -1.158197045326233,
      "logits/rejected": -1.2244586944580078,
      "logps/chosen": -276.658935546875,
      "logps/rejected": -248.20895385742188,
      "loss": 0.6673,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07143920660018921,
      "rewards/margins": 0.021507441997528076,
      "rewards/rejected": -0.09294665604829788,
      "step": 1390
    },
    {
      "epoch": 2.17,
      "learning_rate": 1.5364732912119472e-07,
      "logits/chosen": -1.1375401020050049,
      "logits/rejected": -1.182159185409546,
      "logps/chosen": -309.81658935546875,
      "logps/rejected": -270.9930114746094,
      "loss": 0.6565,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.0605141744017601,
      "rewards/margins": 0.08565731346607208,
      "rewards/rejected": -0.1461714804172516,
      "step": 1400
    },
    {
      "epoch": 2.18,
      "learning_rate": 1.507754164273406e-07,
      "logits/chosen": -1.1340091228485107,
      "logits/rejected": -1.230928897857666,
      "logps/chosen": -296.4352722167969,
      "logps/rejected": -244.35397338867188,
      "loss": 0.659,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.04029616340994835,
      "rewards/margins": 0.09661685675382614,
      "rewards/rejected": -0.13691303133964539,
      "step": 1410
    },
    {
      "epoch": 2.2,
      "learning_rate": 1.479035037334865e-07,
      "logits/chosen": -1.2611504793167114,
      "logits/rejected": -1.2926255464553833,
      "logps/chosen": -294.1080017089844,
      "logps/rejected": -224.856201171875,
      "loss": 0.661,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.060603827238082886,
      "rewards/margins": 0.07348962128162384,
      "rewards/rejected": -0.13409346342086792,
      "step": 1420
    },
    {
      "epoch": 2.22,
      "learning_rate": 1.4503159103963238e-07,
      "logits/chosen": -1.1362766027450562,
      "logits/rejected": -1.2392619848251343,
      "logps/chosen": -318.2626037597656,
      "logps/rejected": -277.73138427734375,
      "loss": 0.6655,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.0653274655342102,
      "rewards/margins": 0.07404564321041107,
      "rewards/rejected": -0.13937310874462128,
      "step": 1430
    },
    {
      "epoch": 2.23,
      "learning_rate": 1.4215967834577827e-07,
      "logits/chosen": -1.1209288835525513,
      "logits/rejected": -1.2438892126083374,
      "logps/chosen": -298.46087646484375,
      "logps/rejected": -234.68795776367188,
      "loss": 0.6601,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.04315265268087387,
      "rewards/margins": 0.07634139806032181,
      "rewards/rejected": -0.11949405819177628,
      "step": 1440
    },
    {
      "epoch": 2.25,
      "learning_rate": 1.392877656519242e-07,
      "logits/chosen": -1.1044480800628662,
      "logits/rejected": -1.1969009637832642,
      "logps/chosen": -299.08575439453125,
      "logps/rejected": -264.5302429199219,
      "loss": 0.6613,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.05541745945811272,
      "rewards/margins": 0.09096468240022659,
      "rewards/rejected": -0.146382138133049,
      "step": 1450
    },
    {
      "epoch": 2.26,
      "learning_rate": 1.3641585295807006e-07,
      "logits/chosen": -1.1369094848632812,
      "logits/rejected": -1.2164510488510132,
      "logps/chosen": -264.0603332519531,
      "logps/rejected": -248.4859161376953,
      "loss": 0.6664,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.045277245342731476,
      "rewards/margins": 0.07062248885631561,
      "rewards/rejected": -0.11589974164962769,
      "step": 1460
    },
    {
      "epoch": 2.28,
      "learning_rate": 1.3354394026421596e-07,
      "logits/chosen": -1.177728533744812,
      "logits/rejected": -1.2632405757904053,
      "logps/chosen": -292.7504577636719,
      "logps/rejected": -247.25338745117188,
      "loss": 0.6652,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.06021810322999954,
      "rewards/margins": 0.07726706564426422,
      "rewards/rejected": -0.13748517632484436,
      "step": 1470
    },
    {
      "epoch": 2.29,
      "learning_rate": 1.3067202757036185e-07,
      "logits/chosen": -1.0999003648757935,
      "logits/rejected": -1.2220264673233032,
      "logps/chosen": -315.4903259277344,
      "logps/rejected": -256.5108947753906,
      "loss": 0.6545,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.026393622159957886,
      "rewards/margins": 0.13147220015525818,
      "rewards/rejected": -0.15786585211753845,
      "step": 1480
    },
    {
      "epoch": 2.31,
      "learning_rate": 1.2780011487650774e-07,
      "logits/chosen": -1.1594305038452148,
      "logits/rejected": -1.204489827156067,
      "logps/chosen": -248.64297485351562,
      "logps/rejected": -231.457275390625,
      "loss": 0.6575,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06583482772111893,
      "rewards/margins": 0.06832978874444962,
      "rewards/rejected": -0.13416460156440735,
      "step": 1490
    },
    {
      "epoch": 2.32,
      "learning_rate": 1.2492820218265364e-07,
      "logits/chosen": -1.2294750213623047,
      "logits/rejected": -1.2381101846694946,
      "logps/chosen": -301.78668212890625,
      "logps/rejected": -262.041259765625,
      "loss": 0.6591,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.0509830042719841,
      "rewards/margins": 0.07165484875440598,
      "rewards/rejected": -0.12263785302639008,
      "step": 1500
    },
    {
      "epoch": 2.34,
      "learning_rate": 1.2205628948879953e-07,
      "logits/chosen": -1.187576413154602,
      "logits/rejected": -1.208719253540039,
      "logps/chosen": -278.07391357421875,
      "logps/rejected": -239.7738800048828,
      "loss": 0.6629,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07879756391048431,
      "rewards/margins": 0.06315238773822784,
      "rewards/rejected": -0.14194993674755096,
      "step": 1510
    },
    {
      "epoch": 2.35,
      "learning_rate": 1.1918437679494543e-07,
      "logits/chosen": -1.1406846046447754,
      "logits/rejected": -1.2449123859405518,
      "logps/chosen": -280.1681213378906,
      "logps/rejected": -251.434814453125,
      "loss": 0.6564,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.051693908870220184,
      "rewards/margins": 0.08930797874927521,
      "rewards/rejected": -0.141001895070076,
      "step": 1520
    },
    {
      "epoch": 2.37,
      "learning_rate": 1.1631246410109131e-07,
      "logits/chosen": -1.1076427698135376,
      "logits/rejected": -1.200714111328125,
      "logps/chosen": -302.08673095703125,
      "logps/rejected": -255.66354370117188,
      "loss": 0.6565,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.046904128044843674,
      "rewards/margins": 0.09861443936824799,
      "rewards/rejected": -0.14551857113838196,
      "step": 1530
    },
    {
      "epoch": 2.39,
      "learning_rate": 1.1344055140723722e-07,
      "logits/chosen": -1.152897596359253,
      "logits/rejected": -1.2206456661224365,
      "logps/chosen": -290.43511962890625,
      "logps/rejected": -253.755615234375,
      "loss": 0.6623,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07052378356456757,
      "rewards/margins": 0.06900990009307861,
      "rewards/rejected": -0.13953366875648499,
      "step": 1540
    },
    {
      "epoch": 2.4,
      "learning_rate": 1.1056863871338311e-07,
      "logits/chosen": -1.1606311798095703,
      "logits/rejected": -1.2234033346176147,
      "logps/chosen": -275.7051086425781,
      "logps/rejected": -257.5712585449219,
      "loss": 0.6545,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.06670763343572617,
      "rewards/margins": 0.07591570168733597,
      "rewards/rejected": -0.14262333512306213,
      "step": 1550
    },
    {
      "epoch": 2.42,
      "learning_rate": 1.07696726019529e-07,
      "logits/chosen": -1.0971261262893677,
      "logits/rejected": -1.2159391641616821,
      "logps/chosen": -303.5472717285156,
      "logps/rejected": -262.9560546875,
      "loss": 0.6559,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06790264695882797,
      "rewards/margins": 0.08240754902362823,
      "rewards/rejected": -0.1503101885318756,
      "step": 1560
    },
    {
      "epoch": 2.43,
      "learning_rate": 1.0482481332567489e-07,
      "logits/chosen": -1.1438579559326172,
      "logits/rejected": -1.2413270473480225,
      "logps/chosen": -302.71905517578125,
      "logps/rejected": -233.7013702392578,
      "loss": 0.6586,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.049414295703172684,
      "rewards/margins": 0.07926493883132935,
      "rewards/rejected": -0.12867923080921173,
      "step": 1570
    },
    {
      "epoch": 2.45,
      "learning_rate": 1.0195290063182078e-07,
      "logits/chosen": -1.1184961795806885,
      "logits/rejected": -1.191981315612793,
      "logps/chosen": -271.0045471191406,
      "logps/rejected": -243.72982788085938,
      "loss": 0.6574,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0692010149359703,
      "rewards/margins": 0.06468804180622101,
      "rewards/rejected": -0.13388904929161072,
      "step": 1580
    },
    {
      "epoch": 2.46,
      "learning_rate": 9.908098793796668e-08,
      "logits/chosen": -1.1387625932693481,
      "logits/rejected": -1.1771440505981445,
      "logps/chosen": -298.8290710449219,
      "logps/rejected": -284.1943054199219,
      "loss": 0.6578,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.049336209893226624,
      "rewards/margins": 0.08348609507083893,
      "rewards/rejected": -0.13282229006290436,
      "step": 1590
    },
    {
      "epoch": 2.48,
      "learning_rate": 9.620907524411257e-08,
      "logits/chosen": -1.1552401781082153,
      "logits/rejected": -1.2284502983093262,
      "logps/chosen": -289.09979248046875,
      "logps/rejected": -264.1502685546875,
      "loss": 0.6579,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.06869687885046005,
      "rewards/margins": 0.05455329269170761,
      "rewards/rejected": -0.12325018644332886,
      "step": 1600
    },
    {
      "epoch": 2.49,
      "learning_rate": 9.333716255025848e-08,
      "logits/chosen": -1.2256122827529907,
      "logits/rejected": -1.3060810565948486,
      "logps/chosen": -299.1298522949219,
      "logps/rejected": -260.7080993652344,
      "loss": 0.6597,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.05211590602993965,
      "rewards/margins": 0.09913495182991028,
      "rewards/rejected": -0.15125085413455963,
      "step": 1610
    },
    {
      "epoch": 2.51,
      "learning_rate": 9.046524985640436e-08,
      "logits/chosen": -1.196408987045288,
      "logits/rejected": -1.234323263168335,
      "logps/chosen": -276.99462890625,
      "logps/rejected": -269.7616882324219,
      "loss": 0.6645,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06248649209737778,
      "rewards/margins": 0.0762564167380333,
      "rewards/rejected": -0.13874289393424988,
      "step": 1620
    },
    {
      "epoch": 2.53,
      "learning_rate": 8.759333716255026e-08,
      "logits/chosen": -1.1722522974014282,
      "logits/rejected": -1.2352186441421509,
      "logps/chosen": -317.6283264160156,
      "logps/rejected": -265.194580078125,
      "loss": 0.656,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.06108713895082474,
      "rewards/margins": 0.09596574306488037,
      "rewards/rejected": -0.1570528745651245,
      "step": 1630
    },
    {
      "epoch": 2.54,
      "learning_rate": 8.472142446869615e-08,
      "logits/chosen": -1.253702163696289,
      "logits/rejected": -1.2648756504058838,
      "logps/chosen": -311.29071044921875,
      "logps/rejected": -259.5384826660156,
      "loss": 0.6568,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.05935510993003845,
      "rewards/margins": 0.08772717416286469,
      "rewards/rejected": -0.14708228409290314,
      "step": 1640
    },
    {
      "epoch": 2.56,
      "learning_rate": 8.184951177484204e-08,
      "logits/chosen": -1.1213136911392212,
      "logits/rejected": -1.1935571432113647,
      "logps/chosen": -298.7812194824219,
      "logps/rejected": -263.63189697265625,
      "loss": 0.6595,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.04742642492055893,
      "rewards/margins": 0.12104301154613495,
      "rewards/rejected": -0.16846942901611328,
      "step": 1650
    },
    {
      "epoch": 2.57,
      "learning_rate": 7.897759908098793e-08,
      "logits/chosen": -1.1366965770721436,
      "logits/rejected": -1.2882981300354004,
      "logps/chosen": -298.13671875,
      "logps/rejected": -255.32351684570312,
      "loss": 0.6562,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.06480282545089722,
      "rewards/margins": 0.11208094656467438,
      "rewards/rejected": -0.1768837720155716,
      "step": 1660
    },
    {
      "epoch": 2.59,
      "learning_rate": 7.610568638713383e-08,
      "logits/chosen": -1.153519630432129,
      "logits/rejected": -1.1792590618133545,
      "logps/chosen": -274.3094177246094,
      "logps/rejected": -277.9222717285156,
      "loss": 0.6529,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.060344815254211426,
      "rewards/margins": 0.07477900385856628,
      "rewards/rejected": -0.1351238191127777,
      "step": 1670
    },
    {
      "epoch": 2.6,
      "learning_rate": 7.323377369327973e-08,
      "logits/chosen": -1.1138644218444824,
      "logits/rejected": -1.2077605724334717,
      "logps/chosen": -284.9149475097656,
      "logps/rejected": -238.51315307617188,
      "loss": 0.6584,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.05787507817149162,
      "rewards/margins": 0.08078931272029877,
      "rewards/rejected": -0.1386643946170807,
      "step": 1680
    },
    {
      "epoch": 2.62,
      "learning_rate": 7.036186099942562e-08,
      "logits/chosen": -1.1559264659881592,
      "logits/rejected": -1.2110865116119385,
      "logps/chosen": -273.95074462890625,
      "logps/rejected": -252.70968627929688,
      "loss": 0.6559,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.036359649151563644,
      "rewards/margins": 0.1054670661687851,
      "rewards/rejected": -0.14182671904563904,
      "step": 1690
    },
    {
      "epoch": 2.63,
      "learning_rate": 6.74899483055715e-08,
      "logits/chosen": -1.208526849746704,
      "logits/rejected": -1.26242196559906,
      "logps/chosen": -297.33843994140625,
      "logps/rejected": -268.06573486328125,
      "loss": 0.6589,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.046420447528362274,
      "rewards/margins": 0.08910312503576279,
      "rewards/rejected": -0.13552358746528625,
      "step": 1700
    },
    {
      "epoch": 2.65,
      "learning_rate": 6.46180356117174e-08,
      "logits/chosen": -1.1605157852172852,
      "logits/rejected": -1.1535265445709229,
      "logps/chosen": -276.1454772949219,
      "logps/rejected": -264.06512451171875,
      "loss": 0.6648,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.06386712193489075,
      "rewards/margins": 0.06835543364286423,
      "rewards/rejected": -0.13222254812717438,
      "step": 1710
    },
    {
      "epoch": 2.66,
      "learning_rate": 6.174612291786329e-08,
      "logits/chosen": -1.1071873903274536,
      "logits/rejected": -1.2749332189559937,
      "logps/chosen": -312.7781066894531,
      "logps/rejected": -263.4851989746094,
      "loss": 0.654,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07527683675289154,
      "rewards/margins": 0.07555703818798065,
      "rewards/rejected": -0.1508338749408722,
      "step": 1720
    },
    {
      "epoch": 2.68,
      "learning_rate": 5.887421022400919e-08,
      "logits/chosen": -1.132735252380371,
      "logits/rejected": -1.2301145792007446,
      "logps/chosen": -309.93048095703125,
      "logps/rejected": -240.8332061767578,
      "loss": 0.6553,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07498300075531006,
      "rewards/margins": 0.08411093801259995,
      "rewards/rejected": -0.1590939462184906,
      "step": 1730
    },
    {
      "epoch": 2.7,
      "learning_rate": 5.600229753015508e-08,
      "logits/chosen": -1.1434319019317627,
      "logits/rejected": -1.2648731470108032,
      "logps/chosen": -310.2062683105469,
      "logps/rejected": -241.0065155029297,
      "loss": 0.6521,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.058048129081726074,
      "rewards/margins": 0.09094297885894775,
      "rewards/rejected": -0.14899109303951263,
      "step": 1740
    },
    {
      "epoch": 2.71,
      "learning_rate": 5.313038483630097e-08,
      "logits/chosen": -1.1104892492294312,
      "logits/rejected": -1.2646420001983643,
      "logps/chosen": -319.0297546386719,
      "logps/rejected": -264.7847595214844,
      "loss": 0.6509,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.03740755841135979,
      "rewards/margins": 0.09465880692005157,
      "rewards/rejected": -0.13206636905670166,
      "step": 1750
    },
    {
      "epoch": 2.73,
      "learning_rate": 5.025847214244687e-08,
      "logits/chosen": -1.1556150913238525,
      "logits/rejected": -1.1965545415878296,
      "logps/chosen": -302.27960205078125,
      "logps/rejected": -237.3252716064453,
      "loss": 0.6585,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.0522063672542572,
      "rewards/margins": 0.08026628196239471,
      "rewards/rejected": -0.13247264921665192,
      "step": 1760
    },
    {
      "epoch": 2.74,
      "learning_rate": 4.738655944859276e-08,
      "logits/chosen": -1.1446633338928223,
      "logits/rejected": -1.2452003955841064,
      "logps/chosen": -330.00531005859375,
      "logps/rejected": -293.3304138183594,
      "loss": 0.6612,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07666303217411041,
      "rewards/margins": 0.08382611721754074,
      "rewards/rejected": -0.16048914194107056,
      "step": 1770
    },
    {
      "epoch": 2.76,
      "learning_rate": 4.4514646754738654e-08,
      "logits/chosen": -1.1485297679901123,
      "logits/rejected": -1.2428683042526245,
      "logps/chosen": -301.80657958984375,
      "logps/rejected": -243.60043334960938,
      "loss": 0.6477,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.060366760939359665,
      "rewards/margins": 0.09364822506904602,
      "rewards/rejected": -0.15401498973369598,
      "step": 1780
    },
    {
      "epoch": 2.77,
      "learning_rate": 4.164273406088455e-08,
      "logits/chosen": -1.1510671377182007,
      "logits/rejected": -1.2577351331710815,
      "logps/chosen": -299.68896484375,
      "logps/rejected": -282.103515625,
      "loss": 0.6607,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.05730317905545235,
      "rewards/margins": 0.09459875524044037,
      "rewards/rejected": -0.1519019454717636,
      "step": 1790
    },
    {
      "epoch": 2.79,
      "learning_rate": 3.877082136703044e-08,
      "logits/chosen": -1.06912100315094,
      "logits/rejected": -1.2072498798370361,
      "logps/chosen": -320.6122131347656,
      "logps/rejected": -260.58599853515625,
      "loss": 0.6506,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0490909144282341,
      "rewards/margins": 0.12078486382961273,
      "rewards/rejected": -0.16987577080726624,
      "step": 1800
    },
    {
      "epoch": 2.8,
      "learning_rate": 3.589890867317633e-08,
      "logits/chosen": -1.0877912044525146,
      "logits/rejected": -1.1389758586883545,
      "logps/chosen": -312.76190185546875,
      "logps/rejected": -258.6143798828125,
      "loss": 0.6573,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.05302386358380318,
      "rewards/margins": 0.0961315929889679,
      "rewards/rejected": -0.14915546774864197,
      "step": 1810
    },
    {
      "epoch": 2.82,
      "learning_rate": 3.302699597932223e-08,
      "logits/chosen": -1.1518110036849976,
      "logits/rejected": -1.1895443201065063,
      "logps/chosen": -286.28955078125,
      "logps/rejected": -237.89675903320312,
      "loss": 0.6577,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.03918531537055969,
      "rewards/margins": 0.10102204233407974,
      "rewards/rejected": -0.14020733535289764,
      "step": 1820
    },
    {
      "epoch": 2.84,
      "learning_rate": 3.015508328546812e-08,
      "logits/chosen": -1.161084771156311,
      "logits/rejected": -1.2788031101226807,
      "logps/chosen": -280.8258972167969,
      "logps/rejected": -248.14028930664062,
      "loss": 0.6586,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.06762611865997314,
      "rewards/margins": 0.08084806054830551,
      "rewards/rejected": -0.14847418665885925,
      "step": 1830
    },
    {
      "epoch": 2.85,
      "learning_rate": 2.728317059161401e-08,
      "logits/chosen": -1.144046664237976,
      "logits/rejected": -1.2353025674819946,
      "logps/chosen": -296.1619567871094,
      "logps/rejected": -254.094970703125,
      "loss": 0.6511,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06453101336956024,
      "rewards/margins": 0.08718563616275787,
      "rewards/rejected": -0.1517166644334793,
      "step": 1840
    },
    {
      "epoch": 2.87,
      "learning_rate": 2.4411257897759906e-08,
      "logits/chosen": -1.0756458044052124,
      "logits/rejected": -1.1395611763000488,
      "logps/chosen": -272.8941955566406,
      "logps/rejected": -248.87109375,
      "loss": 0.6567,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.06941407173871994,
      "rewards/margins": 0.057349514216184616,
      "rewards/rejected": -0.12676359713077545,
      "step": 1850
    },
    {
      "epoch": 2.88,
      "learning_rate": 2.1539345203905797e-08,
      "logits/chosen": -1.2316334247589111,
      "logits/rejected": -1.2592030763626099,
      "logps/chosen": -285.9391784667969,
      "logps/rejected": -258.1458740234375,
      "loss": 0.6625,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0662040263414383,
      "rewards/margins": 0.08364922553300858,
      "rewards/rejected": -0.14985325932502747,
      "step": 1860
    },
    {
      "epoch": 2.9,
      "learning_rate": 1.8667432510051692e-08,
      "logits/chosen": -1.1365151405334473,
      "logits/rejected": -1.273109793663025,
      "logps/chosen": -293.24676513671875,
      "logps/rejected": -247.6940460205078,
      "loss": 0.6536,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07556969672441483,
      "rewards/margins": 0.10609056055545807,
      "rewards/rejected": -0.1816602647304535,
      "step": 1870
    },
    {
      "epoch": 2.91,
      "learning_rate": 1.5795519816197586e-08,
      "logits/chosen": -1.20346999168396,
      "logits/rejected": -1.225865364074707,
      "logps/chosen": -296.02911376953125,
      "logps/rejected": -248.4931640625,
      "loss": 0.6531,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.04528854414820671,
      "rewards/margins": 0.09233234822750092,
      "rewards/rejected": -0.13762088119983673,
      "step": 1880
    },
    {
      "epoch": 2.93,
      "learning_rate": 1.2923607122343481e-08,
      "logits/chosen": -1.1362476348876953,
      "logits/rejected": -1.1712617874145508,
      "logps/chosen": -327.4137268066406,
      "logps/rejected": -262.9008483886719,
      "loss": 0.6538,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.06112866848707199,
      "rewards/margins": 0.09199599176645279,
      "rewards/rejected": -0.15312466025352478,
      "step": 1890
    },
    {
      "epoch": 2.94,
      "learning_rate": 1.0051694428489374e-08,
      "logits/chosen": -1.1542044878005981,
      "logits/rejected": -1.2174440622329712,
      "logps/chosen": -274.6893615722656,
      "logps/rejected": -254.463623046875,
      "loss": 0.6558,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.060687076300382614,
      "rewards/margins": 0.0842573344707489,
      "rewards/rejected": -0.14494439959526062,
      "step": 1900
    },
    {
      "epoch": 2.96,
      "learning_rate": 7.179781734635267e-09,
      "logits/chosen": -1.108367919921875,
      "logits/rejected": -1.16150963306427,
      "logps/chosen": -288.6197509765625,
      "logps/rejected": -249.8366241455078,
      "loss": 0.6545,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.060938142240047455,
      "rewards/margins": 0.08311086148023605,
      "rewards/rejected": -0.1440490037202835,
      "step": 1910
    },
    {
      "epoch": 2.97,
      "learning_rate": 4.30786904078116e-09,
      "logits/chosen": -1.0845935344696045,
      "logits/rejected": -1.1811341047286987,
      "logps/chosen": -297.39599609375,
      "logps/rejected": -259.75,
      "loss": 0.6557,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.04327515885233879,
      "rewards/margins": 0.0852704644203186,
      "rewards/rejected": -0.1285456269979477,
      "step": 1920
    },
    {
      "epoch": 2.99,
      "learning_rate": 1.4359563469270533e-09,
      "logits/chosen": -1.2177361249923706,
      "logits/rejected": -1.2058055400848389,
      "logps/chosen": -263.7599182128906,
      "logps/rejected": -247.1952667236328,
      "loss": 0.6518,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.06555221229791641,
      "rewards/margins": 0.0862443745136261,
      "rewards/rejected": -0.15179657936096191,
      "step": 1930
    },
    {
      "epoch": 3.0,
      "eval_logits/chosen": -1.2746007442474365,
      "eval_logits/rejected": -1.4412107467651367,
      "eval_logps/chosen": -271.52374267578125,
      "eval_logps/rejected": -241.7964630126953,
      "eval_loss": 0.6550792455673218,
      "eval_rewards/accuracies": 0.6904761791229248,
      "eval_rewards/chosen": -0.0634559839963913,
      "eval_rewards/margins": 0.09778770059347153,
      "eval_rewards/rejected": -0.16124369204044342,
      "eval_runtime": 340.5371,
      "eval_samples_per_second": 5.873,
      "eval_steps_per_second": 0.247,
      "step": 1935
    },
    {
      "epoch": 3.0,
      "step": 1935,
      "total_flos": 0.0,
      "train_loss": 0.6738453804676539,
      "train_runtime": 49317.7491,
      "train_samples_per_second": 3.769,
      "train_steps_per_second": 0.039
    }
  ],
  "logging_steps": 10,
  "max_steps": 1935,
  "num_train_epochs": 3,
  "save_steps": 500,
  "total_flos": 0.0,
  "trial_name": null,
  "trial_params": null
}