{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 8335,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 5.995203836930456e-09,
      "logits/chosen": -2.424614667892456,
      "logits/rejected": -1.9891018867492676,
      "logps/chosen": -441.5737609863281,
      "logps/rejected": -473.3967590332031,
      "loss": 0.1361,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.0,
      "learning_rate": 5.995203836930456e-08,
      "logits/chosen": -2.110199213027954,
      "logits/rejected": -1.765876054763794,
      "logps/chosen": -209.27218627929688,
      "logps/rejected": -153.5750274658203,
      "loss": 0.2066,
      "rewards/accuracies": 0.3888888955116272,
      "rewards/chosen": 0.0005430497694760561,
      "rewards/margins": 0.0006039439467713237,
      "rewards/rejected": -6.089422822697088e-05,
      "step": 10
    },
    {
      "epoch": 0.0,
      "learning_rate": 1.1990407673860913e-07,
      "logits/chosen": -1.9729121923446655,
      "logits/rejected": -1.6711788177490234,
      "logps/chosen": -187.25914001464844,
      "logps/rejected": -146.9638671875,
      "loss": 0.1876,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.00047091051237657666,
      "rewards/margins": -0.0006188965635374188,
      "rewards/rejected": 0.00014798599295318127,
      "step": 20
    },
    {
      "epoch": 0.0,
      "learning_rate": 1.7985611510791368e-07,
      "logits/chosen": -2.093867063522339,
      "logits/rejected": -1.7798885107040405,
      "logps/chosen": -271.8372802734375,
      "logps/rejected": -197.7427978515625,
      "loss": 0.161,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0006424393504858017,
      "rewards/margins": 0.0006549443351104856,
      "rewards/rejected": -1.2505089216574561e-05,
      "step": 30
    },
    {
      "epoch": 0.0,
      "learning_rate": 2.3980815347721825e-07,
      "logits/chosen": -1.8111674785614014,
      "logits/rejected": -1.651614785194397,
      "logps/chosen": -180.64151000976562,
      "logps/rejected": -205.8025360107422,
      "loss": 0.1737,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0005901036784052849,
      "rewards/margins": 0.0005529513582587242,
      "rewards/rejected": 3.7152261938899755e-05,
      "step": 40
    },
    {
      "epoch": 0.01,
      "learning_rate": 2.997601918465228e-07,
      "logits/chosen": -2.04856538772583,
      "logits/rejected": -1.7901275157928467,
      "logps/chosen": -215.7578582763672,
      "logps/rejected": -220.8831024169922,
      "loss": 0.2306,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.00023345758381765336,
      "rewards/margins": 8.568236808059737e-05,
      "rewards/rejected": -0.00031913991551846266,
      "step": 50
    },
    {
      "epoch": 0.01,
      "learning_rate": 3.5971223021582736e-07,
      "logits/chosen": -2.021206855773926,
      "logits/rejected": -1.532591462135315,
      "logps/chosen": -217.2874298095703,
      "logps/rejected": -155.38461303710938,
      "loss": 0.1551,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0003670873702503741,
      "rewards/margins": 0.004133955575525761,
      "rewards/rejected": -0.004501043353229761,
      "step": 60
    },
    {
      "epoch": 0.01,
      "learning_rate": 4.1966426858513196e-07,
      "logits/chosen": -2.06149959564209,
      "logits/rejected": -1.6334540843963623,
      "logps/chosen": -219.23593139648438,
      "logps/rejected": -171.34017944335938,
      "loss": 0.1949,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.0024479026906192303,
      "rewards/margins": 0.001299393828958273,
      "rewards/rejected": -0.0037472962867468596,
      "step": 70
    },
    {
      "epoch": 0.01,
      "learning_rate": 4.796163069544365e-07,
      "logits/chosen": -2.0656137466430664,
      "logits/rejected": -1.6331923007965088,
      "logps/chosen": -287.973876953125,
      "logps/rejected": -253.8162384033203,
      "loss": 0.1885,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.006006647367030382,
      "rewards/margins": 0.0021041277796030045,
      "rewards/rejected": -0.0081107746809721,
      "step": 80
    },
    {
      "epoch": 0.01,
      "learning_rate": 5.39568345323741e-07,
      "logits/chosen": -2.0038671493530273,
      "logits/rejected": -1.5671374797821045,
      "logps/chosen": -227.6986083984375,
      "logps/rejected": -173.5839385986328,
      "loss": 0.1738,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.004840956535190344,
      "rewards/margins": 0.010292068123817444,
      "rewards/rejected": -0.015133025124669075,
      "step": 90
    },
    {
      "epoch": 0.01,
      "learning_rate": 5.995203836930456e-07,
      "logits/chosen": -1.8383821249008179,
      "logits/rejected": -1.9401063919067383,
      "logps/chosen": -162.48382568359375,
      "logps/rejected": -229.74935913085938,
      "loss": 0.1897,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.013102886267006397,
      "rewards/margins": 0.002711429027840495,
      "rewards/rejected": -0.015814315527677536,
      "step": 100
    },
    {
      "epoch": 0.01,
      "learning_rate": 6.594724220623502e-07,
      "logits/chosen": -2.0046496391296387,
      "logits/rejected": -1.519207239151001,
      "logps/chosen": -167.69444274902344,
      "logps/rejected": -131.64898681640625,
      "loss": 0.2673,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02011437714099884,
      "rewards/margins": 0.024061836302280426,
      "rewards/rejected": -0.04417620971798897,
      "step": 110
    },
    {
      "epoch": 0.01,
      "learning_rate": 7.194244604316547e-07,
      "logits/chosen": -2.0810534954071045,
      "logits/rejected": -1.8244158029556274,
      "logps/chosen": -236.47250366210938,
      "logps/rejected": -229.98471069335938,
      "loss": 0.2358,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.011348506435751915,
      "rewards/margins": 0.03258121386170387,
      "rewards/rejected": -0.04392971843481064,
      "step": 120
    },
    {
      "epoch": 0.02,
      "learning_rate": 7.793764988009593e-07,
      "logits/chosen": -1.749682068824768,
      "logits/rejected": -1.4685245752334595,
      "logps/chosen": -212.50454711914062,
      "logps/rejected": -210.17977905273438,
      "loss": 0.2316,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.017982326447963715,
      "rewards/margins": 0.043081801384687424,
      "rewards/rejected": -0.06106413155794144,
      "step": 130
    },
    {
      "epoch": 0.02,
      "learning_rate": 8.393285371702639e-07,
      "logits/chosen": -1.8063485622406006,
      "logits/rejected": -1.7413132190704346,
      "logps/chosen": -159.96786499023438,
      "logps/rejected": -218.73233032226562,
      "loss": 0.1663,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.042347490787506104,
      "rewards/margins": 0.051009368151426315,
      "rewards/rejected": -0.09335686266422272,
      "step": 140
    },
    {
      "epoch": 0.02,
      "learning_rate": 8.992805755395684e-07,
      "logits/chosen": -1.974597692489624,
      "logits/rejected": -1.5300289392471313,
      "logps/chosen": -221.36166381835938,
      "logps/rejected": -209.5833282470703,
      "loss": 0.1912,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08563482016324997,
      "rewards/margins": 0.06949006021022797,
      "rewards/rejected": -0.15512490272521973,
      "step": 150
    },
    {
      "epoch": 0.02,
      "learning_rate": 9.59232613908873e-07,
      "logits/chosen": -1.9377422332763672,
      "logits/rejected": -1.4812471866607666,
      "logps/chosen": -214.61581420898438,
      "logps/rejected": -158.63467407226562,
      "loss": 0.19,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.19673360884189606,
      "rewards/margins": 0.11003688722848892,
      "rewards/rejected": -0.3067705035209656,
      "step": 160
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.0191846522781776e-06,
      "logits/chosen": -1.8835103511810303,
      "logits/rejected": -1.6908140182495117,
      "logps/chosen": -232.00540161132812,
      "logps/rejected": -224.3048095703125,
      "loss": 0.1984,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.32456761598587036,
      "rewards/margins": 0.03511672466993332,
      "rewards/rejected": -0.35968437790870667,
      "step": 170
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.079136690647482e-06,
      "logits/chosen": -2.0397212505340576,
      "logits/rejected": -1.7811028957366943,
      "logps/chosen": -209.57711791992188,
      "logps/rejected": -219.6051483154297,
      "loss": 0.1525,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.20149996876716614,
      "rewards/margins": 0.10627492517232895,
      "rewards/rejected": -0.3077749013900757,
      "step": 180
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.1390887290167866e-06,
      "logits/chosen": -2.0456180572509766,
      "logits/rejected": -1.645538091659546,
      "logps/chosen": -298.3092346191406,
      "logps/rejected": -260.2908020019531,
      "loss": 0.0944,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1598762422800064,
      "rewards/margins": 0.14658963680267334,
      "rewards/rejected": -0.30646592378616333,
      "step": 190
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.1990407673860912e-06,
      "logits/chosen": -2.1075217723846436,
      "logits/rejected": -1.8570976257324219,
      "logps/chosen": -279.629638671875,
      "logps/rejected": -273.2984924316406,
      "loss": 0.171,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.15068146586418152,
      "rewards/margins": 0.04055650904774666,
      "rewards/rejected": -0.19123797118663788,
      "step": 200
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.2589928057553958e-06,
      "logits/chosen": -2.0482380390167236,
      "logits/rejected": -1.786058783531189,
      "logps/chosen": -206.5548858642578,
      "logps/rejected": -211.97860717773438,
      "loss": 0.1609,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.10160304605960846,
      "rewards/margins": 0.0669463574886322,
      "rewards/rejected": -0.16854938864707947,
      "step": 210
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.3189448441247004e-06,
      "logits/chosen": -2.084348678588867,
      "logits/rejected": -1.6052637100219727,
      "logps/chosen": -251.0631866455078,
      "logps/rejected": -206.6727752685547,
      "loss": 0.1085,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13653235137462616,
      "rewards/margins": 0.06996998935937881,
      "rewards/rejected": -0.20650234818458557,
      "step": 220
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.378896882494005e-06,
      "logits/chosen": -2.140784502029419,
      "logits/rejected": -1.4730150699615479,
      "logps/chosen": -233.042236328125,
      "logps/rejected": -207.8023681640625,
      "loss": 0.1887,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.13673333823680878,
      "rewards/margins": 0.11993058770895004,
      "rewards/rejected": -0.25666388869285583,
      "step": 230
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.4388489208633094e-06,
      "logits/chosen": -1.9343608617782593,
      "logits/rejected": -1.6981518268585205,
      "logps/chosen": -254.0943145751953,
      "logps/rejected": -275.8316650390625,
      "loss": 0.1273,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13066630065441132,
      "rewards/margins": 0.07596530020236969,
      "rewards/rejected": -0.2066315859556198,
      "step": 240
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.4988009592326142e-06,
      "logits/chosen": -1.952183485031128,
      "logits/rejected": -1.8142400979995728,
      "logps/chosen": -236.6318359375,
      "logps/rejected": -268.00537109375,
      "loss": 0.17,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.22585809230804443,
      "rewards/margins": 0.0762966051697731,
      "rewards/rejected": -0.30215469002723694,
      "step": 250
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.5587529976019186e-06,
      "logits/chosen": -1.9224618673324585,
      "logits/rejected": -1.6340528726577759,
      "logps/chosen": -196.9944305419922,
      "logps/rejected": -205.39712524414062,
      "loss": 0.2002,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.2947540879249573,
      "rewards/margins": 0.09202824532985687,
      "rewards/rejected": -0.38678231835365295,
      "step": 260
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.618705035971223e-06,
      "logits/chosen": -1.6605432033538818,
      "logits/rejected": -1.3576246500015259,
      "logps/chosen": -190.0495147705078,
      "logps/rejected": -203.37741088867188,
      "loss": 0.1713,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.2563242018222809,
      "rewards/margins": 0.13959848880767822,
      "rewards/rejected": -0.3959227204322815,
      "step": 270
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.6786570743405278e-06,
      "logits/chosen": -2.081996440887451,
      "logits/rejected": -1.8335282802581787,
      "logps/chosen": -206.75241088867188,
      "logps/rejected": -198.82595825195312,
      "loss": 0.1588,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.2610263228416443,
      "rewards/margins": 0.1122390478849411,
      "rewards/rejected": -0.3732653856277466,
      "step": 280
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.7386091127098322e-06,
      "logits/chosen": -2.127209186553955,
      "logits/rejected": -1.662936806678772,
      "logps/chosen": -278.27166748046875,
      "logps/rejected": -271.78399658203125,
      "loss": 0.1299,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.3486752510070801,
      "rewards/margins": 0.12807399034500122,
      "rewards/rejected": -0.4767492413520813,
      "step": 290
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.7985611510791368e-06,
      "logits/chosen": -1.844478964805603,
      "logits/rejected": -1.6861326694488525,
      "logps/chosen": -323.74169921875,
      "logps/rejected": -350.01019287109375,
      "loss": 0.154,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4493889808654785,
      "rewards/margins": 0.10103818029165268,
      "rewards/rejected": -0.5504271388053894,
      "step": 300
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.8585131894484414e-06,
      "logits/chosen": -2.01519513130188,
      "logits/rejected": -1.7900644540786743,
      "logps/chosen": -268.0411071777344,
      "logps/rejected": -255.30453491210938,
      "loss": 0.1899,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4563392698764801,
      "rewards/margins": 0.09228341281414032,
      "rewards/rejected": -0.5486227869987488,
      "step": 310
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.918465227817746e-06,
      "logits/chosen": -1.924393653869629,
      "logits/rejected": -1.5130094289779663,
      "logps/chosen": -186.55593872070312,
      "logps/rejected": -182.58775329589844,
      "loss": 0.166,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.4110238552093506,
      "rewards/margins": 0.16674764454364777,
      "rewards/rejected": -0.5777715444564819,
      "step": 320
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.9784172661870504e-06,
      "logits/chosen": -2.1037185192108154,
      "logits/rejected": -1.9657186269760132,
      "logps/chosen": -218.725830078125,
      "logps/rejected": -252.91024780273438,
      "loss": 0.1719,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.35161924362182617,
      "rewards/margins": 0.0837181806564331,
      "rewards/rejected": -0.4353373944759369,
      "step": 330
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.0383693045563552e-06,
      "logits/chosen": -2.0712966918945312,
      "logits/rejected": -1.9187800884246826,
      "logps/chosen": -201.36724853515625,
      "logps/rejected": -223.52041625976562,
      "loss": 0.1685,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.2922331988811493,
      "rewards/margins": 0.11911450326442719,
      "rewards/rejected": -0.41134771704673767,
      "step": 340
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.0983213429256596e-06,
      "logits/chosen": -1.9400737285614014,
      "logits/rejected": -1.7396290302276611,
      "logps/chosen": -189.12892150878906,
      "logps/rejected": -177.9490509033203,
      "loss": 0.2454,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.25958988070487976,
      "rewards/margins": 0.03190717101097107,
      "rewards/rejected": -0.29149705171585083,
      "step": 350
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.158273381294964e-06,
      "logits/chosen": -2.010946035385132,
      "logits/rejected": -1.7258754968643188,
      "logps/chosen": -230.20706176757812,
      "logps/rejected": -224.4259033203125,
      "loss": 0.1737,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.28419947624206543,
      "rewards/margins": 0.1424468755722046,
      "rewards/rejected": -0.4266463816165924,
      "step": 360
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.218225419664269e-06,
      "logits/chosen": -1.7381842136383057,
      "logits/rejected": -1.4838992357254028,
      "logps/chosen": -234.2359619140625,
      "logps/rejected": -248.5423583984375,
      "loss": 0.0967,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.3407961428165436,
      "rewards/margins": 0.10874161869287491,
      "rewards/rejected": -0.4495377540588379,
      "step": 370
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.2781774580335732e-06,
      "logits/chosen": -2.022749423980713,
      "logits/rejected": -1.4032447338104248,
      "logps/chosen": -256.8328552246094,
      "logps/rejected": -192.03329467773438,
      "loss": 0.1595,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2618308663368225,
      "rewards/margins": 0.14256241917610168,
      "rewards/rejected": -0.4043932855129242,
      "step": 380
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.3381294964028776e-06,
      "logits/chosen": -1.7514533996582031,
      "logits/rejected": -1.6247230768203735,
      "logps/chosen": -214.0389862060547,
      "logps/rejected": -226.91531372070312,
      "loss": 0.1191,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.2733401656150818,
      "rewards/margins": 0.11437875032424927,
      "rewards/rejected": -0.38771897554397583,
      "step": 390
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.3980815347721824e-06,
      "logits/chosen": -1.9063535928726196,
      "logits/rejected": -1.5726300477981567,
      "logps/chosen": -222.97158813476562,
      "logps/rejected": -203.55575561523438,
      "loss": 0.15,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.2823329567909241,
      "rewards/margins": 0.119574174284935,
      "rewards/rejected": -0.40190714597702026,
      "step": 400
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.458033573141487e-06,
      "logits/chosen": -1.9088506698608398,
      "logits/rejected": -1.5609016418457031,
      "logps/chosen": -262.8750915527344,
      "logps/rejected": -279.5753479003906,
      "loss": 0.1665,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.3804900348186493,
      "rewards/margins": 0.17719073593616486,
      "rewards/rejected": -0.5576807856559753,
      "step": 410
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.5179856115107916e-06,
      "logits/chosen": -1.9670276641845703,
      "logits/rejected": -1.5786240100860596,
      "logps/chosen": -247.9984588623047,
      "logps/rejected": -261.22015380859375,
      "loss": 0.103,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.306973397731781,
      "rewards/margins": 0.17263731360435486,
      "rewards/rejected": -0.4796106815338135,
      "step": 420
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.577937649880096e-06,
      "logits/chosen": -1.9811060428619385,
      "logits/rejected": -1.8827531337738037,
      "logps/chosen": -216.9333953857422,
      "logps/rejected": -262.21697998046875,
      "loss": 0.2221,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.36211085319519043,
      "rewards/margins": 0.09959669411182404,
      "rewards/rejected": -0.46170753240585327,
      "step": 430
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.637889688249401e-06,
      "logits/chosen": -2.250277042388916,
      "logits/rejected": -1.69893479347229,
      "logps/chosen": -390.80206298828125,
      "logps/rejected": -284.90594482421875,
      "loss": 0.1114,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.28247594833374023,
      "rewards/margins": 0.12760603427886963,
      "rewards/rejected": -0.4100819528102875,
      "step": 440
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.6978417266187052e-06,
      "logits/chosen": -1.8655027151107788,
      "logits/rejected": -1.5759456157684326,
      "logps/chosen": -273.0975036621094,
      "logps/rejected": -323.8568420410156,
      "loss": 0.1357,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.33328741788864136,
      "rewards/margins": 0.1260485053062439,
      "rewards/rejected": -0.45933595299720764,
      "step": 450
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.75779376498801e-06,
      "logits/chosen": -2.09033203125,
      "logits/rejected": -1.8335535526275635,
      "logps/chosen": -276.64398193359375,
      "logps/rejected": -283.84088134765625,
      "loss": 0.1603,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.2383461743593216,
      "rewards/margins": 0.10929499566555023,
      "rewards/rejected": -0.3476411700248718,
      "step": 460
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.8177458033573145e-06,
      "logits/chosen": -1.972602128982544,
      "logits/rejected": -1.670013427734375,
      "logps/chosen": -201.99130249023438,
      "logps/rejected": -196.4750518798828,
      "loss": 0.1939,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.2819952368736267,
      "rewards/margins": 0.0845649391412735,
      "rewards/rejected": -0.3665602207183838,
      "step": 470
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.877697841726619e-06,
      "logits/chosen": -1.984262228012085,
      "logits/rejected": -1.9005893468856812,
      "logps/chosen": -280.5291442871094,
      "logps/rejected": -247.97720336914062,
      "loss": 0.1729,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.35804125666618347,
      "rewards/margins": 0.015171018429100513,
      "rewards/rejected": -0.3732122778892517,
      "step": 480
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.9376498800959237e-06,
      "logits/chosen": -2.09686541557312,
      "logits/rejected": -1.6708438396453857,
      "logps/chosen": -238.1042022705078,
      "logps/rejected": -228.37350463867188,
      "loss": 0.1234,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.3339739739894867,
      "rewards/margins": 0.13829635083675385,
      "rewards/rejected": -0.47227030992507935,
      "step": 490
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.9976019184652285e-06,
      "logits/chosen": -1.8362640142440796,
      "logits/rejected": -1.5808006525039673,
      "logps/chosen": -247.06204223632812,
      "logps/rejected": -234.0608673095703,
      "loss": 0.1674,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.24948985874652863,
      "rewards/margins": 0.061165668070316315,
      "rewards/rejected": -0.31065553426742554,
      "step": 500
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.0575539568345324e-06,
      "logits/chosen": -2.108161449432373,
      "logits/rejected": -1.9195600748062134,
      "logps/chosen": -204.02523803710938,
      "logps/rejected": -211.36361694335938,
      "loss": 0.1593,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.23356959223747253,
      "rewards/margins": 0.07123871147632599,
      "rewards/rejected": -0.3048083186149597,
      "step": 510
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.1175059952038373e-06,
      "logits/chosen": -1.8966169357299805,
      "logits/rejected": -1.4281069040298462,
      "logps/chosen": -257.47454833984375,
      "logps/rejected": -211.0913543701172,
      "loss": 0.1349,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2831900119781494,
      "rewards/margins": 0.1322624385356903,
      "rewards/rejected": -0.41545242071151733,
      "step": 520
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.177458033573142e-06,
      "logits/chosen": -2.059906482696533,
      "logits/rejected": -1.6643825769424438,
      "logps/chosen": -228.86172485351562,
      "logps/rejected": -192.63961791992188,
      "loss": 0.1881,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.3060818314552307,
      "rewards/margins": 0.1667357236146927,
      "rewards/rejected": -0.4728175103664398,
      "step": 530
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.237410071942446e-06,
      "logits/chosen": -1.9420620203018188,
      "logits/rejected": -1.6259702444076538,
      "logps/chosen": -239.92581176757812,
      "logps/rejected": -244.3850555419922,
      "loss": 0.1508,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.26758164167404175,
      "rewards/margins": 0.09906923025846481,
      "rewards/rejected": -0.36665090918540955,
      "step": 540
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.297362110311751e-06,
      "logits/chosen": -2.0125441551208496,
      "logits/rejected": -1.77133047580719,
      "logps/chosen": -234.72738647460938,
      "logps/rejected": -229.4871368408203,
      "loss": 0.0946,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.36053597927093506,
      "rewards/margins": 0.1191805824637413,
      "rewards/rejected": -0.47971653938293457,
      "step": 550
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.3573141486810557e-06,
      "logits/chosen": -1.9631010293960571,
      "logits/rejected": -1.7263110876083374,
      "logps/chosen": -264.1136169433594,
      "logps/rejected": -245.3027801513672,
      "loss": 0.1203,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.26941028237342834,
      "rewards/margins": 0.09951646625995636,
      "rewards/rejected": -0.3689267337322235,
      "step": 560
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.4172661870503596e-06,
      "logits/chosen": -2.090639114379883,
      "logits/rejected": -1.617297887802124,
      "logps/chosen": -253.80575561523438,
      "logps/rejected": -249.5729217529297,
      "loss": 0.1359,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.16714581847190857,
      "rewards/margins": 0.15788118541240692,
      "rewards/rejected": -0.3250270485877991,
      "step": 570
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.4772182254196645e-06,
      "logits/chosen": -2.1856539249420166,
      "logits/rejected": -1.682244896888733,
      "logps/chosen": -291.01373291015625,
      "logps/rejected": -219.6407470703125,
      "loss": 0.2091,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.12654975056648254,
      "rewards/margins": 0.10359915345907211,
      "rewards/rejected": -0.23014888167381287,
      "step": 580
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.5371702637889693e-06,
      "logits/chosen": -1.8060592412948608,
      "logits/rejected": -1.5262947082519531,
      "logps/chosen": -242.6085968017578,
      "logps/rejected": -213.41592407226562,
      "loss": 0.1047,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.306922972202301,
      "rewards/margins": 0.15780650079250336,
      "rewards/rejected": -0.4647294580936432,
      "step": 590
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.5971223021582737e-06,
      "logits/chosen": -1.937294602394104,
      "logits/rejected": -1.5673654079437256,
      "logps/chosen": -255.3214874267578,
      "logps/rejected": -233.734619140625,
      "loss": 0.1647,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5621557831764221,
      "rewards/margins": 0.1001255139708519,
      "rewards/rejected": -0.6622812747955322,
      "step": 600
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.657074340527578e-06,
      "logits/chosen": -1.9981091022491455,
      "logits/rejected": -1.8456417322158813,
      "logps/chosen": -284.6530456542969,
      "logps/rejected": -266.3650207519531,
      "loss": 0.1632,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5589288473129272,
      "rewards/margins": 0.10421963781118393,
      "rewards/rejected": -0.6631485223770142,
      "step": 610
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.717026378896883e-06,
      "logits/chosen": -1.7741851806640625,
      "logits/rejected": -1.5398646593093872,
      "logps/chosen": -253.56478881835938,
      "logps/rejected": -286.1531066894531,
      "loss": 0.1523,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.6353442072868347,
      "rewards/margins": 0.179083913564682,
      "rewards/rejected": -0.8144281506538391,
      "step": 620
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.7769784172661873e-06,
      "logits/chosen": -1.7653591632843018,
      "logits/rejected": -1.5256303548812866,
      "logps/chosen": -309.58636474609375,
      "logps/rejected": -303.3143615722656,
      "loss": 0.0848,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.5480080842971802,
      "rewards/margins": 0.16620375216007233,
      "rewards/rejected": -0.7142117023468018,
      "step": 630
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.836930455635492e-06,
      "logits/chosen": -1.8177156448364258,
      "logits/rejected": -1.576643705368042,
      "logps/chosen": -297.94183349609375,
      "logps/rejected": -282.4010314941406,
      "loss": 0.1422,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5166171193122864,
      "rewards/margins": 0.1333894580602646,
      "rewards/rejected": -0.6500065922737122,
      "step": 640
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.896882494004797e-06,
      "logits/chosen": -1.9186363220214844,
      "logits/rejected": -1.5935009717941284,
      "logps/chosen": -284.55377197265625,
      "logps/rejected": -241.8046875,
      "loss": 0.1217,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.34077686071395874,
      "rewards/margins": 0.04739413410425186,
      "rewards/rejected": -0.38817098736763,
      "step": 650
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.956834532374101e-06,
      "logits/chosen": -1.8101074695587158,
      "logits/rejected": -1.6274917125701904,
      "logps/chosen": -198.5729522705078,
      "logps/rejected": -253.2472686767578,
      "loss": 0.1605,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.3132355511188507,
      "rewards/margins": 0.13122674822807312,
      "rewards/rejected": -0.44446223974227905,
      "step": 660
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.016786570743406e-06,
      "logits/chosen": -1.9853845834732056,
      "logits/rejected": -1.614189863204956,
      "logps/chosen": -191.61289978027344,
      "logps/rejected": -172.0723114013672,
      "loss": 0.0808,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.32061904668807983,
      "rewards/margins": 0.17392602562904358,
      "rewards/rejected": -0.4945450723171234,
      "step": 670
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.0767386091127105e-06,
      "logits/chosen": -1.895282506942749,
      "logits/rejected": -1.4025895595550537,
      "logps/chosen": -287.64459228515625,
      "logps/rejected": -237.231201171875,
      "loss": 0.1305,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.34143179655075073,
      "rewards/margins": 0.1731572449207306,
      "rewards/rejected": -0.5145890116691589,
      "step": 680
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.1366906474820145e-06,
      "logits/chosen": -1.9859917163848877,
      "logits/rejected": -1.811034917831421,
      "logps/chosen": -267.02532958984375,
      "logps/rejected": -248.8046417236328,
      "loss": 0.1599,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.4106379449367523,
      "rewards/margins": 0.09379793703556061,
      "rewards/rejected": -0.5044358968734741,
      "step": 690
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.196642685851319e-06,
      "logits/chosen": -2.0239920616149902,
      "logits/rejected": -1.7091875076293945,
      "logps/chosen": -163.7296600341797,
      "logps/rejected": -176.8492431640625,
      "loss": 0.1985,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.22491566836833954,
      "rewards/margins": 0.10180320590734482,
      "rewards/rejected": -0.32671886682510376,
      "step": 700
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.256594724220624e-06,
      "logits/chosen": -2.042534828186035,
      "logits/rejected": -1.7031242847442627,
      "logps/chosen": -243.00247192382812,
      "logps/rejected": -238.3159637451172,
      "loss": 0.1317,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.22081628441810608,
      "rewards/margins": 0.13557776808738708,
      "rewards/rejected": -0.3563940227031708,
      "step": 710
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.316546762589928e-06,
      "logits/chosen": -2.0568203926086426,
      "logits/rejected": -1.7259998321533203,
      "logps/chosen": -263.7857666015625,
      "logps/rejected": -256.0942077636719,
      "loss": 0.0914,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.3193433880805969,
      "rewards/margins": 0.14199210703372955,
      "rewards/rejected": -0.4613354802131653,
      "step": 720
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.376498800959233e-06,
      "logits/chosen": -1.8908824920654297,
      "logits/rejected": -1.599169135093689,
      "logps/chosen": -250.081787109375,
      "logps/rejected": -221.8983917236328,
      "loss": 0.176,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.42117080092430115,
      "rewards/margins": 0.1439433991909027,
      "rewards/rejected": -0.5651142001152039,
      "step": 730
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.436450839328538e-06,
      "logits/chosen": -1.81271231174469,
      "logits/rejected": -1.4758561849594116,
      "logps/chosen": -237.4412841796875,
      "logps/rejected": -207.1601104736328,
      "loss": 0.189,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.4822634756565094,
      "rewards/margins": 0.16576936841011047,
      "rewards/rejected": -0.6480327844619751,
      "step": 740
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.496402877697842e-06,
      "logits/chosen": -2.2231035232543945,
      "logits/rejected": -1.774987816810608,
      "logps/chosen": -291.08026123046875,
      "logps/rejected": -247.5023193359375,
      "loss": 0.09,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4788135886192322,
      "rewards/margins": 0.17467689514160156,
      "rewards/rejected": -0.6534904837608337,
      "step": 750
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.5563549160671465e-06,
      "logits/chosen": -1.9474821090698242,
      "logits/rejected": -1.523559808731079,
      "logps/chosen": -256.2854919433594,
      "logps/rejected": -199.56689453125,
      "loss": 0.2096,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.525553286075592,
      "rewards/margins": 0.10277509689331055,
      "rewards/rejected": -0.6283284425735474,
      "step": 760
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.616306954436451e-06,
      "logits/chosen": -1.9608790874481201,
      "logits/rejected": -1.600482702255249,
      "logps/chosen": -234.0125274658203,
      "logps/rejected": -219.60494995117188,
      "loss": 0.1606,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.560684084892273,
      "rewards/margins": 0.11568351089954376,
      "rewards/rejected": -0.6763675808906555,
      "step": 770
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.676258992805755e-06,
      "logits/chosen": -1.9117473363876343,
      "logits/rejected": -1.6412442922592163,
      "logps/chosen": -225.1289520263672,
      "logps/rejected": -219.20339965820312,
      "loss": 0.1631,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.4210142195224762,
      "rewards/margins": 0.12721626460552216,
      "rewards/rejected": -0.5482303500175476,
      "step": 780
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.73621103117506e-06,
      "logits/chosen": -1.8688926696777344,
      "logits/rejected": -1.7802883386611938,
      "logps/chosen": -220.6570281982422,
      "logps/rejected": -236.92758178710938,
      "loss": 0.1411,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.46808117628097534,
      "rewards/margins": 0.0750129297375679,
      "rewards/rejected": -0.5430941581726074,
      "step": 790
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.796163069544365e-06,
      "logits/chosen": -2.0156643390655518,
      "logits/rejected": -1.6202924251556396,
      "logps/chosen": -296.9110412597656,
      "logps/rejected": -225.12802124023438,
      "loss": 0.1625,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5204383730888367,
      "rewards/margins": 0.12295063585042953,
      "rewards/rejected": -0.6433890461921692,
      "step": 800
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.856115107913669e-06,
      "logits/chosen": -1.9585834741592407,
      "logits/rejected": -1.6052249670028687,
      "logps/chosen": -224.037353515625,
      "logps/rejected": -237.3111572265625,
      "loss": 0.1651,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.4183143675327301,
      "rewards/margins": 0.10342450439929962,
      "rewards/rejected": -0.5217388868331909,
      "step": 810
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.916067146282974e-06,
      "logits/chosen": -2.0472521781921387,
      "logits/rejected": -1.5536268949508667,
      "logps/chosen": -236.7208251953125,
      "logps/rejected": -195.6331787109375,
      "loss": 0.137,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.36916661262512207,
      "rewards/margins": 0.11299363523721695,
      "rewards/rejected": -0.4821602702140808,
      "step": 820
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.9760191846522785e-06,
      "logits/chosen": -1.9616940021514893,
      "logits/rejected": -1.7898718118667603,
      "logps/chosen": -208.87741088867188,
      "logps/rejected": -228.8385467529297,
      "loss": 0.1618,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3800353407859802,
      "rewards/margins": 0.12817123532295227,
      "rewards/rejected": -0.5082066059112549,
      "step": 830
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.9999921064257284e-06,
      "logits/chosen": -1.8127410411834717,
      "logits/rejected": -1.3970979452133179,
      "logps/chosen": -278.39019775390625,
      "logps/rejected": -293.3469543457031,
      "loss": 0.1529,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.4711076319217682,
      "rewards/margins": 0.15813672542572021,
      "rewards/rejected": -0.629244327545166,
      "step": 840
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.9999438680968e-06,
      "logits/chosen": -1.7962703704833984,
      "logits/rejected": -1.365013837814331,
      "logps/chosen": -249.14828491210938,
      "logps/rejected": -232.2996063232422,
      "loss": 0.1558,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.6188076734542847,
      "rewards/margins": 0.18656638264656067,
      "rewards/rejected": -0.8053741455078125,
      "step": 850
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999851777603122e-06,
      "logits/chosen": -1.9362220764160156,
      "logits/rejected": -1.697091817855835,
      "logps/chosen": -292.0437927246094,
      "logps/rejected": -294.3341369628906,
      "loss": 0.1133,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5470017194747925,
      "rewards/margins": 0.12919080257415771,
      "rewards/rejected": -0.6761925220489502,
      "step": 860
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999715836560074e-06,
      "logits/chosen": -1.907859206199646,
      "logits/rejected": -1.5174684524536133,
      "logps/chosen": -218.5150909423828,
      "logps/rejected": -220.261474609375,
      "loss": 0.2033,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.3629061281681061,
      "rewards/margins": 0.12141053378582001,
      "rewards/rejected": -0.4843166768550873,
      "step": 870
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.999536047352236e-06,
      "logits/chosen": -1.909115195274353,
      "logits/rejected": -1.6440328359603882,
      "logps/chosen": -206.3199005126953,
      "logps/rejected": -194.60552978515625,
      "loss": 0.1926,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.3590332567691803,
      "rewards/margins": 0.04666576534509659,
      "rewards/rejected": -0.4056990146636963,
      "step": 880
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.999312413133335e-06,
      "logits/chosen": -2.02655029296875,
      "logits/rejected": -1.555143117904663,
      "logps/chosen": -290.0566711425781,
      "logps/rejected": -258.43951416015625,
      "loss": 0.1552,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.21334341168403625,
      "rewards/margins": 0.13854533433914185,
      "rewards/rejected": -0.3518887460231781,
      "step": 890
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.999044937826198e-06,
      "logits/chosen": -1.7870111465454102,
      "logits/rejected": -1.3078781366348267,
      "logps/chosen": -232.5644073486328,
      "logps/rejected": -228.1163330078125,
      "loss": 0.167,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.22880849242210388,
      "rewards/margins": 0.1350013017654419,
      "rewards/rejected": -0.36380982398986816,
      "step": 900
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.998733626122679e-06,
      "logits/chosen": -1.9320144653320312,
      "logits/rejected": -1.7529146671295166,
      "logps/chosen": -230.58480834960938,
      "logps/rejected": -215.9065399169922,
      "loss": 0.1188,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.2430393248796463,
      "rewards/margins": 0.10922437906265259,
      "rewards/rejected": -0.3522637188434601,
      "step": 910
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.998378483483577e-06,
      "logits/chosen": -2.0543458461761475,
      "logits/rejected": -1.576468586921692,
      "logps/chosen": -215.76681518554688,
      "logps/rejected": -143.02210998535156,
      "loss": 0.1665,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.19551251828670502,
      "rewards/margins": 0.15126529335975647,
      "rewards/rejected": -0.3467778265476227,
      "step": 920
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.997979516138542e-06,
      "logits/chosen": -1.7736568450927734,
      "logits/rejected": -1.427062749862671,
      "logps/chosen": -193.51675415039062,
      "logps/rejected": -190.2722625732422,
      "loss": 0.1579,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.2125972956418991,
      "rewards/margins": 0.12206624448299408,
      "rewards/rejected": -0.3346635401248932,
      "step": 930
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.997536731085962e-06,
      "logits/chosen": -2.076658248901367,
      "logits/rejected": -1.6957495212554932,
      "logps/chosen": -275.86004638671875,
      "logps/rejected": -257.83856201171875,
      "loss": 0.1156,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.15670546889305115,
      "rewards/margins": 0.1267530769109726,
      "rewards/rejected": -0.28345853090286255,
      "step": 940
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.997050136092847e-06,
      "logits/chosen": -1.9965837001800537,
      "logits/rejected": -1.5267733335494995,
      "logps/chosen": -234.33804321289062,
      "logps/rejected": -168.39871215820312,
      "loss": 0.1527,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.16364577412605286,
      "rewards/margins": 0.10411280393600464,
      "rewards/rejected": -0.2677585780620575,
      "step": 950
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.996519739694684e-06,
      "logits/chosen": -1.9764916896820068,
      "logits/rejected": -1.8170015811920166,
      "logps/chosen": -261.2804260253906,
      "logps/rejected": -244.72708129882812,
      "loss": 0.1467,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2282881736755371,
      "rewards/margins": 0.0751405730843544,
      "rewards/rejected": -0.3034287393093109,
      "step": 960
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.995945551195296e-06,
      "logits/chosen": -1.8942501544952393,
      "logits/rejected": -1.576650857925415,
      "logps/chosen": -216.11184692382812,
      "logps/rejected": -192.7135467529297,
      "loss": 0.1492,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2776258587837219,
      "rewards/margins": 0.18120935559272766,
      "rewards/rejected": -0.4588352143764496,
      "step": 970
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.995327580666672e-06,
      "logits/chosen": -2.1704368591308594,
      "logits/rejected": -1.4965741634368896,
      "logps/chosen": -241.30697631835938,
      "logps/rejected": -190.32650756835938,
      "loss": 0.1123,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.27652624249458313,
      "rewards/margins": 0.24555762112140656,
      "rewards/rejected": -0.5220838785171509,
      "step": 980
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.994665838948792e-06,
      "logits/chosen": -1.9886589050292969,
      "logits/rejected": -1.8707554340362549,
      "logps/chosen": -235.10440063476562,
      "logps/rejected": -289.9771423339844,
      "loss": 0.1256,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1920485645532608,
      "rewards/margins": 0.11377612501382828,
      "rewards/rejected": -0.3058246970176697,
      "step": 990
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.993960337649441e-06,
      "logits/chosen": -2.0039737224578857,
      "logits/rejected": -1.454641342163086,
      "logps/chosen": -253.69454956054688,
      "logps/rejected": -198.70730590820312,
      "loss": 0.2516,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21102575957775116,
      "rewards/margins": 0.06709831953048706,
      "rewards/rejected": -0.2781240940093994,
      "step": 1000
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.993211089144e-06,
      "logits/chosen": -1.9574140310287476,
      "logits/rejected": -1.4666115045547485,
      "logps/chosen": -264.51251220703125,
      "logps/rejected": -221.65234375,
      "loss": 0.1448,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.2643020749092102,
      "rewards/margins": 0.15589205920696259,
      "rewards/rejected": -0.420194149017334,
      "step": 1010
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.992418106575232e-06,
      "logits/chosen": -2.2091064453125,
      "logits/rejected": -1.704564094543457,
      "logps/chosen": -330.94915771484375,
      "logps/rejected": -254.61880493164062,
      "loss": 0.14,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.32784610986709595,
      "rewards/margins": 0.08099902421236038,
      "rewards/rejected": -0.40884512662887573,
      "step": 1020
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.9915814038530505e-06,
      "logits/chosen": -2.091163396835327,
      "logits/rejected": -1.8074891567230225,
      "logps/chosen": -215.7232208251953,
      "logps/rejected": -192.0024871826172,
      "loss": 0.1666,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.18186353147029877,
      "rewards/margins": 0.08887914568185806,
      "rewards/rejected": -0.2707426846027374,
      "step": 1030
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.990700995654274e-06,
      "logits/chosen": -2.0555949211120605,
      "logits/rejected": -1.72856867313385,
      "logps/chosen": -213.6540069580078,
      "logps/rejected": -205.2044677734375,
      "loss": 0.1204,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.35042351484298706,
      "rewards/margins": 0.1402900069952011,
      "rewards/rejected": -0.49071353673934937,
      "step": 1040
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.9897768974223726e-06,
      "logits/chosen": -2.166123867034912,
      "logits/rejected": -1.7946255207061768,
      "logps/chosen": -231.53933715820312,
      "logps/rejected": -211.31204223632812,
      "loss": 0.178,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.20556874573230743,
      "rewards/margins": 0.16699795424938202,
      "rewards/rejected": -0.37256669998168945,
      "step": 1050
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.9888091253671925e-06,
      "logits/chosen": -2.0272960662841797,
      "logits/rejected": -1.4488000869750977,
      "logps/chosen": -219.529052734375,
      "logps/rejected": -171.55601501464844,
      "loss": 0.1258,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1403190791606903,
      "rewards/margins": 0.1539257913827896,
      "rewards/rejected": -0.2942448556423187,
      "step": 1060
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.9877976964646755e-06,
      "logits/chosen": -2.0916085243225098,
      "logits/rejected": -1.8679349422454834,
      "logps/chosen": -252.82199096679688,
      "logps/rejected": -225.4739532470703,
      "loss": 0.1693,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.19687017798423767,
      "rewards/margins": 0.07593067735433578,
      "rewards/rejected": -0.27280086278915405,
      "step": 1070
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.986742628456559e-06,
      "logits/chosen": -2.1024928092956543,
      "logits/rejected": -1.534501075744629,
      "logps/chosen": -257.09710693359375,
      "logps/rejected": -182.4271240234375,
      "loss": 0.179,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.15705084800720215,
      "rewards/margins": 0.11441371589899063,
      "rewards/rejected": -0.27146458625793457,
      "step": 1080
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.985643939850063e-06,
      "logits/chosen": -2.1568455696105957,
      "logits/rejected": -1.6811933517456055,
      "logps/chosen": -275.54144287109375,
      "logps/rejected": -227.31417846679688,
      "loss": 0.1206,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2653849124908447,
      "rewards/margins": 0.15564067661762238,
      "rewards/rejected": -0.4210255742073059,
      "step": 1090
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.984501649917573e-06,
      "logits/chosen": -1.9597032070159912,
      "logits/rejected": -1.5438346862792969,
      "logps/chosen": -223.2861785888672,
      "logps/rejected": -213.6127166748047,
      "loss": 0.1441,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.42509156465530396,
      "rewards/margins": 0.1514272391796112,
      "rewards/rejected": -0.5765187740325928,
      "step": 1100
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.98331577869629e-06,
      "logits/chosen": -1.9955313205718994,
      "logits/rejected": -1.7289683818817139,
      "logps/chosen": -316.2930908203125,
      "logps/rejected": -297.17388916015625,
      "loss": 0.1012,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.4474318027496338,
      "rewards/margins": 0.11068395525217056,
      "rewards/rejected": -0.5581157207489014,
      "step": 1110
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.982086346987891e-06,
      "logits/chosen": -1.8174870014190674,
      "logits/rejected": -1.5796586275100708,
      "logps/chosen": -252.7262725830078,
      "logps/rejected": -244.24661254882812,
      "loss": 0.1905,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.3619054853916168,
      "rewards/margins": 0.08687237650156021,
      "rewards/rejected": -0.44877785444259644,
      "step": 1120
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.980813376358157e-06,
      "logits/chosen": -1.8165006637573242,
      "logits/rejected": -1.5728943347930908,
      "logps/chosen": -238.3287353515625,
      "logps/rejected": -264.8955993652344,
      "loss": 0.1116,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.4419601857662201,
      "rewards/margins": 0.14847253262996674,
      "rewards/rejected": -0.5904327630996704,
      "step": 1130
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.9794968891365955e-06,
      "logits/chosen": -1.9940223693847656,
      "logits/rejected": -1.602085828781128,
      "logps/chosen": -283.71234130859375,
      "logps/rejected": -260.02947998046875,
      "loss": 0.1771,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.42881909012794495,
      "rewards/margins": 0.11037082970142365,
      "rewards/rejected": -0.5391899347305298,
      "step": 1140
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.978136908416052e-06,
      "logits/chosen": -2.128349781036377,
      "logits/rejected": -1.6948553323745728,
      "logps/chosen": -193.53390502929688,
      "logps/rejected": -213.319091796875,
      "loss": 0.1714,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.41444897651672363,
      "rewards/margins": 0.15515312552452087,
      "rewards/rejected": -0.5696021318435669,
      "step": 1150
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.976733458052301e-06,
      "logits/chosen": -2.0403997898101807,
      "logits/rejected": -1.6035503149032593,
      "logps/chosen": -197.5802001953125,
      "logps/rejected": -190.8645782470703,
      "loss": 0.1026,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.46744871139526367,
      "rewards/margins": 0.2037152796983719,
      "rewards/rejected": -0.6711639165878296,
      "step": 1160
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.975286562663629e-06,
      "logits/chosen": -2.0656538009643555,
      "logits/rejected": -1.832098364830017,
      "logps/chosen": -282.55438232421875,
      "logps/rejected": -244.44912719726562,
      "loss": 0.1259,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.4940463900566101,
      "rewards/margins": 0.11086853593587875,
      "rewards/rejected": -0.6049149036407471,
      "step": 1170
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.9737962476304045e-06,
      "logits/chosen": -1.9830493927001953,
      "logits/rejected": -1.647684097290039,
      "logps/chosen": -220.96340942382812,
      "logps/rejected": -208.0186767578125,
      "loss": 0.1464,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.48578500747680664,
      "rewards/margins": 0.15359191596508026,
      "rewards/rejected": -0.6393769383430481,
      "step": 1180
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.972262539094633e-06,
      "logits/chosen": -1.978539228439331,
      "logits/rejected": -1.682488203048706,
      "logps/chosen": -226.36874389648438,
      "logps/rejected": -201.7385711669922,
      "loss": 0.1508,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.38268616795539856,
      "rewards/margins": 0.08130475878715515,
      "rewards/rejected": -0.46399086713790894,
      "step": 1190
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.970685463959489e-06,
      "logits/chosen": -2.1193437576293945,
      "logits/rejected": -1.6852171421051025,
      "logps/chosen": -204.7485809326172,
      "logps/rejected": -179.21803283691406,
      "loss": 0.1222,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.29113954305648804,
      "rewards/margins": 0.18358632922172546,
      "rewards/rejected": -0.4747259020805359,
      "step": 1200
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.969065049888861e-06,
      "logits/chosen": -2.1825406551361084,
      "logits/rejected": -1.6184136867523193,
      "logps/chosen": -186.78878784179688,
      "logps/rejected": -203.55813598632812,
      "loss": 0.1486,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20323626697063446,
      "rewards/margins": 0.19247011840343475,
      "rewards/rejected": -0.3957063853740692,
      "step": 1210
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.9674013253068535e-06,
      "logits/chosen": -2.1128785610198975,
      "logits/rejected": -1.8958683013916016,
      "logps/chosen": -235.69161987304688,
      "logps/rejected": -235.4266357421875,
      "loss": 0.1277,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.22614555060863495,
      "rewards/margins": 0.1264582872390747,
      "rewards/rejected": -0.35260388255119324,
      "step": 1220
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.96569431939729e-06,
      "logits/chosen": -1.8693885803222656,
      "logits/rejected": -1.6573035717010498,
      "logps/chosen": -216.2715301513672,
      "logps/rejected": -202.7554473876953,
      "loss": 0.1266,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3473548889160156,
      "rewards/margins": 0.13557687401771545,
      "rewards/rejected": -0.4829317033290863,
      "step": 1230
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.963944062103205e-06,
      "logits/chosen": -2.0532517433166504,
      "logits/rejected": -1.7747167348861694,
      "logps/chosen": -238.31423950195312,
      "logps/rejected": -231.4252166748047,
      "loss": 0.163,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.2758443355560303,
      "rewards/margins": 0.12897524237632751,
      "rewards/rejected": -0.4048195779323578,
      "step": 1240
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.9621505841263155e-06,
      "logits/chosen": -1.9278017282485962,
      "logits/rejected": -1.6317808628082275,
      "logps/chosen": -209.70388793945312,
      "logps/rejected": -204.15481567382812,
      "loss": 0.1274,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.28794723749160767,
      "rewards/margins": 0.08738715946674347,
      "rewards/rejected": -0.37533441185951233,
      "step": 1250
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.960313916926486e-06,
      "logits/chosen": -1.9663488864898682,
      "logits/rejected": -1.8491098880767822,
      "logps/chosen": -225.07937622070312,
      "logps/rejected": -231.40744018554688,
      "loss": 0.1182,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.31935936212539673,
      "rewards/margins": 0.10864345729351044,
      "rewards/rejected": -0.42800283432006836,
      "step": 1260
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.958434092721172e-06,
      "logits/chosen": -1.9907243251800537,
      "logits/rejected": -1.6675913333892822,
      "logps/chosen": -199.87515258789062,
      "logps/rejected": -212.6792755126953,
      "loss": 0.1512,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.2891536056995392,
      "rewards/margins": 0.11914797872304916,
      "rewards/rejected": -0.40830159187316895,
      "step": 1270
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.956511144484858e-06,
      "logits/chosen": -2.0224695205688477,
      "logits/rejected": -1.5479836463928223,
      "logps/chosen": -303.0483703613281,
      "logps/rejected": -241.37060546875,
      "loss": 0.1655,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4082934856414795,
      "rewards/margins": 0.11875990778207779,
      "rewards/rejected": -0.5270534157752991,
      "step": 1280
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.954545105948479e-06,
      "logits/chosen": -2.2034153938293457,
      "logits/rejected": -2.0540757179260254,
      "logps/chosen": -282.18048095703125,
      "logps/rejected": -286.74310302734375,
      "loss": 0.163,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.4291737973690033,
      "rewards/margins": 0.12008972465991974,
      "rewards/rejected": -0.5492635369300842,
      "step": 1290
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.952536011598828e-06,
      "logits/chosen": -1.9675910472869873,
      "logits/rejected": -1.8413200378417969,
      "logps/chosen": -218.7751922607422,
      "logps/rejected": -267.6961669921875,
      "loss": 0.2403,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.3774875998497009,
      "rewards/margins": 0.08021329343318939,
      "rewards/rejected": -0.45770081877708435,
      "step": 1300
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.950483896677949e-06,
      "logits/chosen": -1.9533389806747437,
      "logits/rejected": -1.5656068325042725,
      "logps/chosen": -268.5809020996094,
      "logps/rejected": -251.406494140625,
      "loss": 0.0979,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.43018245697021484,
      "rewards/margins": 0.2314719408750534,
      "rewards/rejected": -0.6616543531417847,
      "step": 1310
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.948388797182525e-06,
      "logits/chosen": -1.9137376546859741,
      "logits/rejected": -1.8603311777114868,
      "logps/chosen": -179.66416931152344,
      "logps/rejected": -240.1300506591797,
      "loss": 0.2446,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3417154848575592,
      "rewards/margins": 0.15184545516967773,
      "rewards/rejected": -0.4935609698295593,
      "step": 1320
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.9462507498632404e-06,
      "logits/chosen": -1.8757511377334595,
      "logits/rejected": -1.556334376335144,
      "logps/chosen": -170.3055419921875,
      "logps/rejected": -155.99923706054688,
      "loss": 0.1384,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.09021402895450592,
      "rewards/margins": 0.061974525451660156,
      "rewards/rejected": -0.15218856930732727,
      "step": 1330
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.944069792224138e-06,
      "logits/chosen": -2.04624080657959,
      "logits/rejected": -1.6785959005355835,
      "logps/chosen": -259.4488525390625,
      "logps/rejected": -208.564697265625,
      "loss": 0.1482,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11900661140680313,
      "rewards/margins": 0.09059344977140427,
      "rewards/rejected": -0.2096000462770462,
      "step": 1340
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.941845962521961e-06,
      "logits/chosen": -2.206084728240967,
      "logits/rejected": -1.8071863651275635,
      "logps/chosen": -174.04827880859375,
      "logps/rejected": -162.29518127441406,
      "loss": 0.1481,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.22567129135131836,
      "rewards/margins": 0.115481436252594,
      "rewards/rejected": -0.34115272760391235,
      "step": 1350
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.939579299765485e-06,
      "logits/chosen": -2.1437458992004395,
      "logits/rejected": -1.828401803970337,
      "logps/chosen": -189.3341522216797,
      "logps/rejected": -232.9872283935547,
      "loss": 0.1491,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3480433225631714,
      "rewards/margins": 0.12997707724571228,
      "rewards/rejected": -0.47802042961120605,
      "step": 1360
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.937269843714831e-06,
      "logits/chosen": -1.8172667026519775,
      "logits/rejected": -1.5740686655044556,
      "logps/chosen": -230.97781372070312,
      "logps/rejected": -243.69906616210938,
      "loss": 0.1532,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.3387555181980133,
      "rewards/margins": 0.12509162724018097,
      "rewards/rejected": -0.46384716033935547,
      "step": 1370
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.934917634880766e-06,
      "logits/chosen": -1.8426252603530884,
      "logits/rejected": -1.6131852865219116,
      "logps/chosen": -208.56918334960938,
      "logps/rejected": -232.096435546875,
      "loss": 0.1422,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.3319644033908844,
      "rewards/margins": 0.11488902568817139,
      "rewards/rejected": -0.4468534588813782,
      "step": 1380
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.932522714523996e-06,
      "logits/chosen": -1.8893773555755615,
      "logits/rejected": -1.7410329580307007,
      "logps/chosen": -207.05514526367188,
      "logps/rejected": -207.72817993164062,
      "loss": 0.1441,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.40233850479125977,
      "rewards/margins": 0.09210322797298431,
      "rewards/rejected": -0.4944417476654053,
      "step": 1390
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.930085124654443e-06,
      "logits/chosen": -2.0386033058166504,
      "logits/rejected": -1.3505053520202637,
      "logps/chosen": -369.05743408203125,
      "logps/rejected": -262.7327575683594,
      "loss": 0.144,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.38073664903640747,
      "rewards/margins": 0.19237622618675232,
      "rewards/rejected": -0.5731129050254822,
      "step": 1400
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.927604908030503e-06,
      "logits/chosen": -1.893441915512085,
      "logits/rejected": -1.7213819026947021,
      "logps/chosen": -203.71377563476562,
      "logps/rejected": -231.38339233398438,
      "loss": 0.2063,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2586003541946411,
      "rewards/margins": 0.1567048281431198,
      "rewards/rejected": -0.4153051972389221,
      "step": 1410
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.9250821081583e-06,
      "logits/chosen": -1.9134151935577393,
      "logits/rejected": -1.5941439867019653,
      "logps/chosen": -238.3423309326172,
      "logps/rejected": -239.90673828125,
      "loss": 0.1188,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.3424052298069,
      "rewards/margins": 0.14903725683689117,
      "rewards/rejected": -0.49144238233566284,
      "step": 1420
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.922516769290921e-06,
      "logits/chosen": -2.1062850952148438,
      "logits/rejected": -1.5940475463867188,
      "logps/chosen": -285.4846496582031,
      "logps/rejected": -267.6663818359375,
      "loss": 0.0838,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.41442838311195374,
      "rewards/margins": 0.16315698623657227,
      "rewards/rejected": -0.5775853395462036,
      "step": 1430
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.919908936427643e-06,
      "logits/chosen": -1.9641939401626587,
      "logits/rejected": -1.6408805847167969,
      "logps/chosen": -233.9027862548828,
      "logps/rejected": -241.10244750976562,
      "loss": 0.1349,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.31380248069763184,
      "rewards/margins": 0.1510525941848755,
      "rewards/rejected": -0.4648551344871521,
      "step": 1440
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.917258655313137e-06,
      "logits/chosen": -1.9187742471694946,
      "logits/rejected": -1.7553646564483643,
      "logps/chosen": -183.12078857421875,
      "logps/rejected": -215.263916015625,
      "loss": 0.1463,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.45299237966537476,
      "rewards/margins": 0.14117896556854248,
      "rewards/rejected": -0.5941713452339172,
      "step": 1450
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.914565972436677e-06,
      "logits/chosen": -1.9224255084991455,
      "logits/rejected": -1.568176031112671,
      "logps/chosen": -243.716552734375,
      "logps/rejected": -229.509765625,
      "loss": 0.1259,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.32048431038856506,
      "rewards/margins": 0.1995118111371994,
      "rewards/rejected": -0.5199961066246033,
      "step": 1460
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.911830935031308e-06,
      "logits/chosen": -1.7421767711639404,
      "logits/rejected": -1.6233383417129517,
      "logps/chosen": -248.0158233642578,
      "logps/rejected": -251.9866485595703,
      "loss": 0.1802,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.37991154193878174,
      "rewards/margins": 0.10848450660705566,
      "rewards/rejected": -0.4883960783481598,
      "step": 1470
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.909053591073034e-06,
      "logits/chosen": -1.8475421667099,
      "logits/rejected": -1.5178005695343018,
      "logps/chosen": -234.1063995361328,
      "logps/rejected": -198.0278778076172,
      "loss": 0.188,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.21273323893547058,
      "rewards/margins": 0.10500024259090424,
      "rewards/rejected": -0.3177334666252136,
      "step": 1480
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.906233989279967e-06,
      "logits/chosen": -2.128617525100708,
      "logits/rejected": -1.6844680309295654,
      "logps/chosen": -254.1748504638672,
      "logps/rejected": -218.5316162109375,
      "loss": 0.1283,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.23462820053100586,
      "rewards/margins": 0.07426507025957108,
      "rewards/rejected": -0.30889326333999634,
      "step": 1490
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.903372179111473e-06,
      "logits/chosen": -1.8551340103149414,
      "logits/rejected": -1.7969564199447632,
      "logps/chosen": -245.1498565673828,
      "logps/rejected": -237.3717498779297,
      "loss": 0.1349,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.26474660634994507,
      "rewards/margins": 0.06484004855155945,
      "rewards/rejected": -0.3295866847038269,
      "step": 1500
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.900468210767309e-06,
      "logits/chosen": -1.875862717628479,
      "logits/rejected": -1.5309476852416992,
      "logps/chosen": -220.53408813476562,
      "logps/rejected": -184.7772674560547,
      "loss": 0.2182,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20155379176139832,
      "rewards/margins": 0.08043310791254044,
      "rewards/rejected": -0.28198689222335815,
      "step": 1510
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.897522135186737e-06,
      "logits/chosen": -2.022017478942871,
      "logits/rejected": -1.7306649684906006,
      "logps/chosen": -266.8653869628906,
      "logps/rejected": -258.17962646484375,
      "loss": 0.1517,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2780263423919678,
      "rewards/margins": 0.12058179080486298,
      "rewards/rejected": -0.39860814809799194,
      "step": 1520
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.894534004047635e-06,
      "logits/chosen": -2.042154312133789,
      "logits/rejected": -1.7138588428497314,
      "logps/chosen": -322.60791015625,
      "logps/rejected": -300.8125915527344,
      "loss": 0.0724,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.37096601724624634,
      "rewards/margins": 0.13116590678691864,
      "rewards/rejected": -0.502131998538971,
      "step": 1530
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.891503869765586e-06,
      "logits/chosen": -2.0083236694335938,
      "logits/rejected": -1.7603801488876343,
      "logps/chosen": -266.07330322265625,
      "logps/rejected": -251.0727081298828,
      "loss": 0.0935,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.27762866020202637,
      "rewards/margins": 0.21909542381763458,
      "rewards/rejected": -0.49672412872314453,
      "step": 1540
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.888431785492964e-06,
      "logits/chosen": -2.025075912475586,
      "logits/rejected": -1.55344820022583,
      "logps/chosen": -198.4198455810547,
      "logps/rejected": -193.17062377929688,
      "loss": 0.1424,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.22466003894805908,
      "rewards/margins": 0.1589314192533493,
      "rewards/rejected": -0.3835914731025696,
      "step": 1550
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.8853178051179965e-06,
      "logits/chosen": -2.004646062850952,
      "logits/rejected": -1.5767240524291992,
      "logps/chosen": -275.49566650390625,
      "logps/rejected": -211.9617462158203,
      "loss": 0.1153,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.24958455562591553,
      "rewards/margins": 0.12301850318908691,
      "rewards/rejected": -0.37260305881500244,
      "step": 1560
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.882161983263822e-06,
      "logits/chosen": -1.9383262395858765,
      "logits/rejected": -1.6864608526229858,
      "logps/chosen": -215.1851348876953,
      "logps/rejected": -219.91763305664062,
      "loss": 0.1282,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.21753136813640594,
      "rewards/margins": 0.07913483679294586,
      "rewards/rejected": -0.2966662049293518,
      "step": 1570
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.8789643752875315e-06,
      "logits/chosen": -2.1681385040283203,
      "logits/rejected": -1.5037636756896973,
      "logps/chosen": -296.7191467285156,
      "logps/rejected": -210.6960906982422,
      "loss": 0.0908,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.13743911683559418,
      "rewards/margins": 0.14550727605819702,
      "rewards/rejected": -0.2829464077949524,
      "step": 1580
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.875725037279197e-06,
      "logits/chosen": -2.100879669189453,
      "logits/rejected": -1.6887743473052979,
      "logps/chosen": -275.7843322753906,
      "logps/rejected": -254.24166870117188,
      "loss": 0.106,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.2630919814109802,
      "rewards/margins": 0.1653970181941986,
      "rewards/rejected": -0.4284890294075012,
      "step": 1590
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.8724440260608885e-06,
      "logits/chosen": -2.100240707397461,
      "logits/rejected": -1.6715034246444702,
      "logps/chosen": -215.9812774658203,
      "logps/rejected": -220.94790649414062,
      "loss": 0.1162,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.312126100063324,
      "rewards/margins": 0.20332340896129608,
      "rewards/rejected": -0.515449583530426,
      "step": 1600
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.8691213991856755e-06,
      "logits/chosen": -2.143995523452759,
      "logits/rejected": -1.8109419345855713,
      "logps/chosen": -219.9774932861328,
      "logps/rejected": -207.9420623779297,
      "loss": 0.1514,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.3066982924938202,
      "rewards/margins": 0.12182126939296722,
      "rewards/rejected": -0.4285196363925934,
      "step": 1610
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.8657572149366195e-06,
      "logits/chosen": -2.0417563915252686,
      "logits/rejected": -1.8656389713287354,
      "logps/chosen": -240.6365203857422,
      "logps/rejected": -234.8029022216797,
      "loss": 0.1367,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.2173691689968109,
      "rewards/margins": 0.11171890795230865,
      "rewards/rejected": -0.32908809185028076,
      "step": 1620
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.8623515323257496e-06,
      "logits/chosen": -1.8849719762802124,
      "logits/rejected": -1.764593482017517,
      "logps/chosen": -237.77490234375,
      "logps/rejected": -266.98089599609375,
      "loss": 0.138,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.3375057876110077,
      "rewards/margins": 0.1129181832075119,
      "rewards/rejected": -0.4504240155220032,
      "step": 1630
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.85890441109303e-06,
      "logits/chosen": -1.9908783435821533,
      "logits/rejected": -1.6857595443725586,
      "logps/chosen": -254.00650024414062,
      "logps/rejected": -248.4944610595703,
      "loss": 0.1031,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.35663676261901855,
      "rewards/margins": 0.18786312639713287,
      "rewards/rejected": -0.544499933719635,
      "step": 1640
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.855415911705308e-06,
      "logits/chosen": -2.0321478843688965,
      "logits/rejected": -1.9031927585601807,
      "logps/chosen": -243.13436889648438,
      "logps/rejected": -255.0236053466797,
      "loss": 0.1124,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.29369717836380005,
      "rewards/margins": 0.11365096271038055,
      "rewards/rejected": -0.4073481559753418,
      "step": 1650
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.851886095355259e-06,
      "logits/chosen": -2.0734264850616455,
      "logits/rejected": -1.5513179302215576,
      "logps/chosen": -311.870849609375,
      "logps/rejected": -246.34963989257812,
      "loss": 0.3146,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.44044867157936096,
      "rewards/margins": 0.06915672868490219,
      "rewards/rejected": -0.509605348110199,
      "step": 1660
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.848315023960308e-06,
      "logits/chosen": -2.0195250511169434,
      "logits/rejected": -1.5690950155258179,
      "logps/chosen": -245.0861053466797,
      "logps/rejected": -176.3268280029297,
      "loss": 0.1332,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.32303136587142944,
      "rewards/margins": 0.14561942219734192,
      "rewards/rejected": -0.46865081787109375,
      "step": 1670
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.844702760161546e-06,
      "logits/chosen": -1.9474140405654907,
      "logits/rejected": -1.498877763748169,
      "logps/chosen": -214.76220703125,
      "logps/rejected": -202.29042053222656,
      "loss": 0.1529,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.34490758180618286,
      "rewards/margins": 0.16737958788871765,
      "rewards/rejected": -0.5122871398925781,
      "step": 1680
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.841049367322631e-06,
      "logits/chosen": -1.878689169883728,
      "logits/rejected": -1.5493450164794922,
      "logps/chosen": -263.93896484375,
      "logps/rejected": -252.0777587890625,
      "loss": 0.112,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.32466429471969604,
      "rewards/margins": 0.15016327798366547,
      "rewards/rejected": -0.4748276174068451,
      "step": 1690
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.837354909528675e-06,
      "logits/chosen": -1.8449478149414062,
      "logits/rejected": -1.8139785528182983,
      "logps/chosen": -187.8353271484375,
      "logps/rejected": -203.24066162109375,
      "loss": 0.2003,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.4078170359134674,
      "rewards/margins": 0.08712447434663773,
      "rewards/rejected": -0.49494147300720215,
      "step": 1700
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.833619451585122e-06,
      "logits/chosen": -1.8499475717544556,
      "logits/rejected": -1.498587727546692,
      "logps/chosen": -243.45242309570312,
      "logps/rejected": -231.98251342773438,
      "loss": 0.1135,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3529844284057617,
      "rewards/margins": 0.16045571863651276,
      "rewards/rejected": -0.5134401321411133,
      "step": 1710
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.829843059016611e-06,
      "logits/chosen": -1.81149423122406,
      "logits/rejected": -1.4992173910140991,
      "logps/chosen": -184.52310180664062,
      "logps/rejected": -201.0103759765625,
      "loss": 0.1903,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.35175323486328125,
      "rewards/margins": 0.14889448881149292,
      "rewards/rejected": -0.5006477236747742,
      "step": 1720
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.826025798065823e-06,
      "logits/chosen": -1.8906818628311157,
      "logits/rejected": -1.7450675964355469,
      "logps/chosen": -254.50241088867188,
      "logps/rejected": -233.2667999267578,
      "loss": 0.195,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.4035833775997162,
      "rewards/margins": 0.10915178060531616,
      "rewards/rejected": -0.5127351880073547,
      "step": 1730
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.8221677356923255e-06,
      "logits/chosen": -1.822003722190857,
      "logits/rejected": -1.6503994464874268,
      "logps/chosen": -194.3645782470703,
      "logps/rejected": -234.00198364257812,
      "loss": 0.19,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.42598286271095276,
      "rewards/margins": 0.17088128626346588,
      "rewards/rejected": -0.5968641638755798,
      "step": 1740
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.8182689395713925e-06,
      "logits/chosen": -1.911811113357544,
      "logits/rejected": -1.4322118759155273,
      "logps/chosen": -241.7019500732422,
      "logps/rejected": -241.4373016357422,
      "loss": 0.1291,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.4872862696647644,
      "rewards/margins": 0.21816511452198029,
      "rewards/rejected": -0.7054513692855835,
      "step": 1750
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.814329478092818e-06,
      "logits/chosen": -2.0462019443511963,
      "logits/rejected": -1.638604760169983,
      "logps/chosen": -279.9619445800781,
      "logps/rejected": -274.87408447265625,
      "loss": 0.1234,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.4822389483451843,
      "rewards/margins": 0.10693428665399551,
      "rewards/rejected": -0.5891731977462769,
      "step": 1760
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.810349420359722e-06,
      "logits/chosen": -1.8592302799224854,
      "logits/rejected": -1.4212658405303955,
      "logps/chosen": -241.1697998046875,
      "logps/rejected": -256.21026611328125,
      "loss": 0.1746,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5237879753112793,
      "rewards/margins": 0.18233174085617065,
      "rewards/rejected": -0.70611971616745,
      "step": 1770
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.806328836187328e-06,
      "logits/chosen": -1.9457238912582397,
      "logits/rejected": -1.5514074563980103,
      "logps/chosen": -258.5703430175781,
      "logps/rejected": -231.8943634033203,
      "loss": 0.1483,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.43616265058517456,
      "rewards/margins": 0.18835784494876862,
      "rewards/rejected": -0.624520480632782,
      "step": 1780
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.802267796101749e-06,
      "logits/chosen": -1.8042892217636108,
      "logits/rejected": -1.4870389699935913,
      "logps/chosen": -296.75115966796875,
      "logps/rejected": -246.88882446289062,
      "loss": 0.1334,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.48000073432922363,
      "rewards/margins": 0.14861378073692322,
      "rewards/rejected": -0.6286145448684692,
      "step": 1790
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.798166371338745e-06,
      "logits/chosen": -1.9880012273788452,
      "logits/rejected": -1.7663252353668213,
      "logps/chosen": -252.8795623779297,
      "logps/rejected": -283.4180603027344,
      "loss": 0.1518,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.3115619122982025,
      "rewards/margins": 0.16061726212501526,
      "rewards/rejected": -0.4721791744232178,
      "step": 1800
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.79402463384247e-06,
      "logits/chosen": -1.9161767959594727,
      "logits/rejected": -1.753260850906372,
      "logps/chosen": -244.0667266845703,
      "logps/rejected": -268.25164794921875,
      "loss": 0.1697,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.299142062664032,
      "rewards/margins": 0.11554409563541412,
      "rewards/rejected": -0.4146861433982849,
      "step": 1810
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.78984265626422e-06,
      "logits/chosen": -1.8210163116455078,
      "logits/rejected": -1.640275001525879,
      "logps/chosen": -195.47975158691406,
      "logps/rejected": -204.36692810058594,
      "loss": 0.1215,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.2043401002883911,
      "rewards/margins": 0.11669757217168808,
      "rewards/rejected": -0.3210376501083374,
      "step": 1820
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.785620511961148e-06,
      "logits/chosen": -2.0741043090820312,
      "logits/rejected": -1.788116455078125,
      "logps/chosen": -263.5960388183594,
      "logps/rejected": -245.22525024414062,
      "loss": 0.1444,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.22155031561851501,
      "rewards/margins": 0.10210974514484406,
      "rewards/rejected": -0.3236600458621979,
      "step": 1830
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.781358274994985e-06,
      "logits/chosen": -2.1329751014709473,
      "logits/rejected": -1.7640917301177979,
      "logps/chosen": -227.3170928955078,
      "logps/rejected": -210.49746704101562,
      "loss": 0.1666,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2090965211391449,
      "rewards/margins": 0.1362505704164505,
      "rewards/rejected": -0.3453471064567566,
      "step": 1840
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.777056020130737e-06,
      "logits/chosen": -2.188413143157959,
      "logits/rejected": -1.6932777166366577,
      "logps/chosen": -314.6272277832031,
      "logps/rejected": -289.4335632324219,
      "loss": 0.1244,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.3082244098186493,
      "rewards/margins": 0.21150562167167664,
      "rewards/rejected": -0.5197300314903259,
      "step": 1850
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.772713822835374e-06,
      "logits/chosen": -1.8019129037857056,
      "logits/rejected": -1.4269254207611084,
      "logps/chosen": -215.6371307373047,
      "logps/rejected": -216.8037567138672,
      "loss": 0.1824,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.3367902636528015,
      "rewards/margins": 0.19317738711833954,
      "rewards/rejected": -0.5299676656723022,
      "step": 1860
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.768331759276506e-06,
      "logits/chosen": -2.0352442264556885,
      "logits/rejected": -1.8071515560150146,
      "logps/chosen": -285.3484802246094,
      "logps/rejected": -280.1053161621094,
      "loss": 0.084,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.29302871227264404,
      "rewards/margins": 0.1241946592926979,
      "rewards/rejected": -0.41722336411476135,
      "step": 1870
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.763909906321048e-06,
      "logits/chosen": -1.9953176975250244,
      "logits/rejected": -1.617118239402771,
      "logps/chosen": -248.2249755859375,
      "logps/rejected": -225.35562133789062,
      "loss": 0.1598,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.24114413559436798,
      "rewards/margins": 0.1202860102057457,
      "rewards/rejected": -0.36143016815185547,
      "step": 1880
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.759448341533872e-06,
      "logits/chosen": -1.7579656839370728,
      "logits/rejected": -1.539206624031067,
      "logps/chosen": -255.2971649169922,
      "logps/rejected": -270.1570739746094,
      "loss": 0.1665,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.401685893535614,
      "rewards/margins": 0.19057750701904297,
      "rewards/rejected": -0.5922634601593018,
      "step": 1890
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.754947143176445e-06,
      "logits/chosen": -1.880765676498413,
      "logits/rejected": -1.4084635972976685,
      "logps/chosen": -198.8912353515625,
      "logps/rejected": -170.9627227783203,
      "loss": 0.1344,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.33145859837532043,
      "rewards/margins": 0.16721078753471375,
      "rewards/rejected": -0.4986693859100342,
      "step": 1900
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.750406390205456e-06,
      "logits/chosen": -1.9753971099853516,
      "logits/rejected": -1.8784997463226318,
      "logps/chosen": -285.95343017578125,
      "logps/rejected": -265.814453125,
      "loss": 0.1751,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.28710874915122986,
      "rewards/margins": 0.07001986354589462,
      "rewards/rejected": -0.3571286201477051,
      "step": 1910
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.745826162271433e-06,
      "logits/chosen": -2.2576117515563965,
      "logits/rejected": -1.8165054321289062,
      "logps/chosen": -266.011474609375,
      "logps/rejected": -255.41897583007812,
      "loss": 0.1212,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.22302386164665222,
      "rewards/margins": 0.15649394690990448,
      "rewards/rejected": -0.3795178532600403,
      "step": 1920
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.741206539717343e-06,
      "logits/chosen": -2.0917410850524902,
      "logits/rejected": -1.5043323040008545,
      "logps/chosen": -253.68002319335938,
      "logps/rejected": -229.77005004882812,
      "loss": 0.1366,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18172211945056915,
      "rewards/margins": 0.23254887759685516,
      "rewards/rejected": -0.41427096724510193,
      "step": 1930
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.736547603577185e-06,
      "logits/chosen": -1.7454341650009155,
      "logits/rejected": -1.6577666997909546,
      "logps/chosen": -199.49789428710938,
      "logps/rejected": -199.80279541015625,
      "loss": 0.1921,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.3379337191581726,
      "rewards/margins": 0.12971019744873047,
      "rewards/rejected": -0.4676439166069031,
      "step": 1940
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.731849435574568e-06,
      "logits/chosen": -2.069859504699707,
      "logits/rejected": -1.7830657958984375,
      "logps/chosen": -232.3899688720703,
      "logps/rejected": -232.227294921875,
      "loss": 0.1621,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2579636573791504,
      "rewards/margins": 0.18101055920124054,
      "rewards/rejected": -0.4389742314815521,
      "step": 1950
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.727112118121279e-06,
      "logits/chosen": -2.024989366531372,
      "logits/rejected": -1.809133768081665,
      "logps/chosen": -225.9645538330078,
      "logps/rejected": -218.0653533935547,
      "loss": 0.1256,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.27211037278175354,
      "rewards/margins": 0.13195666670799255,
      "rewards/rejected": -0.4040670394897461,
      "step": 1960
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.722335734315833e-06,
      "logits/chosen": -1.990189790725708,
      "logits/rejected": -1.5594546794891357,
      "logps/chosen": -290.8906555175781,
      "logps/rejected": -227.53616333007812,
      "loss": 0.0908,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.2791885733604431,
      "rewards/margins": 0.1558622419834137,
      "rewards/rejected": -0.4350507855415344,
      "step": 1970
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.7175203679420175e-06,
      "logits/chosen": -1.9072424173355103,
      "logits/rejected": -1.51438307762146,
      "logps/chosen": -216.40524291992188,
      "logps/rejected": -228.6339874267578,
      "loss": 0.1292,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.37562626600265503,
      "rewards/margins": 0.20719614624977112,
      "rewards/rejected": -0.582822322845459,
      "step": 1980
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.712666103467428e-06,
      "logits/chosen": -2.0311849117279053,
      "logits/rejected": -1.8723666667938232,
      "logps/chosen": -233.4354705810547,
      "logps/rejected": -218.10659790039062,
      "loss": 0.155,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2636135220527649,
      "rewards/margins": 0.1285993456840515,
      "rewards/rejected": -0.3922128677368164,
      "step": 1990
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.707773026041975e-06,
      "logits/chosen": -2.1100738048553467,
      "logits/rejected": -1.8569968938827515,
      "logps/chosen": -268.63885498046875,
      "logps/rejected": -232.58578491210938,
      "loss": 0.1733,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.27281227707862854,
      "rewards/margins": 0.13495466113090515,
      "rewards/rejected": -0.4077669084072113,
      "step": 2000
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.702841221496403e-06,
      "logits/chosen": -1.991676688194275,
      "logits/rejected": -1.5846397876739502,
      "logps/chosen": -257.11932373046875,
      "logps/rejected": -221.46017456054688,
      "loss": 0.1592,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.24664053320884705,
      "rewards/margins": 0.13256794214248657,
      "rewards/rejected": -0.3792084753513336,
      "step": 2010
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.697870776340776e-06,
      "logits/chosen": -2.2750840187072754,
      "logits/rejected": -1.7563819885253906,
      "logps/chosen": -235.4178009033203,
      "logps/rejected": -190.757080078125,
      "loss": 0.1656,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1487046480178833,
      "rewards/margins": 0.1518375426530838,
      "rewards/rejected": -0.3005422055721283,
      "step": 2020
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.692861777762963e-06,
      "logits/chosen": -2.0295958518981934,
      "logits/rejected": -1.5150290727615356,
      "logps/chosen": -232.8603973388672,
      "logps/rejected": -198.23338317871094,
      "loss": 0.119,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18991556763648987,
      "rewards/margins": 0.1996304988861084,
      "rewards/rejected": -0.3895460069179535,
      "step": 2030
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.68781431362711e-06,
      "logits/chosen": -2.090059518814087,
      "logits/rejected": -1.5975253582000732,
      "logps/chosen": -268.8729553222656,
      "logps/rejected": -242.9998779296875,
      "loss": 0.1739,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.427692174911499,
      "rewards/margins": 0.1998465359210968,
      "rewards/rejected": -0.6275386810302734,
      "step": 2040
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.6827284724720955e-06,
      "logits/chosen": -2.05842924118042,
      "logits/rejected": -1.7655296325683594,
      "logps/chosen": -258.84442138671875,
      "logps/rejected": -226.68350219726562,
      "loss": 0.1977,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.3383828103542328,
      "rewards/margins": 0.13462142646312714,
      "rewards/rejected": -0.4730042517185211,
      "step": 2050
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.677604343509981e-06,
      "logits/chosen": -2.037433385848999,
      "logits/rejected": -1.5807982683181763,
      "logps/chosen": -238.64389038085938,
      "logps/rejected": -213.24490356445312,
      "loss": 0.1261,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.2423369586467743,
      "rewards/margins": 0.174465611577034,
      "rewards/rejected": -0.4168025553226471,
      "step": 2060
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.672442016624444e-06,
      "logits/chosen": -2.1892218589782715,
      "logits/rejected": -1.862447738647461,
      "logps/chosen": -290.03985595703125,
      "logps/rejected": -240.31771850585938,
      "loss": 0.1574,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2680838704109192,
      "rewards/margins": 0.11730837821960449,
      "rewards/rejected": -0.3853922486305237,
      "step": 2070
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.6672415823692e-06,
      "logits/chosen": -1.8660366535186768,
      "logits/rejected": -1.5226314067840576,
      "logps/chosen": -304.72833251953125,
      "logps/rejected": -276.5460205078125,
      "loss": 0.1431,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.32084205746650696,
      "rewards/margins": 0.15598034858703613,
      "rewards/rejected": -0.4768224358558655,
      "step": 2080
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.662003131966418e-06,
      "logits/chosen": -2.167304277420044,
      "logits/rejected": -1.6622650623321533,
      "logps/chosen": -235.3962860107422,
      "logps/rejected": -217.3386688232422,
      "loss": 0.1545,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.2755299210548401,
      "rewards/margins": 0.10942339897155762,
      "rewards/rejected": -0.3849533200263977,
      "step": 2090
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.6567267573051176e-06,
      "logits/chosen": -1.8638086318969727,
      "logits/rejected": -1.7130645513534546,
      "logps/chosen": -219.14736938476562,
      "logps/rejected": -230.8884735107422,
      "loss": 0.1861,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.282665878534317,
      "rewards/margins": 0.05729461461305618,
      "rewards/rejected": -0.339960515499115,
      "step": 2100
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.651412550939556e-06,
      "logits/chosen": -2.023266553878784,
      "logits/rejected": -1.448335886001587,
      "logps/chosen": -222.55819702148438,
      "logps/rejected": -192.7770538330078,
      "loss": 0.1121,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2549906075000763,
      "rewards/margins": 0.18708500266075134,
      "rewards/rejected": -0.44207563996315,
      "step": 2110
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.646060606087608e-06,
      "logits/chosen": -1.9137452840805054,
      "logits/rejected": -1.637158751487732,
      "logps/chosen": -258.3423767089844,
      "logps/rejected": -228.5346221923828,
      "loss": 0.1564,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.3416040241718292,
      "rewards/margins": 0.14088527858257294,
      "rewards/rejected": -0.4824892580509186,
      "step": 2120
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.640671016629129e-06,
      "logits/chosen": -1.8286240100860596,
      "logits/rejected": -1.660211205482483,
      "logps/chosen": -262.9971008300781,
      "logps/rejected": -271.8037109375,
      "loss": 0.1143,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.4632445275783539,
      "rewards/margins": 0.1317237764596939,
      "rewards/rejected": -0.594968318939209,
      "step": 2130
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.635243877104307e-06,
      "logits/chosen": -2.006418466567993,
      "logits/rejected": -1.716923475265503,
      "logps/chosen": -250.9329376220703,
      "logps/rejected": -260.55462646484375,
      "loss": 0.1458,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.34360507130622864,
      "rewards/margins": 0.0931440070271492,
      "rewards/rejected": -0.43674907088279724,
      "step": 2140
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.629779282712007e-06,
      "logits/chosen": -1.8346214294433594,
      "logits/rejected": -1.4906994104385376,
      "logps/chosen": -262.6698913574219,
      "logps/rejected": -255.5764923095703,
      "loss": 0.1224,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4487072825431824,
      "rewards/margins": 0.2172963172197342,
      "rewards/rejected": -0.6660035848617554,
      "step": 2150
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.6242773293080965e-06,
      "logits/chosen": -2.074744462966919,
      "logits/rejected": -1.5857570171356201,
      "logps/chosen": -322.7615966796875,
      "logps/rejected": -300.53790283203125,
      "loss": 0.076,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.47309666872024536,
      "rewards/margins": 0.2397969663143158,
      "rewards/rejected": -0.712893545627594,
      "step": 2160
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.618738113403772e-06,
      "logits/chosen": -1.9601848125457764,
      "logits/rejected": -1.3724687099456787,
      "logps/chosen": -329.15814208984375,
      "logps/rejected": -296.91790771484375,
      "loss": 0.0874,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.4118635058403015,
      "rewards/margins": 0.20115897059440613,
      "rewards/rejected": -0.6130224466323853,
      "step": 2170
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.613161732163857e-06,
      "logits/chosen": -2.002680540084839,
      "logits/rejected": -1.7260891199111938,
      "logps/chosen": -218.44058227539062,
      "logps/rejected": -206.19589233398438,
      "loss": 0.1605,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4169228672981262,
      "rewards/margins": 0.15920642018318176,
      "rewards/rejected": -0.5761292576789856,
      "step": 2180
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.607548283405103e-06,
      "logits/chosen": -2.2463881969451904,
      "logits/rejected": -1.870919942855835,
      "logps/chosen": -271.3766784667969,
      "logps/rejected": -237.7291259765625,
      "loss": 0.1687,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.3008974492549896,
      "rewards/margins": 0.21275082230567932,
      "rewards/rejected": -0.5136483311653137,
      "step": 2190
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.601897865594473e-06,
      "logits/chosen": -2.1344265937805176,
      "logits/rejected": -1.807756781578064,
      "logps/chosen": -254.6204833984375,
      "logps/rejected": -278.7408142089844,
      "loss": 0.0969,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2825480103492737,
      "rewards/margins": 0.14841003715991974,
      "rewards/rejected": -0.43095797300338745,
      "step": 2200
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.596210577847415e-06,
      "logits/chosen": -1.8466428518295288,
      "logits/rejected": -1.4773153066635132,
      "logps/chosen": -221.1357421875,
      "logps/rejected": -214.00961303710938,
      "loss": 0.1446,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.2894059717655182,
      "rewards/margins": 0.18356752395629883,
      "rewards/rejected": -0.4729735255241394,
      "step": 2210
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.590486519926118e-06,
      "logits/chosen": -1.8348041772842407,
      "logits/rejected": -1.76007080078125,
      "logps/chosen": -247.6072540283203,
      "logps/rejected": -254.64437866210938,
      "loss": 0.1073,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.30001839995384216,
      "rewards/margins": 0.1468794047832489,
      "rewards/rejected": -0.44689780473709106,
      "step": 2220
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.584725792237772e-06,
      "logits/chosen": -1.8341724872589111,
      "logits/rejected": -1.4840484857559204,
      "logps/chosen": -281.08843994140625,
      "logps/rejected": -287.3544006347656,
      "loss": 0.1452,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3432127833366394,
      "rewards/margins": 0.16836020350456238,
      "rewards/rejected": -0.5115729570388794,
      "step": 2230
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.578928495832795e-06,
      "logits/chosen": -2.124887466430664,
      "logits/rejected": -1.5507080554962158,
      "logps/chosen": -280.98626708984375,
      "logps/rejected": -227.08627319335938,
      "loss": 0.1314,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.43157821893692017,
      "rewards/margins": 0.19229252636432648,
      "rewards/rejected": -0.6238707304000854,
      "step": 2240
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.57309473240307e-06,
      "logits/chosen": -2.0037436485290527,
      "logits/rejected": -1.5078222751617432,
      "logps/chosen": -269.2857360839844,
      "logps/rejected": -206.1649627685547,
      "loss": 0.1401,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.32349663972854614,
      "rewards/margins": 0.14150217175483704,
      "rewards/rejected": -0.4649987816810608,
      "step": 2250
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.567224604280157e-06,
      "logits/chosen": -1.7673028707504272,
      "logits/rejected": -1.6784632205963135,
      "logps/chosen": -184.74407958984375,
      "logps/rejected": -243.2403106689453,
      "loss": 0.1252,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.33205723762512207,
      "rewards/margins": 0.1660684049129486,
      "rewards/rejected": -0.49812570214271545,
      "step": 2260
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.561318214433499e-06,
      "logits/chosen": -1.9934532642364502,
      "logits/rejected": -1.8898242712020874,
      "logps/chosen": -221.0784912109375,
      "logps/rejected": -242.1062469482422,
      "loss": 0.1536,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.31267839670181274,
      "rewards/margins": 0.1026720255613327,
      "rewards/rejected": -0.41535043716430664,
      "step": 2270
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.555375666468613e-06,
      "logits/chosen": -1.9682775735855103,
      "logits/rejected": -1.6195480823516846,
      "logps/chosen": -280.294677734375,
      "logps/rejected": -256.6422119140625,
      "loss": 0.198,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.31996774673461914,
      "rewards/margins": 0.1175389513373375,
      "rewards/rejected": -0.43750667572021484,
      "step": 2280
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.549397064625275e-06,
      "logits/chosen": -1.9350669384002686,
      "logits/rejected": -1.8133732080459595,
      "logps/chosen": -249.39791870117188,
      "logps/rejected": -271.91839599609375,
      "loss": 0.1115,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.44019174575805664,
      "rewards/margins": 0.09418530017137527,
      "rewards/rejected": -0.5343769788742065,
      "step": 2290
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.543382513775696e-06,
      "logits/chosen": -1.925415277481079,
      "logits/rejected": -1.594972014427185,
      "logps/chosen": -220.84228515625,
      "logps/rejected": -211.1090545654297,
      "loss": 0.1146,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.29334786534309387,
      "rewards/margins": 0.1734061986207962,
      "rewards/rejected": -0.4667540490627289,
      "step": 2300
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.5373321194226736e-06,
      "logits/chosen": -1.9605739116668701,
      "logits/rejected": -1.6391212940216064,
      "logps/chosen": -259.51397705078125,
      "logps/rejected": -275.52398681640625,
      "loss": 0.0944,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.35873842239379883,
      "rewards/margins": 0.1706543266773224,
      "rewards/rejected": -0.5293928384780884,
      "step": 2310
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.531245987697747e-06,
      "logits/chosen": -2.164452075958252,
      "logits/rejected": -1.8649688959121704,
      "logps/chosen": -275.1927185058594,
      "logps/rejected": -261.66571044921875,
      "loss": 0.1149,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.30722159147262573,
      "rewards/margins": 0.11033214628696442,
      "rewards/rejected": -0.4175536632537842,
      "step": 2320
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.525124225359332e-06,
      "logits/chosen": -2.0339162349700928,
      "logits/rejected": -1.7141647338867188,
      "logps/chosen": -264.71923828125,
      "logps/rejected": -232.3660125732422,
      "loss": 0.1714,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.36529964208602905,
      "rewards/margins": 0.15927986800670624,
      "rewards/rejected": -0.5245795249938965,
      "step": 2330
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.518966939790854e-06,
      "logits/chosen": -2.047182083129883,
      "logits/rejected": -1.7538772821426392,
      "logps/chosen": -294.2006530761719,
      "logps/rejected": -253.27822875976562,
      "loss": 0.2142,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.3689686954021454,
      "rewards/margins": 0.10386856645345688,
      "rewards/rejected": -0.47283726930618286,
      "step": 2340
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.512774238998858e-06,
      "logits/chosen": -1.9125845432281494,
      "logits/rejected": -1.6135631799697876,
      "logps/chosen": -214.25314331054688,
      "logps/rejected": -214.7703399658203,
      "loss": 0.1415,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.4276903569698334,
      "rewards/margins": 0.14196541905403137,
      "rewards/rejected": -0.56965571641922,
      "step": 2350
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.506546231611116e-06,
      "logits/chosen": -1.8613827228546143,
      "logits/rejected": -1.7953588962554932,
      "logps/chosen": -261.28729248046875,
      "logps/rejected": -297.7659606933594,
      "loss": 0.0916,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.47510260343551636,
      "rewards/margins": 0.18565957248210907,
      "rewards/rejected": -0.6607621908187866,
      "step": 2360
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.500283026874724e-06,
      "logits/chosen": -2.1421940326690674,
      "logits/rejected": -1.7659165859222412,
      "logps/chosen": -278.3591613769531,
      "logps/rejected": -258.3917236328125,
      "loss": 0.1477,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.37536367774009705,
      "rewards/margins": 0.14370563626289368,
      "rewards/rejected": -0.5190693140029907,
      "step": 2370
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.493984734654184e-06,
      "logits/chosen": -2.0281529426574707,
      "logits/rejected": -1.8278974294662476,
      "logps/chosen": -231.8653106689453,
      "logps/rejected": -220.0163116455078,
      "loss": 0.1186,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3119848370552063,
      "rewards/margins": 0.14551366865634918,
      "rewards/rejected": -0.4574984908103943,
      "step": 2380
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.487651465429475e-06,
      "logits/chosen": -2.177546501159668,
      "logits/rejected": -1.955910086631775,
      "logps/chosen": -248.14846801757812,
      "logps/rejected": -242.93801879882812,
      "loss": 0.2459,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.27779582142829895,
      "rewards/margins": 0.20447003841400146,
      "rewards/rejected": -0.4822658598423004,
      "step": 2390
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.481283330294118e-06,
      "logits/chosen": -1.8666213750839233,
      "logits/rejected": -1.3665393590927124,
      "logps/chosen": -222.7278594970703,
      "logps/rejected": -199.8502655029297,
      "loss": 0.1838,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.35523027181625366,
      "rewards/margins": 0.17108853161334991,
      "rewards/rejected": -0.5263187885284424,
      "step": 2400
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.474880440953224e-06,
      "logits/chosen": -1.9999549388885498,
      "logits/rejected": -1.8370367288589478,
      "logps/chosen": -192.89273071289062,
      "logps/rejected": -223.0652618408203,
      "loss": 0.076,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.2936464548110962,
      "rewards/margins": 0.11866404116153717,
      "rewards/rejected": -0.41231051087379456,
      "step": 2410
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.468442909721541e-06,
      "logits/chosen": -1.9979664087295532,
      "logits/rejected": -1.8018696308135986,
      "logps/chosen": -215.09585571289062,
      "logps/rejected": -227.23757934570312,
      "loss": 0.1393,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.29200148582458496,
      "rewards/margins": 0.10804203897714615,
      "rewards/rejected": -0.4000435471534729,
      "step": 2420
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.4619708495214735e-06,
      "logits/chosen": -2.1377148628234863,
      "logits/rejected": -1.6982520818710327,
      "logps/chosen": -303.42315673828125,
      "logps/rejected": -226.8529815673828,
      "loss": 0.1636,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.2613959312438965,
      "rewards/margins": 0.09153415262699127,
      "rewards/rejected": -0.35293012857437134,
      "step": 2430
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.455464373881112e-06,
      "logits/chosen": -1.9143394231796265,
      "logits/rejected": -1.7412408590316772,
      "logps/chosen": -237.32070922851562,
      "logps/rejected": -219.79324340820312,
      "loss": 0.1836,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.26246917247772217,
      "rewards/margins": 0.10806657373905182,
      "rewards/rejected": -0.3705357313156128,
      "step": 2440
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.4489235969322355e-06,
      "logits/chosen": -2.121340751647949,
      "logits/rejected": -1.8871597051620483,
      "logps/chosen": -190.05088806152344,
      "logps/rejected": -195.73678588867188,
      "loss": 0.1619,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.3710538446903229,
      "rewards/margins": 0.08781547844409943,
      "rewards/rejected": -0.4588693082332611,
      "step": 2450
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.442348633408312e-06,
      "logits/chosen": -1.9419047832489014,
      "logits/rejected": -1.5559477806091309,
      "logps/chosen": -200.26173400878906,
      "logps/rejected": -201.53518676757812,
      "loss": 0.2091,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.4545482099056244,
      "rewards/margins": 0.18632353842258453,
      "rewards/rejected": -0.6408717632293701,
      "step": 2460
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.435739598642484e-06,
      "logits/chosen": -2.016679525375366,
      "logits/rejected": -1.7562650442123413,
      "logps/chosen": -288.39404296875,
      "logps/rejected": -258.4088439941406,
      "loss": 0.0716,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.3196231424808502,
      "rewards/margins": 0.10171397030353546,
      "rewards/rejected": -0.4213371276855469,
      "step": 2470
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.429096608565547e-06,
      "logits/chosen": -1.806492805480957,
      "logits/rejected": -1.3850048780441284,
      "logps/chosen": -249.5767364501953,
      "logps/rejected": -222.6543426513672,
      "loss": 0.1036,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3670658767223358,
      "rewards/margins": 0.16034328937530518,
      "rewards/rejected": -0.5274091958999634,
      "step": 2480
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.422419779703916e-06,
      "logits/chosen": -2.2245254516601562,
      "logits/rejected": -1.865247130393982,
      "logps/chosen": -208.52560424804688,
      "logps/rejected": -190.5004425048828,
      "loss": 0.1605,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.3057347238063812,
      "rewards/margins": 0.11808328330516815,
      "rewards/rejected": -0.42381802201271057,
      "step": 2490
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.415709229177579e-06,
      "logits/chosen": -2.0980026721954346,
      "logits/rejected": -1.7750365734100342,
      "logps/chosen": -243.53494262695312,
      "logps/rejected": -276.8687438964844,
      "loss": 0.1665,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.253294974565506,
      "rewards/margins": 0.14929592609405518,
      "rewards/rejected": -0.40259090065956116,
      "step": 2500
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.408965074698048e-06,
      "logits/chosen": -2.0569772720336914,
      "logits/rejected": -1.7046623229980469,
      "logps/chosen": -227.9197998046875,
      "logps/rejected": -218.9676055908203,
      "loss": 0.1427,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.26669183373451233,
      "rewards/margins": 0.17199033498764038,
      "rewards/rejected": -0.4386821687221527,
      "step": 2510
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.402187434566286e-06,
      "logits/chosen": -1.852573037147522,
      "logits/rejected": -1.9005470275878906,
      "logps/chosen": -230.39346313476562,
      "logps/rejected": -252.2911376953125,
      "loss": 0.1693,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.3414091169834137,
      "rewards/margins": 0.12495288997888565,
      "rewards/rejected": -0.46636199951171875,
      "step": 2520
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.395376427670641e-06,
      "logits/chosen": -1.8688786029815674,
      "logits/rejected": -1.7985941171646118,
      "logps/chosen": -279.96453857421875,
      "logps/rejected": -322.30328369140625,
      "loss": 0.0915,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.4272969365119934,
      "rewards/margins": 0.16690507531166077,
      "rewards/rejected": -0.5942019820213318,
      "step": 2530
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.388532173484754e-06,
      "logits/chosen": -2.0615181922912598,
      "logits/rejected": -1.5270709991455078,
      "logps/chosen": -265.37591552734375,
      "logps/rejected": -240.03640747070312,
      "loss": 0.1222,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.28391411900520325,
      "rewards/margins": 0.16852129995822906,
      "rewards/rejected": -0.4524354040622711,
      "step": 2540
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.381654792065464e-06,
      "logits/chosen": -2.1687228679656982,
      "logits/rejected": -1.6613953113555908,
      "logps/chosen": -320.2466735839844,
      "logps/rejected": -226.2439727783203,
      "loss": 0.1973,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3444002866744995,
      "rewards/margins": 0.19264493882656097,
      "rewards/rejected": -0.5370452404022217,
      "step": 2550
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.374744404050706e-06,
      "logits/chosen": -2.1842730045318604,
      "logits/rejected": -1.622300386428833,
      "logps/chosen": -257.1001892089844,
      "logps/rejected": -265.92913818359375,
      "loss": 0.1575,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.26375168561935425,
      "rewards/margins": 0.22742655873298645,
      "rewards/rejected": -0.4911782145500183,
      "step": 2560
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.367801130657391e-06,
      "logits/chosen": -2.060206890106201,
      "logits/rejected": -1.610399603843689,
      "logps/chosen": -314.66949462890625,
      "logps/rejected": -269.8716125488281,
      "loss": 0.1212,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.37169474363327026,
      "rewards/margins": 0.1514434516429901,
      "rewards/rejected": -0.523138165473938,
      "step": 2570
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.3608250936792816e-06,
      "logits/chosen": -2.1835896968841553,
      "logits/rejected": -1.7747691869735718,
      "logps/chosen": -266.17095947265625,
      "logps/rejected": -240.68453979492188,
      "loss": 0.1395,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.31101933121681213,
      "rewards/margins": 0.16181661188602448,
      "rewards/rejected": -0.4728359580039978,
      "step": 2580
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.353816415484853e-06,
      "logits/chosen": -2.2123303413391113,
      "logits/rejected": -1.7858177423477173,
      "logps/chosen": -268.2467956542969,
      "logps/rejected": -238.09500122070312,
      "loss": 0.1023,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.394400417804718,
      "rewards/margins": 0.14999434351921082,
      "rewards/rejected": -0.5443947911262512,
      "step": 2590
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.346775219015152e-06,
      "logits/chosen": -2.0210156440734863,
      "logits/rejected": -1.6547054052352905,
      "logps/chosen": -292.4082946777344,
      "logps/rejected": -289.0941467285156,
      "loss": 0.1894,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.4957256317138672,
      "rewards/margins": 0.1184227466583252,
      "rewards/rejected": -0.6141483783721924,
      "step": 2600
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.339701627781633e-06,
      "logits/chosen": -1.906998634338379,
      "logits/rejected": -1.7141332626342773,
      "logps/chosen": -258.7335205078125,
      "logps/rejected": -251.07754516601562,
      "loss": 0.0969,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.3772934079170227,
      "rewards/margins": 0.09553851187229156,
      "rewards/rejected": -0.47283196449279785,
      "step": 2610
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.332595765863998e-06,
      "logits/chosen": -1.8866933584213257,
      "logits/rejected": -1.5196045637130737,
      "logps/chosen": -179.44906616210938,
      "logps/rejected": -181.3992156982422,
      "loss": 0.0869,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.376947820186615,
      "rewards/margins": 0.14659300446510315,
      "rewards/rejected": -0.5235407948493958,
      "step": 2620
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.325457757908016e-06,
      "logits/chosen": -1.9919402599334717,
      "logits/rejected": -1.5705921649932861,
      "logps/chosen": -260.714111328125,
      "logps/rejected": -241.1851043701172,
      "loss": 0.1377,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.39705362915992737,
      "rewards/margins": 0.15993081033229828,
      "rewards/rejected": -0.5569844841957092,
      "step": 2630
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.3182877291233395e-06,
      "logits/chosen": -1.9707670211791992,
      "logits/rejected": -1.495273232460022,
      "logps/chosen": -213.5478515625,
      "logps/rejected": -201.37350463867188,
      "loss": 0.1589,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.3921014368534088,
      "rewards/margins": 0.1725640594959259,
      "rewards/rejected": -0.5646654367446899,
      "step": 2640
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.311085805281306e-06,
      "logits/chosen": -1.9549287557601929,
      "logits/rejected": -1.630378007888794,
      "logps/chosen": -322.6145935058594,
      "logps/rejected": -297.1085205078125,
      "loss": 0.1446,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.41820430755615234,
      "rewards/margins": 0.1227729320526123,
      "rewards/rejected": -0.5409771800041199,
      "step": 2650
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.303852112712731e-06,
      "logits/chosen": -2.088381767272949,
      "logits/rejected": -1.612980604171753,
      "logps/chosen": -291.43255615234375,
      "logps/rejected": -240.37240600585938,
      "loss": 0.0888,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.3984852731227875,
      "rewards/margins": 0.1815904676914215,
      "rewards/rejected": -0.580075740814209,
      "step": 2660
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.2965867783056965e-06,
      "logits/chosen": -2.1672182083129883,
      "logits/rejected": -1.5168484449386597,
      "logps/chosen": -244.7864990234375,
      "logps/rejected": -220.2649688720703,
      "loss": 0.0939,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.24389946460723877,
      "rewards/margins": 0.23934423923492432,
      "rewards/rejected": -0.4832437038421631,
      "step": 2670
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.289289929503319e-06,
      "logits/chosen": -1.6916240453720093,
      "logits/rejected": -1.7404859066009521,
      "logps/chosen": -278.12933349609375,
      "logps/rejected": -313.3092956542969,
      "loss": 0.1081,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.38156622648239136,
      "rewards/margins": 0.08456975966691971,
      "rewards/rejected": -0.4661359190940857,
      "step": 2680
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.28196169430152e-06,
      "logits/chosen": -2.0158498287200928,
      "logits/rejected": -1.6960630416870117,
      "logps/chosen": -214.83908081054688,
      "logps/rejected": -203.30886840820312,
      "loss": 0.1377,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.23143234848976135,
      "rewards/margins": 0.13869646191596985,
      "rewards/rejected": -0.3701288104057312,
      "step": 2690
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.274602201246775e-06,
      "logits/chosen": -2.104879856109619,
      "logits/rejected": -1.873944878578186,
      "logps/chosen": -237.60867309570312,
      "logps/rejected": -253.7798309326172,
      "loss": 0.1325,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.32146531343460083,
      "rewards/margins": 0.13118143379688263,
      "rewards/rejected": -0.45264673233032227,
      "step": 2700
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.267211579433865e-06,
      "logits/chosen": -2.124311923980713,
      "logits/rejected": -1.6988563537597656,
      "logps/chosen": -248.6872100830078,
      "logps/rejected": -259.9998779296875,
      "loss": 0.123,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2223828136920929,
      "rewards/margins": 0.22472596168518066,
      "rewards/rejected": -0.44710874557495117,
      "step": 2710
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.259789958503606e-06,
      "logits/chosen": -1.808075189590454,
      "logits/rejected": -1.4258639812469482,
      "logps/chosen": -288.0134582519531,
      "logps/rejected": -270.99285888671875,
      "loss": 0.1181,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.48417121171951294,
      "rewards/margins": 0.1431351900100708,
      "rewards/rejected": -0.627306342124939,
      "step": 2720
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.252337468640578e-06,
      "logits/chosen": -1.8779484033584595,
      "logits/rejected": -1.4368770122528076,
      "logps/chosen": -182.4998321533203,
      "logps/rejected": -176.48782348632812,
      "loss": 0.1357,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.3015367388725281,
      "rewards/margins": 0.1921067237854004,
      "rewards/rejected": -0.49364346265792847,
      "step": 2730
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.244854240570844e-06,
      "logits/chosen": -1.8997386693954468,
      "logits/rejected": -1.638164758682251,
      "logps/chosen": -261.68792724609375,
      "logps/rejected": -275.615478515625,
      "loss": 0.1552,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.38944321870803833,
      "rewards/margins": 0.14735980331897736,
      "rewards/rejected": -0.536803126335144,
      "step": 2740
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.237340405559648e-06,
      "logits/chosen": -2.111983060836792,
      "logits/rejected": -1.8002418279647827,
      "logps/chosen": -276.4152526855469,
      "logps/rejected": -253.03970336914062,
      "loss": 0.1451,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.43595629930496216,
      "rewards/margins": 0.1535700559616089,
      "rewards/rejected": -0.589526355266571,
      "step": 2750
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.229796095409124e-06,
      "logits/chosen": -1.9869279861450195,
      "logits/rejected": -1.6609262228012085,
      "logps/chosen": -243.32666015625,
      "logps/rejected": -212.46484375,
      "loss": 0.1604,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.41807931661605835,
      "rewards/margins": 0.15857204794883728,
      "rewards/rejected": -0.5766514539718628,
      "step": 2760
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.222221442455975e-06,
      "logits/chosen": -1.7951889038085938,
      "logits/rejected": -1.6626970767974854,
      "logps/chosen": -270.48785400390625,
      "logps/rejected": -270.6944274902344,
      "loss": 0.1321,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5235955119132996,
      "rewards/margins": 0.13521410524845123,
      "rewards/rejected": -0.6588095426559448,
      "step": 2770
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.2146165795691565e-06,
      "logits/chosen": -2.0419421195983887,
      "logits/rejected": -1.6905419826507568,
      "logps/chosen": -274.6120910644531,
      "logps/rejected": -232.2598876953125,
      "loss": 0.1655,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5508221387863159,
      "rewards/margins": 0.0936468318104744,
      "rewards/rejected": -0.6444690823554993,
      "step": 2780
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.206981640147543e-06,
      "logits/chosen": -1.9829498529434204,
      "logits/rejected": -1.5640804767608643,
      "logps/chosen": -227.0321044921875,
      "logps/rejected": -209.35629272460938,
      "loss": 0.1261,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.38011568784713745,
      "rewards/margins": 0.20434486865997314,
      "rewards/rejected": -0.5844606161117554,
      "step": 2790
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.199316758117592e-06,
      "logits/chosen": -1.7994956970214844,
      "logits/rejected": -1.328776240348816,
      "logps/chosen": -219.972900390625,
      "logps/rejected": -209.31918334960938,
      "loss": 0.1531,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.39323854446411133,
      "rewards/margins": 0.17654483020305634,
      "rewards/rejected": -0.5697833895683289,
      "step": 2800
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.191622067930987e-06,
      "logits/chosen": -1.9456312656402588,
      "logits/rejected": -1.5701932907104492,
      "logps/chosen": -287.2892761230469,
      "logps/rejected": -293.2528076171875,
      "loss": 0.0978,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5495396852493286,
      "rewards/margins": 0.13945366442203522,
      "rewards/rejected": -0.6889933943748474,
      "step": 2810
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.1838977045622884e-06,
      "logits/chosen": -2.122058629989624,
      "logits/rejected": -1.8984161615371704,
      "logps/chosen": -296.1750793457031,
      "logps/rejected": -291.369384765625,
      "loss": 0.1468,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5357397794723511,
      "rewards/margins": 0.10289420932531357,
      "rewards/rejected": -0.6386340260505676,
      "step": 2820
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.1761438035065624e-06,
      "logits/chosen": -1.9847745895385742,
      "logits/rejected": -1.5842864513397217,
      "logps/chosen": -265.0511779785156,
      "logps/rejected": -265.59912109375,
      "loss": 0.171,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.4800674319267273,
      "rewards/margins": 0.20678965747356415,
      "rewards/rejected": -0.6868571639060974,
      "step": 2830
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.168360500777e-06,
      "logits/chosen": -1.9825668334960938,
      "logits/rejected": -1.8264620304107666,
      "logps/chosen": -270.49163818359375,
      "logps/rejected": -263.13958740234375,
      "loss": 0.1304,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.6147781014442444,
      "rewards/margins": 0.15809233486652374,
      "rewards/rejected": -0.7728704214096069,
      "step": 2840
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.160547932902536e-06,
      "logits/chosen": -1.998263955116272,
      "logits/rejected": -1.4734325408935547,
      "logps/chosen": -305.92901611328125,
      "logps/rejected": -268.21624755859375,
      "loss": 0.1526,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.6971688866615295,
      "rewards/margins": 0.1615949124097824,
      "rewards/rejected": -0.8587638139724731,
      "step": 2850
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.152706236925453e-06,
      "logits/chosen": -1.8893840312957764,
      "logits/rejected": -1.5303113460540771,
      "logps/chosen": -264.53741455078125,
      "logps/rejected": -240.29244995117188,
      "loss": 0.1553,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.7637086510658264,
      "rewards/margins": 0.09846861660480499,
      "rewards/rejected": -0.8621772527694702,
      "step": 2860
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.144835550398977e-06,
      "logits/chosen": -2.0382955074310303,
      "logits/rejected": -1.6921682357788086,
      "logps/chosen": -293.18017578125,
      "logps/rejected": -257.85516357421875,
      "loss": 0.1353,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5966418385505676,
      "rewards/margins": 0.15557792782783508,
      "rewards/rejected": -0.7522197961807251,
      "step": 2870
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.136936011384864e-06,
      "logits/chosen": -1.9725558757781982,
      "logits/rejected": -1.6349289417266846,
      "logps/chosen": -282.499267578125,
      "logps/rejected": -256.969970703125,
      "loss": 0.1057,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.5478484034538269,
      "rewards/margins": 0.13796645402908325,
      "rewards/rejected": -0.6858149170875549,
      "step": 2880
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.129007758450982e-06,
      "logits/chosen": -1.8872253894805908,
      "logits/rejected": -1.4205682277679443,
      "logps/chosen": -262.5359802246094,
      "logps/rejected": -235.87744140625,
      "loss": 0.1245,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.6125961542129517,
      "rewards/margins": 0.20152945816516876,
      "rewards/rejected": -0.8141257166862488,
      "step": 2890
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.121050930668871e-06,
      "logits/chosen": -2.047837734222412,
      "logits/rejected": -1.848854422569275,
      "logps/chosen": -243.146728515625,
      "logps/rejected": -233.36685180664062,
      "loss": 0.1355,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4853152334690094,
      "rewards/margins": 0.10266610234975815,
      "rewards/rejected": -0.5879813432693481,
      "step": 2900
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.113065667611319e-06,
      "logits/chosen": -2.1213645935058594,
      "logits/rejected": -1.6058502197265625,
      "logps/chosen": -282.0854187011719,
      "logps/rejected": -236.03237915039062,
      "loss": 0.133,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5683524012565613,
      "rewards/margins": 0.13890430331230164,
      "rewards/rejected": -0.7072567939758301,
      "step": 2910
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.105052109349896e-06,
      "logits/chosen": -1.961520791053772,
      "logits/rejected": -1.6209933757781982,
      "logps/chosen": -235.26229858398438,
      "logps/rejected": -211.75735473632812,
      "loss": 0.1818,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5107260942459106,
      "rewards/margins": 0.1078595519065857,
      "rewards/rejected": -0.6185856461524963,
      "step": 2920
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.097010396452511e-06,
      "logits/chosen": -1.7602100372314453,
      "logits/rejected": -1.5986255407333374,
      "logps/chosen": -221.6525115966797,
      "logps/rejected": -236.3260955810547,
      "loss": 0.099,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5530390739440918,
      "rewards/margins": 0.15462610125541687,
      "rewards/rejected": -0.7076650857925415,
      "step": 2930
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.088940669980936e-06,
      "logits/chosen": -1.833754301071167,
      "logits/rejected": -1.4045370817184448,
      "logps/chosen": -229.57437133789062,
      "logps/rejected": -231.51205444335938,
      "loss": 0.1555,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.5543310046195984,
      "rewards/margins": 0.21897678077220917,
      "rewards/rejected": -0.7733078002929688,
      "step": 2940
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.080843071488343e-06,
      "logits/chosen": -1.7528629302978516,
      "logits/rejected": -1.5004993677139282,
      "logps/chosen": -324.9387512207031,
      "logps/rejected": -295.79315185546875,
      "loss": 0.1262,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.6013875007629395,
      "rewards/margins": 0.061654604971408844,
      "rewards/rejected": -0.6630421876907349,
      "step": 2950
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.072717743016807e-06,
      "logits/chosen": -1.9874687194824219,
      "logits/rejected": -1.7957178354263306,
      "logps/chosen": -243.137939453125,
      "logps/rejected": -279.1502685546875,
      "loss": 0.1195,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.45743948221206665,
      "rewards/margins": 0.1679219901561737,
      "rewards/rejected": -0.6253615021705627,
      "step": 2960
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.064564827094827e-06,
      "logits/chosen": -2.1176095008850098,
      "logits/rejected": -1.8404308557510376,
      "logps/chosen": -246.0503692626953,
      "logps/rejected": -247.1431121826172,
      "loss": 0.1031,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.41762226819992065,
      "rewards/margins": 0.1749526411294937,
      "rewards/rejected": -0.592574954032898,
      "step": 2970
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.056384466734819e-06,
      "logits/chosen": -1.7445135116577148,
      "logits/rejected": -1.2714130878448486,
      "logps/chosen": -256.58575439453125,
      "logps/rejected": -227.7979736328125,
      "loss": 0.1514,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5112208127975464,
      "rewards/margins": 0.19044998288154602,
      "rewards/rejected": -0.70167076587677,
      "step": 2980
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.048176805430608e-06,
      "logits/chosen": -1.8863022327423096,
      "logits/rejected": -1.7394742965698242,
      "logps/chosen": -262.2151794433594,
      "logps/rejected": -257.8638000488281,
      "loss": 0.1186,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.4821039140224457,
      "rewards/margins": 0.17492111027240753,
      "rewards/rejected": -0.6570249795913696,
      "step": 2990
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.039941987154913e-06,
      "logits/chosen": -2.089552879333496,
      "logits/rejected": -1.5143569707870483,
      "logps/chosen": -255.2841339111328,
      "logps/rejected": -214.74667358398438,
      "loss": 0.1349,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.41421470046043396,
      "rewards/margins": 0.21286948025226593,
      "rewards/rejected": -0.6270841360092163,
      "step": 3000
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.031680156356822e-06,
      "logits/chosen": -2.152740478515625,
      "logits/rejected": -1.648754358291626,
      "logps/chosen": -298.00860595703125,
      "logps/rejected": -279.27215576171875,
      "loss": 0.088,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.45244675874710083,
      "rewards/margins": 0.17495819926261902,
      "rewards/rejected": -0.6274049282073975,
      "step": 3010
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.023391457959253e-06,
      "logits/chosen": -1.9636989831924438,
      "logits/rejected": -1.5016404390335083,
      "logps/chosen": -223.6481475830078,
      "logps/rejected": -208.9552001953125,
      "loss": 0.1553,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.3230467140674591,
      "rewards/margins": 0.15630824863910675,
      "rewards/rejected": -0.47935494780540466,
      "step": 3020
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.015076037356419e-06,
      "logits/chosen": -1.778830885887146,
      "logits/rejected": -1.504024624824524,
      "logps/chosen": -261.44805908203125,
      "logps/rejected": -237.22036743164062,
      "loss": 0.2152,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.4342494606971741,
      "rewards/margins": 0.06908075511455536,
      "rewards/rejected": -0.5033301711082458,
      "step": 3030
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.006734040411272e-06,
      "logits/chosen": -1.8755178451538086,
      "logits/rejected": -1.488073706626892,
      "logps/chosen": -233.17788696289062,
      "logps/rejected": -202.04881286621094,
      "loss": 0.1823,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4887026846408844,
      "rewards/margins": 0.13193130493164062,
      "rewards/rejected": -0.6206339597702026,
      "step": 3040
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.998365613452947e-06,
      "logits/chosen": -1.744222640991211,
      "logits/rejected": -1.7371858358383179,
      "logps/chosen": -213.4022979736328,
      "logps/rejected": -271.8200378417969,
      "loss": 0.1179,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.43451786041259766,
      "rewards/margins": 0.1312752515077591,
      "rewards/rejected": -0.5657930970191956,
      "step": 3050
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.9899709032741955e-06,
      "logits/chosen": -2.135042190551758,
      "logits/rejected": -1.7216142416000366,
      "logps/chosen": -226.56991577148438,
      "logps/rejected": -227.9345245361328,
      "loss": 0.1873,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.442862331867218,
      "rewards/margins": 0.20581674575805664,
      "rewards/rejected": -0.6486790776252747,
      "step": 3060
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.981550057128809e-06,
      "logits/chosen": -2.0724985599517822,
      "logits/rejected": -1.5731353759765625,
      "logps/chosen": -249.2626953125,
      "logps/rejected": -205.86062622070312,
      "loss": 0.0977,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.3592108488082886,
      "rewards/margins": 0.1588134914636612,
      "rewards/rejected": -0.5180243253707886,
      "step": 3070
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.973103222729037e-06,
      "logits/chosen": -1.9891624450683594,
      "logits/rejected": -1.8182249069213867,
      "logps/chosen": -238.1395263671875,
      "logps/rejected": -248.2894744873047,
      "loss": 0.1503,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.3745599687099457,
      "rewards/margins": 0.1474495679140091,
      "rewards/rejected": -0.522009551525116,
      "step": 3080
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.964630548242997e-06,
      "logits/chosen": -1.7449464797973633,
      "logits/rejected": -1.3936296701431274,
      "logps/chosen": -234.7018585205078,
      "logps/rejected": -203.70974731445312,
      "loss": 0.1525,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.3542986512184143,
      "rewards/margins": 0.15336424112319946,
      "rewards/rejected": -0.5076628923416138,
      "step": 3090
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.956132182292071e-06,
      "logits/chosen": -1.9436609745025635,
      "logits/rejected": -1.6176378726959229,
      "logps/chosen": -306.6236572265625,
      "logps/rejected": -285.08624267578125,
      "loss": 0.1063,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.48512548208236694,
      "rewards/margins": 0.1552090346813202,
      "rewards/rejected": -0.6403344869613647,
      "step": 3100
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.947608273948305e-06,
      "logits/chosen": -1.9343887567520142,
      "logits/rejected": -1.7104957103729248,
      "logps/chosen": -197.42628479003906,
      "logps/rejected": -188.55636596679688,
      "loss": 0.1288,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.4073019027709961,
      "rewards/margins": 0.13703104853630066,
      "rewards/rejected": -0.5443329811096191,
      "step": 3110
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.939058972731788e-06,
      "logits/chosen": -2.057648181915283,
      "logits/rejected": -1.7952289581298828,
      "logps/chosen": -184.43569946289062,
      "logps/rejected": -189.1503143310547,
      "loss": 0.156,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.3685847818851471,
      "rewards/margins": 0.1822405755519867,
      "rewards/rejected": -0.5508254170417786,
      "step": 3120
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.9304844286080356e-06,
      "logits/chosen": -1.9299640655517578,
      "logits/rejected": -1.5476219654083252,
      "logps/chosen": -265.6641540527344,
      "logps/rejected": -238.1685791015625,
      "loss": 0.0987,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.4248623847961426,
      "rewards/margins": 0.14193350076675415,
      "rewards/rejected": -0.566795825958252,
      "step": 3130
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.921884791985351e-06,
      "logits/chosen": -2.0945184230804443,
      "logits/rejected": -1.710710883140564,
      "logps/chosen": -289.3420715332031,
      "logps/rejected": -286.7973327636719,
      "loss": 0.1331,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.41171398758888245,
      "rewards/margins": 0.18646354973316193,
      "rewards/rejected": -0.5981774926185608,
      "step": 3140
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.913260213712195e-06,
      "logits/chosen": -2.005298614501953,
      "logits/rejected": -1.6120986938476562,
      "logps/chosen": -271.31695556640625,
      "logps/rejected": -271.58599853515625,
      "loss": 0.1618,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.39514026045799255,
      "rewards/margins": 0.18233473598957062,
      "rewards/rejected": -0.5774749517440796,
      "step": 3150
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.9046108450745365e-06,
      "logits/chosen": -1.9153554439544678,
      "logits/rejected": -1.6038618087768555,
      "logps/chosen": -244.7465057373047,
      "logps/rejected": -224.35009765625,
      "loss": 0.1676,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.4914848804473877,
      "rewards/margins": 0.13527749478816986,
      "rewards/rejected": -0.626762330532074,
      "step": 3160
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.895936837793195e-06,
      "logits/chosen": -2.1196136474609375,
      "logits/rejected": -1.9197361469268799,
      "logps/chosen": -272.1467590332031,
      "logps/rejected": -282.51373291015625,
      "loss": 0.1016,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.3920500576496124,
      "rewards/margins": 0.1278650313615799,
      "rewards/rejected": -0.5199151039123535,
      "step": 3170
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.887238344021187e-06,
      "logits/chosen": -1.9512029886245728,
      "logits/rejected": -1.5371229648590088,
      "logps/chosen": -229.00180053710938,
      "logps/rejected": -227.99154663085938,
      "loss": 0.1223,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.4040389060974121,
      "rewards/margins": 0.25474390387535095,
      "rewards/rejected": -0.6587827801704407,
      "step": 3180
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.878515516341051e-06,
      "logits/chosen": -1.8965469598770142,
      "logits/rejected": -1.5892069339752197,
      "logps/chosen": -312.53717041015625,
      "logps/rejected": -321.0782165527344,
      "loss": 0.1367,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.38909637928009033,
      "rewards/margins": 0.18783049285411835,
      "rewards/rejected": -0.5769269466400146,
      "step": 3190
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.869768507762174e-06,
      "logits/chosen": -1.8793041706085205,
      "logits/rejected": -1.5246042013168335,
      "logps/chosen": -206.9381866455078,
      "logps/rejected": -187.6604461669922,
      "loss": 0.1119,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.5590367913246155,
      "rewards/margins": 0.09890522062778473,
      "rewards/rejected": -0.6579420566558838,
      "step": 3200
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.860997471718103e-06,
      "logits/chosen": -2.1226069927215576,
      "logits/rejected": -1.4512499570846558,
      "logps/chosen": -252.26736450195312,
      "logps/rejected": -193.675048828125,
      "loss": 0.1548,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3612119257450104,
      "rewards/margins": 0.20598213374614716,
      "rewards/rejected": -0.5671939849853516,
      "step": 3210
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.852202562063861e-06,
      "logits/chosen": -1.9937137365341187,
      "logits/rejected": -1.6922187805175781,
      "logps/chosen": -273.3023376464844,
      "logps/rejected": -247.2145233154297,
      "loss": 0.1343,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.48126834630966187,
      "rewards/margins": 0.08357418328523636,
      "rewards/rejected": -0.5648424625396729,
      "step": 3220
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.843383933073243e-06,
      "logits/chosen": -1.9415899515151978,
      "logits/rejected": -1.546696424484253,
      "logps/chosen": -264.92291259765625,
      "logps/rejected": -254.01150512695312,
      "loss": 0.1317,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4355335235595703,
      "rewards/margins": 0.15428543090820312,
      "rewards/rejected": -0.5898188948631287,
      "step": 3230
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.834541739436111e-06,
      "logits/chosen": -2.0209739208221436,
      "logits/rejected": -1.7102893590927124,
      "logps/chosen": -217.0830078125,
      "logps/rejected": -216.00875854492188,
      "loss": 0.1922,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3377262353897095,
      "rewards/margins": 0.18450435996055603,
      "rewards/rejected": -0.5222306251525879,
      "step": 3240
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.82567613625568e-06,
      "logits/chosen": -2.1100401878356934,
      "logits/rejected": -2.0386178493499756,
      "logps/chosen": -306.51043701171875,
      "logps/rejected": -312.2314758300781,
      "loss": 0.087,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.28856438398361206,
      "rewards/margins": 0.09022587537765503,
      "rewards/rejected": -0.3787902593612671,
      "step": 3250
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.816787279045796e-06,
      "logits/chosen": -1.8298437595367432,
      "logits/rejected": -1.4992341995239258,
      "logps/chosen": -182.92562866210938,
      "logps/rejected": -192.92718505859375,
      "loss": 0.1166,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.2565813660621643,
      "rewards/margins": 0.21510104835033417,
      "rewards/rejected": -0.47168245911598206,
      "step": 3260
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.807875323728216e-06,
      "logits/chosen": -2.188213586807251,
      "logits/rejected": -1.716449499130249,
      "logps/chosen": -218.6008758544922,
      "logps/rejected": -216.4799041748047,
      "loss": 0.1432,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.24077515304088593,
      "rewards/margins": 0.20533184707164764,
      "rewards/rejected": -0.44610700011253357,
      "step": 3270
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.7989404266298614e-06,
      "logits/chosen": -1.775099515914917,
      "logits/rejected": -1.7529404163360596,
      "logps/chosen": -209.0091094970703,
      "logps/rejected": -222.7781982421875,
      "loss": 0.1195,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.41656428575515747,
      "rewards/margins": 0.1831093728542328,
      "rewards/rejected": -0.5996736288070679,
      "step": 3280
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.7899827444800824e-06,
      "logits/chosen": -1.975610375404358,
      "logits/rejected": -1.701148271560669,
      "logps/chosen": -320.1239929199219,
      "logps/rejected": -338.82916259765625,
      "loss": 0.1189,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5537024736404419,
      "rewards/margins": 0.1302124708890915,
      "rewards/rejected": -0.6839149594306946,
      "step": 3290
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7810024344079087e-06,
      "logits/chosen": -1.9031383991241455,
      "logits/rejected": -1.6330820322036743,
      "logps/chosen": -293.07061767578125,
      "logps/rejected": -310.0358581542969,
      "loss": 0.135,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7108369469642639,
      "rewards/margins": 0.24567703902721405,
      "rewards/rejected": -0.9565140008926392,
      "step": 3300
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7719996539392934e-06,
      "logits/chosen": -1.9635547399520874,
      "logits/rejected": -1.790226697921753,
      "logps/chosen": -293.0434875488281,
      "logps/rejected": -275.4864501953125,
      "loss": 0.1856,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.7340956926345825,
      "rewards/margins": 0.12796764075756073,
      "rewards/rejected": -0.8620632886886597,
      "step": 3310
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7629745609943454e-06,
      "logits/chosen": -1.8187742233276367,
      "logits/rejected": -1.5776069164276123,
      "logps/chosen": -245.385498046875,
      "logps/rejected": -267.95513916015625,
      "loss": 0.2146,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.7081323266029358,
      "rewards/margins": 0.12336601316928864,
      "rewards/rejected": -0.8314983248710632,
      "step": 3320
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7539273138845646e-06,
      "logits/chosen": -1.7952165603637695,
      "logits/rejected": -1.5672744512557983,
      "logps/chosen": -287.3976135253906,
      "logps/rejected": -305.989013671875,
      "loss": 0.121,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.774767279624939,
      "rewards/margins": 0.18739402294158936,
      "rewards/rejected": -0.9621612429618835,
      "step": 3330
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.744858071310063e-06,
      "logits/chosen": -1.732142686843872,
      "logits/rejected": -1.4198137521743774,
      "logps/chosen": -244.3140869140625,
      "logps/rejected": -244.147705078125,
      "loss": 0.1903,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.7413711547851562,
      "rewards/margins": 0.11797042936086655,
      "rewards/rejected": -0.8593416213989258,
      "step": 3340
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7357669923567796e-06,
      "logits/chosen": -2.07377290725708,
      "logits/rejected": -1.57345449924469,
      "logps/chosen": -321.8138427734375,
      "logps/rejected": -300.8492736816406,
      "loss": 0.1081,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.709524929523468,
      "rewards/margins": 0.2038944661617279,
      "rewards/rejected": -0.9134193658828735,
      "step": 3350
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.726654236493693e-06,
      "logits/chosen": -1.7073522806167603,
      "logits/rejected": -1.2896873950958252,
      "logps/chosen": -241.73583984375,
      "logps/rejected": -225.9768524169922,
      "loss": 0.1198,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7038867473602295,
      "rewards/margins": 0.18675477802753448,
      "rewards/rejected": -0.8906415700912476,
      "step": 3360
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.71751996357002e-06,
      "logits/chosen": -1.9721448421478271,
      "logits/rejected": -1.5171587467193604,
      "logps/chosen": -273.17547607421875,
      "logps/rejected": -271.9955749511719,
      "loss": 0.1155,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.6923592686653137,
      "rewards/margins": 0.09898529201745987,
      "rewards/rejected": -0.7913444638252258,
      "step": 3370
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.7083643338124148e-06,
      "logits/chosen": -1.9171966314315796,
      "logits/rejected": -1.3887364864349365,
      "logps/chosen": -230.0151824951172,
      "logps/rejected": -235.19412231445312,
      "loss": 0.1357,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.6308478116989136,
      "rewards/margins": 0.2189496010541916,
      "rewards/rejected": -0.8497973680496216,
      "step": 3380
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6991875078221566e-06,
      "logits/chosen": -1.9821815490722656,
      "logits/rejected": -1.4464524984359741,
      "logps/chosen": -298.43017578125,
      "logps/rejected": -268.8391418457031,
      "loss": 0.1016,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6277667284011841,
      "rewards/margins": 0.22099390625953674,
      "rewards/rejected": -0.8487606048583984,
      "step": 3390
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6899896465723352e-06,
      "logits/chosen": -1.9020191431045532,
      "logits/rejected": -1.55315101146698,
      "logps/chosen": -225.8837890625,
      "logps/rejected": -174.9435272216797,
      "loss": 0.1186,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.46708327531814575,
      "rewards/margins": 0.11991620063781738,
      "rewards/rejected": -0.5869995355606079,
      "step": 3400
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6807709114050224e-06,
      "logits/chosen": -1.8173465728759766,
      "logits/rejected": -1.6795600652694702,
      "logps/chosen": -272.4085388183594,
      "logps/rejected": -300.34197998046875,
      "loss": 0.1344,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.6119049787521362,
      "rewards/margins": 0.07957009226083755,
      "rewards/rejected": -0.6914750337600708,
      "step": 3410
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6715314640284465e-06,
      "logits/chosen": -1.95268976688385,
      "logits/rejected": -1.4205843210220337,
      "logps/chosen": -284.33819580078125,
      "logps/rejected": -281.9459533691406,
      "loss": 0.1827,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.6660246253013611,
      "rewards/margins": 0.1357964128255844,
      "rewards/rejected": -0.8018211126327515,
      "step": 3420
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6622714665141555e-06,
      "logits/chosen": -1.760504961013794,
      "logits/rejected": -1.5992462635040283,
      "logps/chosen": -255.13363647460938,
      "logps/rejected": -276.85931396484375,
      "loss": 0.1626,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6074542999267578,
      "rewards/margins": 0.1671931892633438,
      "rewards/rejected": -0.7746474742889404,
      "step": 3430
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6529910812941688e-06,
      "logits/chosen": -1.9736906290054321,
      "logits/rejected": -1.5405575037002563,
      "logps/chosen": -306.52337646484375,
      "logps/rejected": -294.29608154296875,
      "loss": 0.0816,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.6215208768844604,
      "rewards/margins": 0.19987761974334717,
      "rewards/rejected": -0.8213985562324524,
      "step": 3440
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6436904711581358e-06,
      "logits/chosen": -1.7905645370483398,
      "logits/rejected": -1.4196887016296387,
      "logps/chosen": -263.2611389160156,
      "logps/rejected": -266.86700439453125,
      "loss": 0.108,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.576808750629425,
      "rewards/margins": 0.20972958207130432,
      "rewards/rejected": -0.7865381240844727,
      "step": 3450
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.6343697992504745e-06,
      "logits/chosen": -1.8011541366577148,
      "logits/rejected": -1.528407096862793,
      "logps/chosen": -260.6270446777344,
      "logps/rejected": -244.4331817626953,
      "loss": 0.1388,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.6323608160018921,
      "rewards/margins": 0.14937646687030792,
      "rewards/rejected": -0.7817373275756836,
      "step": 3460
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.6250292290675103e-06,
      "logits/chosen": -1.8209354877471924,
      "logits/rejected": -1.6716169118881226,
      "logps/chosen": -279.5582275390625,
      "logps/rejected": -250.3738250732422,
      "loss": 0.2005,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.6096035242080688,
      "rewards/margins": 0.018474172800779343,
      "rewards/rejected": -0.6280776858329773,
      "step": 3470
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.6156689244546135e-06,
      "logits/chosen": -1.9151493310928345,
      "logits/rejected": -1.635745644569397,
      "logps/chosen": -304.9268493652344,
      "logps/rejected": -309.64410400390625,
      "loss": 0.1445,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5671188831329346,
      "rewards/margins": 0.12486696243286133,
      "rewards/rejected": -0.6919858455657959,
      "step": 3480
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.606289049603317e-06,
      "logits/chosen": -1.9070123434066772,
      "logits/rejected": -1.632115125656128,
      "logps/chosen": -217.40640258789062,
      "logps/rejected": -266.7468566894531,
      "loss": 0.1873,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5615465044975281,
      "rewards/margins": 0.10746095329523087,
      "rewards/rejected": -0.6690074801445007,
      "step": 3490
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.596889769048442e-06,
      "logits/chosen": -1.9706356525421143,
      "logits/rejected": -1.8182321786880493,
      "logps/chosen": -245.6522674560547,
      "logps/rejected": -260.2083740234375,
      "loss": 0.1345,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.4987486004829407,
      "rewards/margins": 0.12167295068502426,
      "rewards/rejected": -0.6204215288162231,
      "step": 3500
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.587471247665211e-06,
      "logits/chosen": -1.870273232460022,
      "logits/rejected": -1.4389684200286865,
      "logps/chosen": -282.3717346191406,
      "logps/rejected": -282.10955810546875,
      "loss": 0.1518,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5567461252212524,
      "rewards/margins": 0.15303365886211395,
      "rewards/rejected": -0.7097797393798828,
      "step": 3510
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.578033650666354e-06,
      "logits/chosen": -1.9102929830551147,
      "logits/rejected": -1.6725549697875977,
      "logps/chosen": -269.2619323730469,
      "logps/rejected": -265.96185302734375,
      "loss": 0.1345,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5339959859848022,
      "rewards/margins": 0.11029736697673798,
      "rewards/rejected": -0.6442933678627014,
      "step": 3520
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.56857714359921e-06,
      "logits/chosen": -1.9303925037384033,
      "logits/rejected": -1.450552225112915,
      "logps/chosen": -314.16229248046875,
      "logps/rejected": -267.4084167480469,
      "loss": 0.1383,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5251078605651855,
      "rewards/margins": 0.17124128341674805,
      "rewards/rejected": -0.6963491439819336,
      "step": 3530
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.5591018923428273e-06,
      "logits/chosen": -1.821260690689087,
      "logits/rejected": -1.5743091106414795,
      "logps/chosen": -229.2704620361328,
      "logps/rejected": -216.47482299804688,
      "loss": 0.1376,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.45546379685401917,
      "rewards/margins": 0.09873731434345245,
      "rewards/rejected": -0.5542011857032776,
      "step": 3540
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5496080631050494e-06,
      "logits/chosen": -1.9756050109863281,
      "logits/rejected": -1.7553752660751343,
      "logps/chosen": -242.6219940185547,
      "logps/rejected": -246.052978515625,
      "loss": 0.1605,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.44631749391555786,
      "rewards/margins": 0.12928064167499542,
      "rewards/rejected": -0.5755981206893921,
      "step": 3550
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5400958224196e-06,
      "logits/chosen": -1.7444331645965576,
      "logits/rejected": -1.646104097366333,
      "logps/chosen": -219.2833251953125,
      "logps/rejected": -236.79434204101562,
      "loss": 0.1119,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.3898042142391205,
      "rewards/margins": 0.09327594935894012,
      "rewards/rejected": -0.4830802083015442,
      "step": 3560
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5305653371431635e-06,
      "logits/chosen": -1.8823859691619873,
      "logits/rejected": -1.5607668161392212,
      "logps/chosen": -255.4270782470703,
      "logps/rejected": -250.3773651123047,
      "loss": 0.122,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.502920925617218,
      "rewards/margins": 0.15608903765678406,
      "rewards/rejected": -0.6590099930763245,
      "step": 3570
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.52101677445246e-06,
      "logits/chosen": -1.8646084070205688,
      "logits/rejected": -1.5643223524093628,
      "logps/chosen": -284.89697265625,
      "logps/rejected": -273.821044921875,
      "loss": 0.112,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.5394625067710876,
      "rewards/margins": 0.147272527217865,
      "rewards/rejected": -0.6867350339889526,
      "step": 3580
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5114503018413055e-06,
      "logits/chosen": -2.060659408569336,
      "logits/rejected": -1.689171552658081,
      "logps/chosen": -249.76150512695312,
      "logps/rejected": -236.51171875,
      "loss": 0.1114,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.4521172046661377,
      "rewards/margins": 0.12867510318756104,
      "rewards/rejected": -0.5807923078536987,
      "step": 3590
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5018660871176815e-06,
      "logits/chosen": -2.1247520446777344,
      "logits/rejected": -1.6458734273910522,
      "logps/chosen": -317.2967224121094,
      "logps/rejected": -260.74072265625,
      "loss": 0.1181,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.4817916750907898,
      "rewards/margins": 0.13223214447498322,
      "rewards/rejected": -0.6140238046646118,
      "step": 3600
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.4922642984007888e-06,
      "logits/chosen": -1.8655788898468018,
      "logits/rejected": -1.3581578731536865,
      "logps/chosen": -288.52496337890625,
      "logps/rejected": -243.9176483154297,
      "loss": 0.2081,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5236949324607849,
      "rewards/margins": 0.24149084091186523,
      "rewards/rejected": -0.7651858329772949,
      "step": 3610
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.4826451041180963e-06,
      "logits/chosen": -1.8614925146102905,
      "logits/rejected": -1.6801944971084595,
      "logps/chosen": -224.9128875732422,
      "logps/rejected": -241.3170928955078,
      "loss": 0.1374,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.49704447388648987,
      "rewards/margins": 0.1276617795228958,
      "rewards/rejected": -0.6247062683105469,
      "step": 3620
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4730086730023904e-06,
      "logits/chosen": -1.9381475448608398,
      "logits/rejected": -1.6607654094696045,
      "logps/chosen": -270.886474609375,
      "logps/rejected": -252.22116088867188,
      "loss": 0.1707,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.3630528151988983,
      "rewards/margins": 0.13112984597682953,
      "rewards/rejected": -0.49418267607688904,
      "step": 3630
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4633551740888122e-06,
      "logits/chosen": -2.1135964393615723,
      "logits/rejected": -1.4086754322052002,
      "logps/chosen": -322.132568359375,
      "logps/rejected": -269.0862731933594,
      "loss": 0.0716,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.3996971547603607,
      "rewards/margins": 0.2393627166748047,
      "rewards/rejected": -0.639059841632843,
      "step": 3640
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4536847767118926e-06,
      "logits/chosen": -1.9193264245986938,
      "logits/rejected": -1.5788618326187134,
      "logps/chosen": -240.32687377929688,
      "logps/rejected": -219.1731414794922,
      "loss": 0.1606,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.4481803774833679,
      "rewards/margins": 0.13924534618854523,
      "rewards/rejected": -0.587425708770752,
      "step": 3650
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.443997650502586e-06,
      "logits/chosen": -1.7943519353866577,
      "logits/rejected": -1.4917861223220825,
      "logps/chosen": -238.60647583007812,
      "logps/rejected": -199.71517944335938,
      "loss": 0.1366,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5198334455490112,
      "rewards/margins": 0.13573278486728668,
      "rewards/rejected": -0.6555660963058472,
      "step": 3660
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.434293965385287e-06,
      "logits/chosen": -1.852033257484436,
      "logits/rejected": -1.650202989578247,
      "logps/chosen": -262.7406311035156,
      "logps/rejected": -232.7073211669922,
      "loss": 0.1364,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.40459275245666504,
      "rewards/margins": 0.13505297899246216,
      "rewards/rejected": -0.5396457314491272,
      "step": 3670
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4245738915748584e-06,
      "logits/chosen": -2.122192621231079,
      "logits/rejected": -1.8862508535385132,
      "logps/chosen": -264.1258544921875,
      "logps/rejected": -277.42730712890625,
      "loss": 0.1551,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.42287102341651917,
      "rewards/margins": 0.10563336312770844,
      "rewards/rejected": -0.5285043716430664,
      "step": 3680
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4148375995736395e-06,
      "logits/chosen": -1.9229469299316406,
      "logits/rejected": -1.524235486984253,
      "logps/chosen": -297.52252197265625,
      "logps/rejected": -271.72381591796875,
      "loss": 0.1194,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6332093477249146,
      "rewards/margins": 0.17999136447906494,
      "rewards/rejected": -0.8132007718086243,
      "step": 3690
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4050852601684563e-06,
      "logits/chosen": -1.7078931331634521,
      "logits/rejected": -1.2806559801101685,
      "logps/chosen": -242.444091796875,
      "logps/rejected": -239.5855712890625,
      "loss": 0.1546,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5890045762062073,
      "rewards/margins": 0.20271596312522888,
      "rewards/rejected": -0.7917205095291138,
      "step": 3700
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3953170444276283e-06,
      "logits/chosen": -2.0124032497406006,
      "logits/rejected": -1.6335124969482422,
      "logps/chosen": -294.7106628417969,
      "logps/rejected": -276.8992004394531,
      "loss": 0.094,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5468276143074036,
      "rewards/margins": 0.18210643529891968,
      "rewards/rejected": -0.728934109210968,
      "step": 3710
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.385533123697966e-06,
      "logits/chosen": -1.6806570291519165,
      "logits/rejected": -1.582833170890808,
      "logps/chosen": -250.8227081298828,
      "logps/rejected": -278.7505187988281,
      "loss": 0.1042,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5766712427139282,
      "rewards/margins": 0.1681402027606964,
      "rewards/rejected": -0.744811475276947,
      "step": 3720
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.375733669601763e-06,
      "logits/chosen": -1.9780842065811157,
      "logits/rejected": -1.5905098915100098,
      "logps/chosen": -307.57269287109375,
      "logps/rejected": -258.7306213378906,
      "loss": 0.1604,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6550931930541992,
      "rewards/margins": 0.08839500695466995,
      "rewards/rejected": -0.7434881925582886,
      "step": 3730
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3659188540337884e-06,
      "logits/chosen": -2.0141289234161377,
      "logits/rejected": -1.7356878519058228,
      "logps/chosen": -234.14389038085938,
      "logps/rejected": -253.2290496826172,
      "loss": 0.1203,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.424830824136734,
      "rewards/margins": 0.12162800878286362,
      "rewards/rejected": -0.5464588403701782,
      "step": 3740
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3560888491582736e-06,
      "logits/chosen": -1.877969741821289,
      "logits/rejected": -1.6804864406585693,
      "logps/chosen": -217.326171875,
      "logps/rejected": -254.08627319335938,
      "loss": 0.1314,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.4663441777229309,
      "rewards/margins": 0.21716144680976868,
      "rewards/rejected": -0.683505654335022,
      "step": 3750
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3462438274058856e-06,
      "logits/chosen": -1.7631876468658447,
      "logits/rejected": -1.5371801853179932,
      "logps/chosen": -281.08563232421875,
      "logps/rejected": -305.8494873046875,
      "loss": 0.1094,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5819532871246338,
      "rewards/margins": 0.18844255805015564,
      "rewards/rejected": -0.770395815372467,
      "step": 3760
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3363839614707094e-06,
      "logits/chosen": -1.8546764850616455,
      "logits/rejected": -1.6666762828826904,
      "logps/chosen": -333.0240783691406,
      "logps/rejected": -356.2004699707031,
      "loss": 0.1544,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5481715798377991,
      "rewards/margins": 0.1419263780117035,
      "rewards/rejected": -0.6900979280471802,
      "step": 3770
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.326509424307214e-06,
      "logits/chosen": -1.9387702941894531,
      "logits/rejected": -1.7059547901153564,
      "logps/chosen": -264.86865234375,
      "logps/rejected": -257.7136535644531,
      "loss": 0.1748,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.57854825258255,
      "rewards/margins": 0.1645454317331314,
      "rewards/rejected": -0.7430936694145203,
      "step": 3780
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3166203891272204e-06,
      "logits/chosen": -2.0821845531463623,
      "logits/rejected": -1.6988433599472046,
      "logps/chosen": -336.27447509765625,
      "logps/rejected": -306.909912109375,
      "loss": 0.1274,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.48699769377708435,
      "rewards/margins": 0.19956621527671814,
      "rewards/rejected": -0.686564028263092,
      "step": 3790
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.306717029396863e-06,
      "logits/chosen": -1.900738000869751,
      "logits/rejected": -1.6325021982192993,
      "logps/chosen": -303.42596435546875,
      "logps/rejected": -268.81109619140625,
      "loss": 0.156,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5264959335327148,
      "rewards/margins": 0.08770108968019485,
      "rewards/rejected": -0.6141969561576843,
      "step": 3800
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2967995188335487e-06,
      "logits/chosen": -2.0487513542175293,
      "logits/rejected": -1.877976655960083,
      "logps/chosen": -187.6756134033203,
      "logps/rejected": -191.01234436035156,
      "loss": 0.1496,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.33429086208343506,
      "rewards/margins": 0.13553500175476074,
      "rewards/rejected": -0.4698258936405182,
      "step": 3810
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2868680314029056e-06,
      "logits/chosen": -2.096329689025879,
      "logits/rejected": -1.7965996265411377,
      "logps/chosen": -288.78558349609375,
      "logps/rejected": -270.60418701171875,
      "loss": 0.1086,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.39450058341026306,
      "rewards/margins": 0.10876335948705673,
      "rewards/rejected": -0.5032640099525452,
      "step": 3820
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2769227413157346e-06,
      "logits/chosen": -1.8762671947479248,
      "logits/rejected": -1.6266272068023682,
      "logps/chosen": -268.38555908203125,
      "logps/rejected": -219.3870849609375,
      "loss": 0.1707,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.29574501514434814,
      "rewards/margins": 0.18245580792427063,
      "rewards/rejected": -0.47820085287094116,
      "step": 3830
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.266963823024951e-06,
      "logits/chosen": -1.7480850219726562,
      "logits/rejected": -1.5056158304214478,
      "logps/chosen": -233.0159149169922,
      "logps/rejected": -227.66030883789062,
      "loss": 0.1303,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.3971914052963257,
      "rewards/margins": 0.13690955936908722,
      "rewards/rejected": -0.5341008901596069,
      "step": 3840
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2569914512225294e-06,
      "logits/chosen": -2.3906702995300293,
      "logits/rejected": -1.7254350185394287,
      "logps/chosen": -289.9384460449219,
      "logps/rejected": -234.61758422851562,
      "loss": 0.2108,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.31785309314727783,
      "rewards/margins": 0.15459677577018738,
      "rewards/rejected": -0.47244992852211,
      "step": 3850
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2470058008364335e-06,
      "logits/chosen": -1.940606713294983,
      "logits/rejected": -1.567697286605835,
      "logps/chosen": -300.01751708984375,
      "logps/rejected": -281.2144775390625,
      "loss": 0.1124,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.48682618141174316,
      "rewards/margins": 0.1738821119070053,
      "rewards/rejected": -0.6607083082199097,
      "step": 3860
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2370070470275493e-06,
      "logits/chosen": -1.9613168239593506,
      "logits/rejected": -1.6750881671905518,
      "logps/chosen": -239.41580200195312,
      "logps/rejected": -267.15350341796875,
      "loss": 0.1649,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.514406681060791,
      "rewards/margins": 0.1650439351797104,
      "rewards/rejected": -0.6794506311416626,
      "step": 3870
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.226995365186616e-06,
      "logits/chosen": -1.8612607717514038,
      "logits/rejected": -1.55705988407135,
      "logps/chosen": -228.90213012695312,
      "logps/rejected": -198.01766967773438,
      "loss": 0.199,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.4910704493522644,
      "rewards/margins": 0.11654232442378998,
      "rewards/rejected": -0.6076127290725708,
      "step": 3880
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.216970930931144e-06,
      "logits/chosen": -2.0845413208007812,
      "logits/rejected": -1.7625007629394531,
      "logps/chosen": -227.1043243408203,
      "logps/rejected": -240.2063751220703,
      "loss": 0.1225,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.4222134053707123,
      "rewards/margins": 0.11798451095819473,
      "rewards/rejected": -0.5401979684829712,
      "step": 3890
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.2069339201023398e-06,
      "logits/chosen": -2.0071322917938232,
      "logits/rejected": -1.961846947669983,
      "logps/chosen": -281.2358703613281,
      "logps/rejected": -288.28924560546875,
      "loss": 0.1283,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.5697019100189209,
      "rewards/margins": 0.07494824379682541,
      "rewards/rejected": -0.6446502208709717,
      "step": 3900
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.196884508762016e-06,
      "logits/chosen": -1.7892014980316162,
      "logits/rejected": -1.560509204864502,
      "logps/chosen": -263.98577880859375,
      "logps/rejected": -225.89773559570312,
      "loss": 0.2099,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.5703567266464233,
      "rewards/margins": 0.1076982244849205,
      "rewards/rejected": -0.6780549883842468,
      "step": 3910
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.186822873189508e-06,
      "logits/chosen": -1.8385652303695679,
      "logits/rejected": -1.474015712738037,
      "logps/chosen": -245.1987762451172,
      "logps/rejected": -248.1842498779297,
      "loss": 0.1091,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5351457595825195,
      "rewards/margins": 0.19029943645000458,
      "rewards/rejected": -0.7254451513290405,
      "step": 3920
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.1767491898785795e-06,
      "logits/chosen": -2.093048572540283,
      "logits/rejected": -1.6442959308624268,
      "logps/chosen": -260.1977233886719,
      "logps/rejected": -194.9726104736328,
      "loss": 0.1783,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.4470372200012207,
      "rewards/margins": 0.11479449272155762,
      "rewards/rejected": -0.5618317127227783,
      "step": 3930
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.166663635534325e-06,
      "logits/chosen": -1.9069023132324219,
      "logits/rejected": -1.7728145122528076,
      "logps/chosen": -251.0937957763672,
      "logps/rejected": -275.21026611328125,
      "loss": 0.0791,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.4740324914455414,
      "rewards/margins": 0.09531258046627045,
      "rewards/rejected": -0.5693451166152954,
      "step": 3940
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.1565663870700735e-06,
      "logits/chosen": -1.7941697835922241,
      "logits/rejected": -1.6212133169174194,
      "logps/chosen": -257.2917785644531,
      "logps/rejected": -283.5328369140625,
      "loss": 0.1458,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.6217584609985352,
      "rewards/margins": 0.12172119319438934,
      "rewards/rejected": -0.7434796690940857,
      "step": 3950
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1464576216042832e-06,
      "logits/chosen": -1.9793428182601929,
      "logits/rejected": -1.592930555343628,
      "logps/chosen": -305.0171813964844,
      "logps/rejected": -260.90753173828125,
      "loss": 0.1064,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.6241774559020996,
      "rewards/margins": 0.1830337643623352,
      "rewards/rejected": -0.8072112202644348,
      "step": 3960
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1363375164574343e-06,
      "logits/chosen": -1.9784101247787476,
      "logits/rejected": -1.7758643627166748,
      "logps/chosen": -251.56320190429688,
      "logps/rejected": -248.0037384033203,
      "loss": 0.1837,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.556761622428894,
      "rewards/margins": 0.11395516246557236,
      "rewards/rejected": -0.6707167625427246,
      "step": 3970
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.126206249148921e-06,
      "logits/chosen": -1.7647323608398438,
      "logits/rejected": -1.493837594985962,
      "logps/chosen": -300.1401062011719,
      "logps/rejected": -318.6617431640625,
      "loss": 0.1218,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.6313523054122925,
      "rewards/margins": 0.19563212990760803,
      "rewards/rejected": -0.8269845247268677,
      "step": 3980
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1160639973939337e-06,
      "logits/chosen": -2.1163768768310547,
      "logits/rejected": -1.8096988201141357,
      "logps/chosen": -311.0527038574219,
      "logps/rejected": -290.04046630859375,
      "loss": 0.2148,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5415524244308472,
      "rewards/margins": 0.08402875810861588,
      "rewards/rejected": -0.6255810856819153,
      "step": 3990
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.105910939100345e-06,
      "logits/chosen": -2.17586088180542,
      "logits/rejected": -1.680841088294983,
      "logps/chosen": -301.86749267578125,
      "logps/rejected": -298.6219177246094,
      "loss": 0.1436,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.5832773447036743,
      "rewards/margins": 0.17088885605335236,
      "rewards/rejected": -0.7541662454605103,
      "step": 4000
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.095747252365588e-06,
      "logits/chosen": -1.8582245111465454,
      "logits/rejected": -1.5364271402359009,
      "logps/chosen": -283.0442810058594,
      "logps/rejected": -277.89788818359375,
      "loss": 0.1558,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.5722718834877014,
      "rewards/margins": 0.09478892385959625,
      "rewards/rejected": -0.6670608520507812,
      "step": 4010
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.0855731154735326e-06,
      "logits/chosen": -1.6970354318618774,
      "logits/rejected": -1.434828281402588,
      "logps/chosen": -237.4835662841797,
      "logps/rejected": -244.4088134765625,
      "loss": 0.1824,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.6819806694984436,
      "rewards/margins": 0.17408792674541473,
      "rewards/rejected": -0.8560686111450195,
      "step": 4020
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.0753887068913545e-06,
      "logits/chosen": -1.896554946899414,
      "logits/rejected": -1.6122684478759766,
      "logps/chosen": -268.3304748535156,
      "logps/rejected": -254.9834747314453,
      "loss": 0.1759,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5617300271987915,
      "rewards/margins": 0.12652353942394257,
      "rewards/rejected": -0.6882535219192505,
      "step": 4030
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.0651942052664117e-06,
      "logits/chosen": -1.7737739086151123,
      "logits/rejected": -1.4258487224578857,
      "logps/chosen": -292.5776672363281,
      "logps/rejected": -271.95257568359375,
      "loss": 0.1443,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5280343890190125,
      "rewards/margins": 0.165075421333313,
      "rewards/rejected": -0.6931098699569702,
      "step": 4040
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0549897894231058e-06,
      "logits/chosen": -1.9711172580718994,
      "logits/rejected": -1.7488377094268799,
      "logps/chosen": -313.0440368652344,
      "logps/rejected": -290.74798583984375,
      "loss": 0.0978,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5002248883247375,
      "rewards/margins": 0.10817272961139679,
      "rewards/rejected": -0.6083976626396179,
      "step": 4050
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0447756383597438e-06,
      "logits/chosen": -1.9547706842422485,
      "logits/rejected": -1.4798098802566528,
      "logps/chosen": -224.11904907226562,
      "logps/rejected": -188.64633178710938,
      "loss": 0.1618,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.5285110473632812,
      "rewards/margins": 0.14819425344467163,
      "rewards/rejected": -0.6767052412033081,
      "step": 4060
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.034551931245404e-06,
      "logits/chosen": -1.902276635169983,
      "logits/rejected": -1.4849553108215332,
      "logps/chosen": -358.1360168457031,
      "logps/rejected": -280.324951171875,
      "loss": 0.1335,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.5318064093589783,
      "rewards/margins": 0.16225329041481018,
      "rewards/rejected": -0.6940596699714661,
      "step": 4070
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0243188474167884e-06,
      "logits/chosen": -1.960026502609253,
      "logits/rejected": -1.5699583292007446,
      "logps/chosen": -248.52590942382812,
      "logps/rejected": -228.6545867919922,
      "loss": 0.1788,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.39242681860923767,
      "rewards/margins": 0.1711881309747696,
      "rewards/rejected": -0.5636149644851685,
      "step": 4080
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.014076566375078e-06,
      "logits/chosen": -2.0379650592803955,
      "logits/rejected": -1.8301384449005127,
      "logps/chosen": -295.1788635253906,
      "logps/rejected": -255.16983032226562,
      "loss": 0.1771,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.46374598145484924,
      "rewards/margins": 0.09922705590724945,
      "rewards/rejected": -0.5629730224609375,
      "step": 4090
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.003825267782785e-06,
      "logits/chosen": -2.1784117221832275,
      "logits/rejected": -1.8092330694198608,
      "logps/chosen": -222.7117156982422,
      "logps/rejected": -212.63888549804688,
      "loss": 0.082,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.3608611524105072,
      "rewards/margins": 0.22283951938152313,
      "rewards/rejected": -0.5837006568908691,
      "step": 4100
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.993565131460602e-06,
      "logits/chosen": -1.8552563190460205,
      "logits/rejected": -1.5919392108917236,
      "logps/chosen": -258.01837158203125,
      "logps/rejected": -256.6094665527344,
      "loss": 0.1408,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.39110246300697327,
      "rewards/margins": 0.14050598442554474,
      "rewards/rejected": -0.5316083431243896,
      "step": 4110
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.9832963373842434e-06,
      "logits/chosen": -1.8685518503189087,
      "logits/rejected": -1.6710237264633179,
      "logps/chosen": -225.02157592773438,
      "logps/rejected": -231.95175170898438,
      "loss": 0.1088,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.4149894714355469,
      "rewards/margins": 0.11046002060174942,
      "rewards/rejected": -0.5254494547843933,
      "step": 4120
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.973019065681294e-06,
      "logits/chosen": -1.994270920753479,
      "logits/rejected": -1.5915766954421997,
      "logps/chosen": -237.4359893798828,
      "logps/rejected": -215.31338500976562,
      "loss": 0.1742,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.4488137364387512,
      "rewards/margins": 0.1414080411195755,
      "rewards/rejected": -0.5902218222618103,
      "step": 4130
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9627334966280474e-06,
      "logits/chosen": -2.0599796772003174,
      "logits/rejected": -1.6930656433105469,
      "logps/chosen": -272.7281799316406,
      "logps/rejected": -230.94287109375,
      "loss": 0.1445,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4368261396884918,
      "rewards/margins": 0.08637617528438568,
      "rewards/rejected": -0.5232023000717163,
      "step": 4140
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.952439810646341e-06,
      "logits/chosen": -1.9677894115447998,
      "logits/rejected": -1.6096127033233643,
      "logps/chosen": -250.97988891601562,
      "logps/rejected": -230.15469360351562,
      "loss": 0.1327,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4640297293663025,
      "rewards/margins": 0.13882431387901306,
      "rewards/rejected": -0.6028540134429932,
      "step": 4150
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.942138188300394e-06,
      "logits/chosen": -1.9296554327011108,
      "logits/rejected": -1.5421892404556274,
      "logps/chosen": -257.2073974609375,
      "logps/rejected": -258.81756591796875,
      "loss": 0.1773,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5215272307395935,
      "rewards/margins": 0.17221274971961975,
      "rewards/rejected": -0.6937400102615356,
      "step": 4160
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.931828810293642e-06,
      "logits/chosen": -2.06691312789917,
      "logits/rejected": -1.559309720993042,
      "logps/chosen": -252.13491821289062,
      "logps/rejected": -233.42626953125,
      "loss": 0.1603,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.46336793899536133,
      "rewards/margins": 0.18448057770729065,
      "rewards/rejected": -0.6478484869003296,
      "step": 4170
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.92151185746556e-06,
      "logits/chosen": -1.9915742874145508,
      "logits/rejected": -1.6672182083129883,
      "logps/chosen": -277.6402893066406,
      "logps/rejected": -279.9922790527344,
      "loss": 0.1693,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5202735066413879,
      "rewards/margins": 0.10429404675960541,
      "rewards/rejected": -0.6245675683021545,
      "step": 4180
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.911187510788498e-06,
      "logits/chosen": -1.9717572927474976,
      "logits/rejected": -1.7132648229599,
      "logps/chosen": -282.0147399902344,
      "logps/rejected": -253.1018524169922,
      "loss": 0.1024,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.41674551367759705,
      "rewards/margins": 0.13815635442733765,
      "rewards/rejected": -0.5549019575119019,
      "step": 4190
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9008559513645033e-06,
      "logits/chosen": -1.9843193292617798,
      "logits/rejected": -1.7526146173477173,
      "logps/chosen": -264.8548278808594,
      "logps/rejected": -241.67941284179688,
      "loss": 0.1593,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.43674373626708984,
      "rewards/margins": 0.1286846250295639,
      "rewards/rejected": -0.5654283761978149,
      "step": 4200
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.890517360422144e-06,
      "logits/chosen": -1.9424211978912354,
      "logits/rejected": -1.694551706314087,
      "logps/chosen": -256.5013732910156,
      "logps/rejected": -248.4475860595703,
      "loss": 0.1429,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.4047975540161133,
      "rewards/margins": 0.13096585869789124,
      "rewards/rejected": -0.5357634425163269,
      "step": 4210
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.880171919313327e-06,
      "logits/chosen": -1.9947586059570312,
      "logits/rejected": -1.516392469406128,
      "logps/chosen": -261.6585388183594,
      "logps/rejected": -189.2516632080078,
      "loss": 0.0992,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.3641483783721924,
      "rewards/margins": 0.1547776162624359,
      "rewards/rejected": -0.5189260244369507,
      "step": 4220
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.869819809510125e-06,
      "logits/chosen": -1.9108898639678955,
      "logits/rejected": -1.6370693445205688,
      "logps/chosen": -228.65298461914062,
      "logps/rejected": -231.56832885742188,
      "loss": 0.1458,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.44458943605422974,
      "rewards/margins": 0.1396978348493576,
      "rewards/rejected": -0.5842872858047485,
      "step": 4230
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8594612126015825e-06,
      "logits/chosen": -2.0889339447021484,
      "logits/rejected": -1.7717602252960205,
      "logps/chosen": -264.8407897949219,
      "logps/rejected": -302.21026611328125,
      "loss": 0.1162,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.3832109570503235,
      "rewards/margins": 0.19960837066173553,
      "rewards/rejected": -0.5828193426132202,
      "step": 4240
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.84909631029054e-06,
      "logits/chosen": -1.8553768396377563,
      "logits/rejected": -1.621514081954956,
      "logps/chosen": -263.96429443359375,
      "logps/rejected": -288.17620849609375,
      "loss": 0.1611,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.6084156036376953,
      "rewards/margins": 0.16580604016780853,
      "rewards/rejected": -0.7742215991020203,
      "step": 4250
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.838725284390441e-06,
      "logits/chosen": -1.8268849849700928,
      "logits/rejected": -1.6455726623535156,
      "logps/chosen": -269.8697204589844,
      "logps/rejected": -297.85150146484375,
      "loss": 0.112,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5495954751968384,
      "rewards/margins": 0.19235818088054657,
      "rewards/rejected": -0.7419536113739014,
      "step": 4260
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.828348316822144e-06,
      "logits/chosen": -1.8502800464630127,
      "logits/rejected": -1.6670821905136108,
      "logps/chosen": -209.65640258789062,
      "logps/rejected": -273.5523986816406,
      "loss": 0.113,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.46271175146102905,
      "rewards/margins": 0.1825498640537262,
      "rewards/rejected": -0.6452616453170776,
      "step": 4270
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.817965589610733e-06,
      "logits/chosen": -1.8152210712432861,
      "logits/rejected": -1.5677975416183472,
      "logps/chosen": -217.03182983398438,
      "logps/rejected": -245.7638702392578,
      "loss": 0.134,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5718420743942261,
      "rewards/margins": 0.1416967660188675,
      "rewards/rejected": -0.7135388255119324,
      "step": 4280
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.807577284882324e-06,
      "logits/chosen": -1.8648059368133545,
      "logits/rejected": -1.4172070026397705,
      "logps/chosen": -214.18099975585938,
      "logps/rejected": -227.8682861328125,
      "loss": 0.1346,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.47108086943626404,
      "rewards/margins": 0.21896126866340637,
      "rewards/rejected": -0.6900421380996704,
      "step": 4290
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.797183584860867e-06,
      "logits/chosen": -1.9184010028839111,
      "logits/rejected": -1.5958585739135742,
      "logps/chosen": -201.8664093017578,
      "logps/rejected": -200.3294219970703,
      "loss": 0.1953,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3741530776023865,
      "rewards/margins": 0.12563326954841614,
      "rewards/rejected": -0.499786376953125,
      "step": 4300
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7867846718649538e-06,
      "logits/chosen": -1.7121245861053467,
      "logits/rejected": -1.4752973318099976,
      "logps/chosen": -242.7071533203125,
      "logps/rejected": -287.45587158203125,
      "loss": 0.1076,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.39267921447753906,
      "rewards/margins": 0.21084478497505188,
      "rewards/rejected": -0.6035240292549133,
      "step": 4310
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7763807283046195e-06,
      "logits/chosen": -2.0703561305999756,
      "logits/rejected": -1.8529870510101318,
      "logps/chosen": -213.96029663085938,
      "logps/rejected": -224.46817016601562,
      "loss": 0.1417,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.37636810541152954,
      "rewards/margins": 0.15098202228546143,
      "rewards/rejected": -0.527350127696991,
      "step": 4320
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.76597193667814e-06,
      "logits/chosen": -2.061995029449463,
      "logits/rejected": -1.6790978908538818,
      "logps/chosen": -291.52508544921875,
      "logps/rejected": -278.72088623046875,
      "loss": 0.1316,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.35219401121139526,
      "rewards/margins": 0.10760994255542755,
      "rewards/rejected": -0.45980390906333923,
      "step": 4330
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7555584795688328e-06,
      "logits/chosen": -1.9189672470092773,
      "logits/rejected": -1.6146653890609741,
      "logps/chosen": -249.65347290039062,
      "logps/rejected": -247.68115234375,
      "loss": 0.1578,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.36561039090156555,
      "rewards/margins": 0.15556563436985016,
      "rewards/rejected": -0.5211759805679321,
      "step": 4340
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7451405396418544e-06,
      "logits/chosen": -1.940999984741211,
      "logits/rejected": -1.4690072536468506,
      "logps/chosen": -244.7572021484375,
      "logps/rejected": -207.64208984375,
      "loss": 0.1639,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.38119882345199585,
      "rewards/margins": 0.1026213988661766,
      "rewards/rejected": -0.48382019996643066,
      "step": 4350
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.734718299640994e-06,
      "logits/chosen": -2.1663918495178223,
      "logits/rejected": -1.8815300464630127,
      "logps/chosen": -250.9813232421875,
      "logps/rejected": -251.88296508789062,
      "loss": 0.1574,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.36953067779541016,
      "rewards/margins": 0.12235681712627411,
      "rewards/rejected": -0.4918874204158783,
      "step": 4360
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.724291942385472e-06,
      "logits/chosen": -2.146113395690918,
      "logits/rejected": -1.609834909439087,
      "logps/chosen": -304.42132568359375,
      "logps/rejected": -268.470947265625,
      "loss": 0.0921,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.35676488280296326,
      "rewards/margins": 0.1696079671382904,
      "rewards/rejected": -0.5263728499412537,
      "step": 4370
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.713861650766729e-06,
      "logits/chosen": -1.9884231090545654,
      "logits/rejected": -1.5890326499938965,
      "logps/chosen": -239.7928924560547,
      "logps/rejected": -229.9420623779297,
      "loss": 0.1299,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.38828951120376587,
      "rewards/margins": 0.1986490935087204,
      "rewards/rejected": -0.5869385600090027,
      "step": 4380
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.703427607745219e-06,
      "logits/chosen": -2.1583807468414307,
      "logits/rejected": -1.7095534801483154,
      "logps/chosen": -290.33868408203125,
      "logps/rejected": -269.4275817871094,
      "loss": 0.1828,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.3558133542537689,
      "rewards/margins": 0.13851602375507355,
      "rewards/rejected": -0.4943293631076813,
      "step": 4390
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6929899963472005e-06,
      "logits/chosen": -1.947405219078064,
      "logits/rejected": -1.5273383855819702,
      "logps/chosen": -235.3554229736328,
      "logps/rejected": -215.59786987304688,
      "loss": 0.1119,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.35831964015960693,
      "rewards/margins": 0.1973123699426651,
      "rewards/rejected": -0.5556319952011108,
      "step": 4400
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6825489996615278e-06,
      "logits/chosen": -1.8226381540298462,
      "logits/rejected": -1.5650604963302612,
      "logps/chosen": -223.3660888671875,
      "logps/rejected": -215.26327514648438,
      "loss": 0.1294,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.4000251293182373,
      "rewards/margins": 0.13242687284946442,
      "rewards/rejected": -0.5324519872665405,
      "step": 4410
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6721048008364343e-06,
      "logits/chosen": -1.968601942062378,
      "logits/rejected": -1.527261734008789,
      "logps/chosen": -263.6767578125,
      "logps/rejected": -242.66275024414062,
      "loss": 0.1512,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.357761949300766,
      "rewards/margins": 0.21579334139823914,
      "rewards/rejected": -0.5735553503036499,
      "step": 4420
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6616575830763247e-06,
      "logits/chosen": -2.044994831085205,
      "logits/rejected": -1.5942163467407227,
      "logps/chosen": -242.64852905273438,
      "logps/rejected": -245.306640625,
      "loss": 0.1581,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.3358486294746399,
      "rewards/margins": 0.13670720160007477,
      "rewards/rejected": -0.47255581617355347,
      "step": 4430
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.651207529638561e-06,
      "logits/chosen": -1.7535009384155273,
      "logits/rejected": -1.373928189277649,
      "logps/chosen": -261.17889404296875,
      "logps/rejected": -223.257080078125,
      "loss": 0.116,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.29363709688186646,
      "rewards/margins": 0.1494956910610199,
      "rewards/rejected": -0.44313281774520874,
      "step": 4440
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.640754823830242e-06,
      "logits/chosen": -2.192082405090332,
      "logits/rejected": -1.9141347408294678,
      "logps/chosen": -312.79693603515625,
      "logps/rejected": -237.3318634033203,
      "loss": 0.1115,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.32631200551986694,
      "rewards/margins": 0.0922769159078598,
      "rewards/rejected": -0.4185889661312103,
      "step": 4450
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.6302996490049983e-06,
      "logits/chosen": -2.042506456375122,
      "logits/rejected": -1.6036508083343506,
      "logps/chosen": -254.78213500976562,
      "logps/rejected": -255.03018188476562,
      "loss": 0.1262,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.35550904273986816,
      "rewards/margins": 0.10495875775814056,
      "rewards/rejected": -0.4604678153991699,
      "step": 4460
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.619842188559765e-06,
      "logits/chosen": -1.9425427913665771,
      "logits/rejected": -1.5557024478912354,
      "logps/chosen": -186.65274047851562,
      "logps/rejected": -203.18185424804688,
      "loss": 0.1508,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.35145506262779236,
      "rewards/margins": 0.2223198413848877,
      "rewards/rejected": -0.5737749338150024,
      "step": 4470
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.609382625931575e-06,
      "logits/chosen": -1.9245996475219727,
      "logits/rejected": -1.6936094760894775,
      "logps/chosen": -244.86160278320312,
      "logps/rejected": -269.80584716796875,
      "loss": 0.1606,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.4344099164009094,
      "rewards/margins": 0.16771261394023895,
      "rewards/rejected": -0.6021225452423096,
      "step": 4480
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.59892114459433e-06,
      "logits/chosen": -1.9008939266204834,
      "logits/rejected": -1.969109296798706,
      "logps/chosen": -238.5630340576172,
      "logps/rejected": -280.3179931640625,
      "loss": 0.1689,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.40993937849998474,
      "rewards/margins": 0.12407805770635605,
      "rewards/rejected": -0.5340174436569214,
      "step": 4490
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.588457928055592e-06,
      "logits/chosen": -1.6659586429595947,
      "logits/rejected": -1.2960065603256226,
      "logps/chosen": -255.4331512451172,
      "logps/rejected": -236.24594116210938,
      "loss": 0.1008,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.39769870042800903,
      "rewards/margins": 0.24053025245666504,
      "rewards/rejected": -0.6382290124893188,
      "step": 4500
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5779931598533624e-06,
      "logits/chosen": -1.9211695194244385,
      "logits/rejected": -1.555855393409729,
      "logps/chosen": -265.3733215332031,
      "logps/rejected": -246.8184814453125,
      "loss": 0.1628,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.4115406572818756,
      "rewards/margins": 0.12964625656604767,
      "rewards/rejected": -0.5411869287490845,
      "step": 4510
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.567527023552857e-06,
      "logits/chosen": -1.9409809112548828,
      "logits/rejected": -1.6917556524276733,
      "logps/chosen": -309.97760009765625,
      "logps/rejected": -264.5470886230469,
      "loss": 0.0704,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4299238324165344,
      "rewards/margins": 0.14857104420661926,
      "rewards/rejected": -0.5784948468208313,
      "step": 4520
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5570597027432907e-06,
      "logits/chosen": -1.9963619709014893,
      "logits/rejected": -1.5232570171356201,
      "logps/chosen": -248.9495391845703,
      "logps/rejected": -209.61312866210938,
      "loss": 0.1608,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.4026300311088562,
      "rewards/margins": 0.1366431713104248,
      "rewards/rejected": -0.5392731428146362,
      "step": 4530
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5465913810346575e-06,
      "logits/chosen": -1.7939636707305908,
      "logits/rejected": -1.6163822412490845,
      "logps/chosen": -263.76019287109375,
      "logps/rejected": -286.8879699707031,
      "loss": 0.1451,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.4445052146911621,
      "rewards/margins": 0.14057457447052002,
      "rewards/rejected": -0.5850798487663269,
      "step": 4540
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.536122242054507e-06,
      "logits/chosen": -1.9959796667099,
      "logits/rejected": -1.377286672592163,
      "logps/chosen": -249.66006469726562,
      "logps/rejected": -221.0861358642578,
      "loss": 0.1297,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.3975772261619568,
      "rewards/margins": 0.19428391754627228,
      "rewards/rejected": -0.5918611288070679,
      "step": 4550
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.525652469444727e-06,
      "logits/chosen": -2.10296368598938,
      "logits/rejected": -1.6731036901474,
      "logps/chosen": -211.2078857421875,
      "logps/rejected": -196.8761444091797,
      "loss": 0.1252,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.3557474613189697,
      "rewards/margins": 0.14848320186138153,
      "rewards/rejected": -0.5042306184768677,
      "step": 4560
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5151822468583165e-06,
      "logits/chosen": -1.8910295963287354,
      "logits/rejected": -1.441156268119812,
      "logps/chosen": -235.1863250732422,
      "logps/rejected": -203.9948272705078,
      "loss": 0.0839,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.3970240652561188,
      "rewards/margins": 0.21173615753650665,
      "rewards/rejected": -0.6087601780891418,
      "step": 4570
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5047117579561703e-06,
      "logits/chosen": -1.858319878578186,
      "logits/rejected": -1.6645057201385498,
      "logps/chosen": -318.26129150390625,
      "logps/rejected": -315.4615478515625,
      "loss": 0.1405,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5496889352798462,
      "rewards/margins": 0.14965248107910156,
      "rewards/rejected": -0.6993414163589478,
      "step": 4580
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.494241186403854e-06,
      "logits/chosen": -2.0156023502349854,
      "logits/rejected": -1.8666023015975952,
      "logps/chosen": -204.9974365234375,
      "logps/rejected": -199.46676635742188,
      "loss": 0.2009,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.38789471983909607,
      "rewards/margins": 0.09470699727535248,
      "rewards/rejected": -0.48260173201560974,
      "step": 4590
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.4837707158683833e-06,
      "logits/chosen": -1.733515739440918,
      "logits/rejected": -1.5892311334609985,
      "logps/chosen": -248.87283325195312,
      "logps/rejected": -265.2043762207031,
      "loss": 0.1352,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.5136191248893738,
      "rewards/margins": 0.11203992366790771,
      "rewards/rejected": -0.6256589889526367,
      "step": 4600
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.473300530015e-06,
      "logits/chosen": -2.1996002197265625,
      "logits/rejected": -1.857081651687622,
      "logps/chosen": -279.1620788574219,
      "logps/rejected": -275.622314453125,
      "loss": 0.1569,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.3480737805366516,
      "rewards/margins": 0.11938655376434326,
      "rewards/rejected": -0.4674603343009949,
      "step": 4610
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.4628308125039557e-06,
      "logits/chosen": -1.8926032781600952,
      "logits/rejected": -1.5367896556854248,
      "logps/chosen": -305.69732666015625,
      "logps/rejected": -276.2022705078125,
      "loss": 0.1795,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.41959348320961,
      "rewards/margins": 0.16269411146640778,
      "rewards/rejected": -0.582287609577179,
      "step": 4620
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.452361746987284e-06,
      "logits/chosen": -1.8755619525909424,
      "logits/rejected": -1.7180248498916626,
      "logps/chosen": -267.4087829589844,
      "logps/rejected": -282.9756774902344,
      "loss": 0.0926,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5185616612434387,
      "rewards/margins": 0.11375057697296143,
      "rewards/rejected": -0.6323122978210449,
      "step": 4630
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4418935171055818e-06,
      "logits/chosen": -1.9167985916137695,
      "logits/rejected": -1.6408929824829102,
      "logps/chosen": -202.68295288085938,
      "logps/rejected": -215.5853729248047,
      "loss": 0.1187,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.41076311469078064,
      "rewards/margins": 0.12577161192893982,
      "rewards/rejected": -0.5365347266197205,
      "step": 4640
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.43142630648479e-06,
      "logits/chosen": -1.941982626914978,
      "logits/rejected": -1.627986192703247,
      "logps/chosen": -308.0236511230469,
      "logps/rejected": -356.6058349609375,
      "loss": 0.0804,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5076587796211243,
      "rewards/margins": 0.15822356939315796,
      "rewards/rejected": -0.665882408618927,
      "step": 4650
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4209602987329685e-06,
      "logits/chosen": -1.7499468326568604,
      "logits/rejected": -1.2955760955810547,
      "logps/chosen": -229.9163055419922,
      "logps/rejected": -189.3719024658203,
      "loss": 0.0907,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.426525354385376,
      "rewards/margins": 0.18848739564418793,
      "rewards/rejected": -0.6150127649307251,
      "step": 4660
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.410495677437076e-06,
      "logits/chosen": -1.9118763208389282,
      "logits/rejected": -1.8392162322998047,
      "logps/chosen": -226.6531219482422,
      "logps/rejected": -220.9828338623047,
      "loss": 0.1856,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.3598722815513611,
      "rewards/margins": 0.13947324454784393,
      "rewards/rejected": -0.49934548139572144,
      "step": 4670
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.400032626159756e-06,
      "logits/chosen": -1.9028289318084717,
      "logits/rejected": -1.7155911922454834,
      "logps/chosen": -224.9667510986328,
      "logps/rejected": -226.68728637695312,
      "loss": 0.1633,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.3874308168888092,
      "rewards/margins": 0.13661661744117737,
      "rewards/rejected": -0.5240474343299866,
      "step": 4680
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.3895713284361065e-06,
      "logits/chosen": -2.0748324394226074,
      "logits/rejected": -1.5768488645553589,
      "logps/chosen": -245.1274871826172,
      "logps/rejected": -222.86328125,
      "loss": 0.1049,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.3151063919067383,
      "rewards/margins": 0.19956035912036896,
      "rewards/rejected": -0.514666736125946,
      "step": 4690
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.3791119677704676e-06,
      "logits/chosen": -2.194417715072632,
      "logits/rejected": -1.6311323642730713,
      "logps/chosen": -287.04107666015625,
      "logps/rejected": -247.39163208007812,
      "loss": 0.1247,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.31668201088905334,
      "rewards/margins": 0.18119558691978455,
      "rewards/rejected": -0.4978775978088379,
      "step": 4700
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3686547276332046e-06,
      "logits/chosen": -2.08101487159729,
      "logits/rejected": -1.6385080814361572,
      "logps/chosen": -264.02655029296875,
      "logps/rejected": -233.8690948486328,
      "loss": 0.1344,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.40405863523483276,
      "rewards/margins": 0.16924302279949188,
      "rewards/rejected": -0.5733017325401306,
      "step": 4710
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3581997914574807e-06,
      "logits/chosen": -1.9531478881835938,
      "logits/rejected": -1.559780240058899,
      "logps/chosen": -237.64059448242188,
      "logps/rejected": -223.76565551757812,
      "loss": 0.1204,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.39436954259872437,
      "rewards/margins": 0.16357873380184174,
      "rewards/rejected": -0.5579482913017273,
      "step": 4720
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3477473426360463e-06,
      "logits/chosen": -2.1687614917755127,
      "logits/rejected": -1.6831060647964478,
      "logps/chosen": -255.7380828857422,
      "logps/rejected": -224.1558837890625,
      "loss": 0.1026,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.3829661011695862,
      "rewards/margins": 0.23224039375782013,
      "rewards/rejected": -0.6152064800262451,
      "step": 4730
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.337297564518024e-06,
      "logits/chosen": -2.1666550636291504,
      "logits/rejected": -1.8726260662078857,
      "logps/chosen": -303.0262451171875,
      "logps/rejected": -272.85894775390625,
      "loss": 0.1091,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.3961569666862488,
      "rewards/margins": 0.1607007533311844,
      "rewards/rejected": -0.556857705116272,
      "step": 4740
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.326850640405684e-06,
      "logits/chosen": -1.865952730178833,
      "logits/rejected": -1.3388252258300781,
      "logps/chosen": -331.45611572265625,
      "logps/rejected": -297.70074462890625,
      "loss": 0.0822,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.35268011689186096,
      "rewards/margins": 0.2889634668827057,
      "rewards/rejected": -0.6416435837745667,
      "step": 4750
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3164067535512353e-06,
      "logits/chosen": -1.8777456283569336,
      "logits/rejected": -1.4981722831726074,
      "logps/chosen": -285.3711853027344,
      "logps/rejected": -237.86270141601562,
      "loss": 0.125,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.4790197014808655,
      "rewards/margins": 0.15934725105762482,
      "rewards/rejected": -0.6383669972419739,
      "step": 4760
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3059660871536123e-06,
      "logits/chosen": -1.6866099834442139,
      "logits/rejected": -1.4525415897369385,
      "logps/chosen": -237.55563354492188,
      "logps/rejected": -251.5872039794922,
      "loss": 0.1271,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5034652948379517,
      "rewards/margins": 0.1651829183101654,
      "rewards/rejected": -0.6686481833457947,
      "step": 4770
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.2955288243552543e-06,
      "logits/chosen": -2.0782809257507324,
      "logits/rejected": -1.6525154113769531,
      "logps/chosen": -335.6650085449219,
      "logps/rejected": -239.81674194335938,
      "loss": 0.1245,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.4862341284751892,
      "rewards/margins": 0.12166903167963028,
      "rewards/rejected": -0.6079031229019165,
      "step": 4780
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.285095148238899e-06,
      "logits/chosen": -1.9941129684448242,
      "logits/rejected": -1.7789087295532227,
      "logps/chosen": -281.5749206542969,
      "logps/rejected": -266.099609375,
      "loss": 0.1526,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.4684416651725769,
      "rewards/margins": 0.17475393414497375,
      "rewards/rejected": -0.643195629119873,
      "step": 4790
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2746652418243714e-06,
      "logits/chosen": -2.0029962062835693,
      "logits/rejected": -1.7494831085205078,
      "logps/chosen": -326.5981140136719,
      "logps/rejected": -310.0174255371094,
      "loss": 0.0976,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5547425150871277,
      "rewards/margins": 0.10356296598911285,
      "rewards/rejected": -0.6583055257797241,
      "step": 4800
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2642392880653677e-06,
      "logits/chosen": -1.9393142461776733,
      "logits/rejected": -1.916164755821228,
      "logps/chosen": -261.62750244140625,
      "logps/rejected": -236.9197235107422,
      "loss": 0.1388,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.49896836280822754,
      "rewards/margins": 0.07649464905261993,
      "rewards/rejected": -0.5754629969596863,
      "step": 4810
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.25381746984625e-06,
      "logits/chosen": -1.9654737710952759,
      "logits/rejected": -1.6573346853256226,
      "logps/chosen": -262.6412658691406,
      "logps/rejected": -285.51361083984375,
      "loss": 0.1336,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.49716243147850037,
      "rewards/margins": 0.175074964761734,
      "rewards/rejected": -0.6722373366355896,
      "step": 4820
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2433999699788404e-06,
      "logits/chosen": -2.004723072052002,
      "logits/rejected": -1.700979471206665,
      "logps/chosen": -265.5523986816406,
      "logps/rejected": -228.4165496826172,
      "loss": 0.124,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5388234257698059,
      "rewards/margins": 0.11548347771167755,
      "rewards/rejected": -0.654306948184967,
      "step": 4830
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2329869711992093e-06,
      "logits/chosen": -1.9097673892974854,
      "logits/rejected": -1.7621214389801025,
      "logps/chosen": -229.69906616210938,
      "logps/rejected": -267.36273193359375,
      "loss": 0.119,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5158060789108276,
      "rewards/margins": 0.14432090520858765,
      "rewards/rejected": -0.6601270437240601,
      "step": 4840
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2225786561644724e-06,
      "logits/chosen": -1.7414562702178955,
      "logits/rejected": -1.63266921043396,
      "logps/chosen": -258.82373046875,
      "logps/rejected": -264.71661376953125,
      "loss": 0.0976,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5453828573226929,
      "rewards/margins": 0.16923405230045319,
      "rewards/rejected": -0.7146168351173401,
      "step": 4850
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.212175207449589e-06,
      "logits/chosen": -1.9261242151260376,
      "logits/rejected": -1.4300401210784912,
      "logps/chosen": -220.4147186279297,
      "logps/rejected": -217.1300811767578,
      "loss": 0.1082,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.5556666254997253,
      "rewards/margins": 0.23020467162132263,
      "rewards/rejected": -0.7858712077140808,
      "step": 4860
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2017768075441544e-06,
      "logits/chosen": -1.9333302974700928,
      "logits/rejected": -1.7991241216659546,
      "logps/chosen": -260.7289123535156,
      "logps/rejected": -277.58941650390625,
      "loss": 0.089,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.6178120374679565,
      "rewards/margins": 0.15292315185070038,
      "rewards/rejected": -0.7707351446151733,
      "step": 4870
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.191383638849201e-06,
      "logits/chosen": -1.6100937128067017,
      "logits/rejected": -1.4786584377288818,
      "logps/chosen": -224.05224609375,
      "logps/rejected": -254.0164031982422,
      "loss": 0.1265,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5358814597129822,
      "rewards/margins": 0.21690325438976288,
      "rewards/rejected": -0.7527847290039062,
      "step": 4880
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.180995883674003e-06,
      "logits/chosen": -2.0832412242889404,
      "logits/rejected": -1.8438653945922852,
      "logps/chosen": -301.72808837890625,
      "logps/rejected": -263.3715515136719,
      "loss": 0.1293,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.5527404546737671,
      "rewards/margins": 0.11939278990030289,
      "rewards/rejected": -0.6721332669258118,
      "step": 4890
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1706137242328708e-06,
      "logits/chosen": -1.8641271591186523,
      "logits/rejected": -1.7302074432373047,
      "logps/chosen": -223.2054443359375,
      "logps/rejected": -246.4233856201172,
      "loss": 0.1346,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.5136536359786987,
      "rewards/margins": 0.13826540112495422,
      "rewards/rejected": -0.6519190669059753,
      "step": 4900
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1602373426419593e-06,
      "logits/chosen": -2.0203075408935547,
      "logits/rejected": -1.7125927209854126,
      "logps/chosen": -247.02804565429688,
      "logps/rejected": -248.95193481445312,
      "loss": 0.156,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5652672648429871,
      "rewards/margins": 0.21123281121253967,
      "rewards/rejected": -0.7765001058578491,
      "step": 4910
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.149866920916075e-06,
      "logits/chosen": -1.9118244647979736,
      "logits/rejected": -1.6481826305389404,
      "logps/chosen": -287.49969482421875,
      "logps/rejected": -266.9239196777344,
      "loss": 0.098,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.507907509803772,
      "rewards/margins": 0.15150879323482513,
      "rewards/rejected": -0.6594163179397583,
      "step": 4920
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1395026409654776e-06,
      "logits/chosen": -2.052753448486328,
      "logits/rejected": -1.707918405532837,
      "logps/chosen": -287.81787109375,
      "logps/rejected": -250.28988647460938,
      "loss": 0.1311,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5523009896278381,
      "rewards/margins": 0.10048754513263702,
      "rewards/rejected": -0.6527885794639587,
      "step": 4930
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.129144684592694e-06,
      "logits/chosen": -1.8895385265350342,
      "logits/rejected": -1.4638690948486328,
      "logps/chosen": -229.0388946533203,
      "logps/rejected": -215.0070343017578,
      "loss": 0.1343,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5384308695793152,
      "rewards/margins": 0.1393643468618393,
      "rewards/rejected": -0.6777952909469604,
      "step": 4940
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1187932334893282e-06,
      "logits/chosen": -2.058537721633911,
      "logits/rejected": -1.811730146408081,
      "logps/chosen": -238.6881866455078,
      "logps/rejected": -239.40774536132812,
      "loss": 0.1401,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.5211200714111328,
      "rewards/margins": 0.11581333726644516,
      "rewards/rejected": -0.636933445930481,
      "step": 4950
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.1084484692328726e-06,
      "logits/chosen": -1.8077147006988525,
      "logits/rejected": -1.6539599895477295,
      "logps/chosen": -324.1515808105469,
      "logps/rejected": -339.98333740234375,
      "loss": 0.0679,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.667458176612854,
      "rewards/margins": 0.1506289839744568,
      "rewards/rejected": -0.8180869817733765,
      "step": 4960
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0981105732835227e-06,
      "logits/chosen": -1.9896256923675537,
      "logits/rejected": -1.4527660608291626,
      "logps/chosen": -269.33746337890625,
      "logps/rejected": -214.31729125976562,
      "loss": 0.1888,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5238735675811768,
      "rewards/margins": 0.20568545162677765,
      "rewards/rejected": -0.729559063911438,
      "step": 4970
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.087779726980999e-06,
      "logits/chosen": -2.0337650775909424,
      "logits/rejected": -1.6298131942749023,
      "logps/chosen": -297.9162292480469,
      "logps/rejected": -280.47637939453125,
      "loss": 0.0769,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5461810827255249,
      "rewards/margins": 0.18490315973758698,
      "rewards/rejected": -0.7310842275619507,
      "step": 4980
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.077456111541359e-06,
      "logits/chosen": -1.915443778038025,
      "logits/rejected": -1.500583291053772,
      "logps/chosen": -290.4164733886719,
      "logps/rejected": -241.181396484375,
      "loss": 0.1659,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.49130645394325256,
      "rewards/margins": 0.18306657671928406,
      "rewards/rejected": -0.6743730306625366,
      "step": 4990
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.067139908053821e-06,
      "logits/chosen": -2.1264405250549316,
      "logits/rejected": -1.8090966939926147,
      "logps/chosen": -281.15185546875,
      "logps/rejected": -267.20489501953125,
      "loss": 0.1224,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.43849506974220276,
      "rewards/margins": 0.1732708215713501,
      "rewards/rejected": -0.6117658019065857,
      "step": 5000
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.056831297477592e-06,
      "logits/chosen": -1.8897788524627686,
      "logits/rejected": -1.8385932445526123,
      "logps/chosen": -329.4559020996094,
      "logps/rejected": -293.82940673828125,
      "loss": 0.136,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.678800106048584,
      "rewards/margins": 0.08193562924861908,
      "rewards/rejected": -0.7607358694076538,
      "step": 5010
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.046530460638687e-06,
      "logits/chosen": -2.099050760269165,
      "logits/rejected": -1.724473237991333,
      "logps/chosen": -318.06732177734375,
      "logps/rejected": -279.4828796386719,
      "loss": 0.1347,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5295699834823608,
      "rewards/margins": 0.18352551758289337,
      "rewards/rejected": -0.7130955457687378,
      "step": 5020
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.036237578226761e-06,
      "logits/chosen": -1.7614809274673462,
      "logits/rejected": -1.4049193859100342,
      "logps/chosen": -224.44375610351562,
      "logps/rejected": -221.57180786132812,
      "loss": 0.1367,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5156166553497314,
      "rewards/margins": 0.1933528184890747,
      "rewards/rejected": -0.7089694738388062,
      "step": 5030
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0259528307919385e-06,
      "logits/chosen": -2.024557590484619,
      "logits/rejected": -1.4408557415008545,
      "logps/chosen": -298.25,
      "logps/rejected": -232.6403045654297,
      "loss": 0.149,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.43420344591140747,
      "rewards/margins": 0.20419923961162567,
      "rewards/rejected": -0.6384027004241943,
      "step": 5040
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.015676398741644e-06,
      "logits/chosen": -1.9110151529312134,
      "logits/rejected": -1.3820592164993286,
      "logps/chosen": -328.24456787109375,
      "logps/rejected": -274.53765869140625,
      "loss": 0.1629,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5336133241653442,
      "rewards/margins": 0.20271578431129456,
      "rewards/rejected": -0.7363291382789612,
      "step": 5050
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.005408462337443e-06,
      "logits/chosen": -2.0011394023895264,
      "logits/rejected": -1.5512298345565796,
      "logps/chosen": -250.78280639648438,
      "logps/rejected": -245.1263885498047,
      "loss": 0.1404,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.49497905373573303,
      "rewards/margins": 0.21909329295158386,
      "rewards/rejected": -0.7140722870826721,
      "step": 5060
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.9951492016918745e-06,
      "logits/chosen": -1.9097583293914795,
      "logits/rejected": -1.5266954898834229,
      "logps/chosen": -195.3928985595703,
      "logps/rejected": -203.9522247314453,
      "loss": 0.1487,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5452216863632202,
      "rewards/margins": 0.17070701718330383,
      "rewards/rejected": -0.7159286737442017,
      "step": 5070
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.984898796765294e-06,
      "logits/chosen": -1.871835470199585,
      "logits/rejected": -1.4449571371078491,
      "logps/chosen": -198.09242248535156,
      "logps/rejected": -197.41738891601562,
      "loss": 0.1232,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.4947785437107086,
      "rewards/margins": 0.17074665427207947,
      "rewards/rejected": -0.6655251979827881,
      "step": 5080
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.974657427362717e-06,
      "logits/chosen": -1.817120909690857,
      "logits/rejected": -1.635406494140625,
      "logps/chosen": -292.73358154296875,
      "logps/rejected": -272.16900634765625,
      "loss": 0.1009,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5638500452041626,
      "rewards/margins": 0.10627535730600357,
      "rewards/rejected": -0.670125424861908,
      "step": 5090
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.9644252731306653e-06,
      "logits/chosen": -1.8737514019012451,
      "logits/rejected": -1.5180460214614868,
      "logps/chosen": -367.1842956542969,
      "logps/rejected": -313.0085144042969,
      "loss": 0.1218,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6011655330657959,
      "rewards/margins": 0.1648952215909958,
      "rewards/rejected": -0.7660607099533081,
      "step": 5100
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.954202513554013e-06,
      "logits/chosen": -1.9836708307266235,
      "logits/rejected": -1.797654390335083,
      "logps/chosen": -243.0941162109375,
      "logps/rejected": -266.05535888671875,
      "loss": 0.156,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.4675242006778717,
      "rewards/margins": 0.17585307359695435,
      "rewards/rejected": -0.6433773636817932,
      "step": 5110
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.943989327952841e-06,
      "logits/chosen": -2.0395166873931885,
      "logits/rejected": -1.6357864141464233,
      "logps/chosen": -350.53411865234375,
      "logps/rejected": -324.0770568847656,
      "loss": 0.0882,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5562902092933655,
      "rewards/margins": 0.15983954071998596,
      "rewards/rejected": -0.7161296606063843,
      "step": 5120
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.9337858954792917e-06,
      "logits/chosen": -1.8916152715682983,
      "logits/rejected": -1.6612498760223389,
      "logps/chosen": -255.1393280029297,
      "logps/rejected": -268.5872802734375,
      "loss": 0.1109,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5940229892730713,
      "rewards/margins": 0.24639251828193665,
      "rewards/rejected": -0.8404154777526855,
      "step": 5130
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.9235923951144246e-06,
      "logits/chosen": -1.9813220500946045,
      "logits/rejected": -1.6410853862762451,
      "logps/chosen": -291.50323486328125,
      "logps/rejected": -276.9924011230469,
      "loss": 0.1087,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.531834602355957,
      "rewards/margins": 0.15474644303321838,
      "rewards/rejected": -0.686581015586853,
      "step": 5140
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.9134090056650764e-06,
      "logits/chosen": -2.085635185241699,
      "logits/rejected": -1.7884467840194702,
      "logps/chosen": -284.26025390625,
      "logps/rejected": -268.23699951171875,
      "loss": 0.1903,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.5530114769935608,
      "rewards/margins": 0.16405172646045685,
      "rewards/rejected": -0.7170631885528564,
      "step": 5150
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.9032359057607272e-06,
      "logits/chosen": -2.1484217643737793,
      "logits/rejected": -1.5867314338684082,
      "logps/chosen": -338.3865661621094,
      "logps/rejected": -318.114990234375,
      "loss": 0.0867,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.4841574728488922,
      "rewards/margins": 0.2181321382522583,
      "rewards/rejected": -0.7022895812988281,
      "step": 5160
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8930732738503652e-06,
      "logits/chosen": -1.991681694984436,
      "logits/rejected": -1.7101647853851318,
      "logps/chosen": -253.91015625,
      "logps/rejected": -210.7447052001953,
      "loss": 0.1055,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5010813474655151,
      "rewards/margins": 0.16278687119483948,
      "rewards/rejected": -0.6638683080673218,
      "step": 5170
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8829212881993553e-06,
      "logits/chosen": -2.1374099254608154,
      "logits/rejected": -1.8165279626846313,
      "logps/chosen": -280.1686096191406,
      "logps/rejected": -253.91220092773438,
      "loss": 0.0657,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.49169284105300903,
      "rewards/margins": 0.12237177044153214,
      "rewards/rejected": -0.6140645742416382,
      "step": 5180
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.872780126886316e-06,
      "logits/chosen": -2.1169066429138184,
      "logits/rejected": -1.6588671207427979,
      "logps/chosen": -282.9520568847656,
      "logps/rejected": -245.0965118408203,
      "loss": 0.1101,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4680394232273102,
      "rewards/margins": 0.19195261597633362,
      "rewards/rejected": -0.659991979598999,
      "step": 5190
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8626499677999915e-06,
      "logits/chosen": -1.8921172618865967,
      "logits/rejected": -1.8109171390533447,
      "logps/chosen": -260.7506408691406,
      "logps/rejected": -277.9801025390625,
      "loss": 0.132,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5560612082481384,
      "rewards/margins": 0.13018205761909485,
      "rewards/rejected": -0.6862432360649109,
      "step": 5200
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8525309886361332e-06,
      "logits/chosen": -1.9643144607543945,
      "logits/rejected": -1.4548659324645996,
      "logps/chosen": -215.2502899169922,
      "logps/rejected": -212.14553833007812,
      "loss": 0.1047,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.5360875725746155,
      "rewards/margins": 0.2591664493083954,
      "rewards/rejected": -0.7952540516853333,
      "step": 5210
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8424233668943844e-06,
      "logits/chosen": -1.8108913898468018,
      "logits/rejected": -1.6711933612823486,
      "logps/chosen": -245.8279571533203,
      "logps/rejected": -260.3638610839844,
      "loss": 0.1462,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5376258492469788,
      "rewards/margins": 0.17290274798870087,
      "rewards/rejected": -0.710528552532196,
      "step": 5220
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8323272798751629e-06,
      "logits/chosen": -1.8469750881195068,
      "logits/rejected": -1.6417795419692993,
      "logps/chosen": -267.5218811035156,
      "logps/rejected": -255.52890014648438,
      "loss": 0.1275,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.6079937219619751,
      "rewards/margins": 0.1432160884141922,
      "rewards/rejected": -0.7512098550796509,
      "step": 5230
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.822242904676552e-06,
      "logits/chosen": -1.8902513980865479,
      "logits/rejected": -1.6447397470474243,
      "logps/chosen": -224.96902465820312,
      "logps/rejected": -236.59140014648438,
      "loss": 0.0791,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5813908576965332,
      "rewards/margins": 0.11333571374416351,
      "rewards/rejected": -0.6947265863418579,
      "step": 5240
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8121704181911989e-06,
      "logits/chosen": -2.0475192070007324,
      "logits/rejected": -1.7533676624298096,
      "logps/chosen": -322.51141357421875,
      "logps/rejected": -286.36376953125,
      "loss": 0.1171,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.5761255025863647,
      "rewards/margins": 0.12248637527227402,
      "rewards/rejected": -0.698611855506897,
      "step": 5250
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8021099971032046e-06,
      "logits/chosen": -1.731256127357483,
      "logits/rejected": -1.3307876586914062,
      "logps/chosen": -248.487060546875,
      "logps/rejected": -219.0608367919922,
      "loss": 0.1045,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5520157217979431,
      "rewards/margins": 0.1579941362142563,
      "rewards/rejected": -0.7100099325180054,
      "step": 5260
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.7920618178850269e-06,
      "logits/chosen": -2.0428383350372314,
      "logits/rejected": -1.7901527881622314,
      "logps/chosen": -307.78533935546875,
      "logps/rejected": -285.2147521972656,
      "loss": 0.1015,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5738735795021057,
      "rewards/margins": 0.16959087550640106,
      "rewards/rejected": -0.743464469909668,
      "step": 5270
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.7820260567943904e-06,
      "logits/chosen": -1.903365135192871,
      "logits/rejected": -1.7534162998199463,
      "logps/chosen": -185.64505004882812,
      "logps/rejected": -201.76344299316406,
      "loss": 0.1676,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.574264407157898,
      "rewards/margins": 0.13275280594825745,
      "rewards/rejected": -0.7070172429084778,
      "step": 5280
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.7720028898711852e-06,
      "logits/chosen": -1.8674421310424805,
      "logits/rejected": -1.413944959640503,
      "logps/chosen": -263.2582092285156,
      "logps/rejected": -235.0648956298828,
      "loss": 0.137,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.491985946893692,
      "rewards/margins": 0.17603492736816406,
      "rewards/rejected": -0.6680207848548889,
      "step": 5290
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7619924929343857e-06,
      "logits/chosen": -1.919923186302185,
      "logits/rejected": -1.7005188465118408,
      "logps/chosen": -273.3664855957031,
      "logps/rejected": -306.89056396484375,
      "loss": 0.1344,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4788680076599121,
      "rewards/margins": 0.18705452978610992,
      "rewards/rejected": -0.6659225225448608,
      "step": 5300
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7519950415789661e-06,
      "logits/chosen": -1.7585570812225342,
      "logits/rejected": -1.5700247287750244,
      "logps/chosen": -251.32113647460938,
      "logps/rejected": -301.25079345703125,
      "loss": 0.1677,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.4546022415161133,
      "rewards/margins": 0.16988904774188995,
      "rewards/rejected": -0.624491274356842,
      "step": 5310
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7420107111728167e-06,
      "logits/chosen": -1.8963468074798584,
      "logits/rejected": -1.7362186908721924,
      "logps/chosen": -206.5269317626953,
      "logps/rejected": -223.75009155273438,
      "loss": 0.08,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.39686816930770874,
      "rewards/margins": 0.15394839644432068,
      "rewards/rejected": -0.550816535949707,
      "step": 5320
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7320396768536695e-06,
      "logits/chosen": -1.9675251245498657,
      "logits/rejected": -1.5709375143051147,
      "logps/chosen": -268.6692199707031,
      "logps/rejected": -247.16464233398438,
      "loss": 0.0934,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.4870396554470062,
      "rewards/margins": 0.1673925369977951,
      "rewards/rejected": -0.6544321775436401,
      "step": 5330
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7220821135260301e-06,
      "logits/chosen": -1.836387038230896,
      "logits/rejected": -1.3881337642669678,
      "logps/chosen": -240.44357299804688,
      "logps/rejected": -221.38931274414062,
      "loss": 0.0791,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5035630464553833,
      "rewards/margins": 0.15110139548778534,
      "rewards/rejected": -0.6546644568443298,
      "step": 5340
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7121381958581018e-06,
      "logits/chosen": -2.0114264488220215,
      "logits/rejected": -1.6229709386825562,
      "logps/chosen": -298.6057434082031,
      "logps/rejected": -229.20565795898438,
      "loss": 0.1362,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5573118925094604,
      "rewards/margins": 0.11680523306131363,
      "rewards/rejected": -0.6741170883178711,
      "step": 5350
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7022080982787259e-06,
      "logits/chosen": -1.8884429931640625,
      "logits/rejected": -1.5066940784454346,
      "logps/chosen": -274.16937255859375,
      "logps/rejected": -251.51052856445312,
      "loss": 0.1055,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5535235404968262,
      "rewards/margins": 0.13713331520557404,
      "rewards/rejected": -0.6906567811965942,
      "step": 5360
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.692291994974326e-06,
      "logits/chosen": -1.9273033142089844,
      "logits/rejected": -1.4788744449615479,
      "logps/chosen": -301.24896240234375,
      "logps/rejected": -266.068115234375,
      "loss": 0.1223,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.4468786120414734,
      "rewards/margins": 0.1857236623764038,
      "rewards/rejected": -0.6326022148132324,
      "step": 5370
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.682390059885845e-06,
      "logits/chosen": -1.962938904762268,
      "logits/rejected": -1.4628360271453857,
      "logps/chosen": -287.84283447265625,
      "logps/rejected": -222.7774658203125,
      "loss": 0.0968,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5061804056167603,
      "rewards/margins": 0.22085240483283997,
      "rewards/rejected": -0.7270327806472778,
      "step": 5380
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6725024667056965e-06,
      "logits/chosen": -1.8040755987167358,
      "logits/rejected": -1.4079840183258057,
      "logps/chosen": -270.97686767578125,
      "logps/rejected": -205.47982788085938,
      "loss": 0.1878,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.4969252645969391,
      "rewards/margins": 0.0869857668876648,
      "rewards/rejected": -0.5839110612869263,
      "step": 5390
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6626293888747238e-06,
      "logits/chosen": -1.9853794574737549,
      "logits/rejected": -1.4653469324111938,
      "logps/chosen": -268.05926513671875,
      "logps/rejected": -262.09197998046875,
      "loss": 0.1083,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.47049784660339355,
      "rewards/margins": 0.1899462640285492,
      "rewards/rejected": -0.6604441404342651,
      "step": 5400
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.652770999579148e-06,
      "logits/chosen": -1.9712364673614502,
      "logits/rejected": -1.6718246936798096,
      "logps/chosen": -248.3401336669922,
      "logps/rejected": -259.27471923828125,
      "loss": 0.1179,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.4843429923057556,
      "rewards/margins": 0.15328797698020935,
      "rewards/rejected": -0.6376310586929321,
      "step": 5410
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6429274717475358e-06,
      "logits/chosen": -1.8927046060562134,
      "logits/rejected": -1.5739778280258179,
      "logps/chosen": -279.84503173828125,
      "logps/rejected": -235.696533203125,
      "loss": 0.0874,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.4386516213417053,
      "rewards/margins": 0.19288429617881775,
      "rewards/rejected": -0.6315358877182007,
      "step": 5420
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6330989780477673e-06,
      "logits/chosen": -1.8618462085723877,
      "logits/rejected": -1.5246363878250122,
      "logps/chosen": -253.5839385986328,
      "logps/rejected": -251.325927734375,
      "loss": 0.151,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5161840319633484,
      "rewards/margins": 0.14887337386608124,
      "rewards/rejected": -0.665057361125946,
      "step": 5430
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6232856908840033e-06,
      "logits/chosen": -2.285269260406494,
      "logits/rejected": -1.713772177696228,
      "logps/chosen": -263.0864562988281,
      "logps/rejected": -217.45498657226562,
      "loss": 0.0977,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.4418443739414215,
      "rewards/margins": 0.17126531898975372,
      "rewards/rejected": -0.6131097078323364,
      "step": 5440
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.613487782393661e-06,
      "logits/chosen": -1.9873231649398804,
      "logits/rejected": -1.5774824619293213,
      "logps/chosen": -259.41546630859375,
      "logps/rejected": -270.5675048828125,
      "loss": 0.122,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.4419049322605133,
      "rewards/margins": 0.1639256477355957,
      "rewards/rejected": -0.6058306097984314,
      "step": 5450
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.6037054244444007e-06,
      "logits/chosen": -1.9209073781967163,
      "logits/rejected": -1.675719976425171,
      "logps/chosen": -248.13119506835938,
      "logps/rejected": -253.267822265625,
      "loss": 0.1288,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5143054127693176,
      "rewards/margins": 0.1403733789920807,
      "rewards/rejected": -0.6546787023544312,
      "step": 5460
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.593938788631103e-06,
      "logits/chosen": -1.6965789794921875,
      "logits/rejected": -1.3882747888565063,
      "logps/chosen": -236.5738067626953,
      "logps/rejected": -283.84869384765625,
      "loss": 0.0971,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.4736576974391937,
      "rewards/margins": 0.199508398771286,
      "rewards/rejected": -0.6731661558151245,
      "step": 5470
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5841880462728626e-06,
      "logits/chosen": -1.8833509683609009,
      "logits/rejected": -1.6429067850112915,
      "logps/chosen": -280.9195861816406,
      "logps/rejected": -278.397705078125,
      "loss": 0.1557,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.46772581338882446,
      "rewards/margins": 0.1507682502269745,
      "rewards/rejected": -0.6184940934181213,
      "step": 5480
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5744533684099861e-06,
      "logits/chosen": -2.0979132652282715,
      "logits/rejected": -1.699033498764038,
      "logps/chosen": -264.59173583984375,
      "logps/rejected": -252.0912628173828,
      "loss": 0.164,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.4788171648979187,
      "rewards/margins": 0.12794797122478485,
      "rewards/rejected": -0.6067651510238647,
      "step": 5490
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5647349258009857e-06,
      "logits/chosen": -1.7671406269073486,
      "logits/rejected": -1.574204683303833,
      "logps/chosen": -282.95458984375,
      "logps/rejected": -298.6538391113281,
      "loss": 0.0751,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5590416193008423,
      "rewards/margins": 0.1640872210264206,
      "rewards/rejected": -0.7231289148330688,
      "step": 5500
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.555032888919586e-06,
      "logits/chosen": -1.686753511428833,
      "logits/rejected": -1.4295735359191895,
      "logps/chosen": -264.7411804199219,
      "logps/rejected": -253.6551971435547,
      "loss": 0.2432,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5408766269683838,
      "rewards/margins": 0.15903475880622864,
      "rewards/rejected": -0.69991135597229,
      "step": 5510
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5453474279517383e-06,
      "logits/chosen": -1.805437684059143,
      "logits/rejected": -1.6262556314468384,
      "logps/chosen": -240.7580108642578,
      "logps/rejected": -236.3534393310547,
      "loss": 0.1229,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.500167727470398,
      "rewards/margins": 0.12385289371013641,
      "rewards/rejected": -0.6240206360816956,
      "step": 5520
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5356787127926285e-06,
      "logits/chosen": -1.9110714197158813,
      "logits/rejected": -1.4245280027389526,
      "logps/chosen": -316.56072998046875,
      "logps/rejected": -266.979248046875,
      "loss": 0.0896,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4973506033420563,
      "rewards/margins": 0.21019454300403595,
      "rewards/rejected": -0.7075451612472534,
      "step": 5530
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.526026913043699e-06,
      "logits/chosen": -1.7721723318099976,
      "logits/rejected": -1.6717960834503174,
      "logps/chosen": -231.5484161376953,
      "logps/rejected": -236.1089324951172,
      "loss": 0.0822,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.5355015397071838,
      "rewards/margins": 0.13346286118030548,
      "rewards/rejected": -0.6689643859863281,
      "step": 5540
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.5163921980096791e-06,
      "logits/chosen": -1.8417619466781616,
      "logits/rejected": -1.7490192651748657,
      "logps/chosen": -259.3507080078125,
      "logps/rejected": -285.2629699707031,
      "loss": 0.1187,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5752219557762146,
      "rewards/margins": 0.11573759466409683,
      "rewards/rejected": -0.6909595727920532,
      "step": 5550
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.5067747366956065e-06,
      "logits/chosen": -2.117729663848877,
      "logits/rejected": -1.7773869037628174,
      "logps/chosen": -260.44342041015625,
      "logps/rejected": -221.59371948242188,
      "loss": 0.1647,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.49646344780921936,
      "rewards/margins": 0.16098496317863464,
      "rewards/rejected": -0.657448410987854,
      "step": 5560
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4971746978038671e-06,
      "logits/chosen": -1.8527145385742188,
      "logits/rejected": -1.7836803197860718,
      "logps/chosen": -257.3758850097656,
      "logps/rejected": -278.2964782714844,
      "loss": 0.1172,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.4473143517971039,
      "rewards/margins": 0.14970967173576355,
      "rewards/rejected": -0.5970240235328674,
      "step": 5570
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4875922497312384e-06,
      "logits/chosen": -1.802384376525879,
      "logits/rejected": -1.3964884281158447,
      "logps/chosen": -257.4482421875,
      "logps/rejected": -256.9828796386719,
      "loss": 0.0714,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.4673423767089844,
      "rewards/margins": 0.22057469189167023,
      "rewards/rejected": -0.6879170536994934,
      "step": 5580
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4780275605659308e-06,
      "logits/chosen": -1.9443477392196655,
      "logits/rejected": -1.470523476600647,
      "logps/chosen": -216.4662628173828,
      "logps/rejected": -213.83154296875,
      "loss": 0.1033,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.47214627265930176,
      "rewards/margins": 0.24475538730621338,
      "rewards/rejected": -0.7169016599655151,
      "step": 5590
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.46848079808464e-06,
      "logits/chosen": -1.811112642288208,
      "logits/rejected": -1.5633313655853271,
      "logps/chosen": -286.9377746582031,
      "logps/rejected": -256.46368408203125,
      "loss": 0.1492,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.4783618450164795,
      "rewards/margins": 0.16989843547344208,
      "rewards/rejected": -0.648260235786438,
      "step": 5600
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4589521297496085e-06,
      "logits/chosen": -1.9072492122650146,
      "logits/rejected": -1.6674668788909912,
      "logps/chosen": -287.529052734375,
      "logps/rejected": -309.4631042480469,
      "loss": 0.1368,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.46460795402526855,
      "rewards/margins": 0.12597152590751648,
      "rewards/rejected": -0.5905795097351074,
      "step": 5610
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4494417227056811e-06,
      "logits/chosen": -1.9489076137542725,
      "logits/rejected": -1.5660401582717896,
      "logps/chosen": -236.4983673095703,
      "logps/rejected": -253.90518188476562,
      "loss": 0.0872,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.41152167320251465,
      "rewards/margins": 0.21388018131256104,
      "rewards/rejected": -0.6254019737243652,
      "step": 5620
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4399497437773786e-06,
      "logits/chosen": -1.9147542715072632,
      "logits/rejected": -1.4943046569824219,
      "logps/chosen": -272.55987548828125,
      "logps/rejected": -264.3128967285156,
      "loss": 0.1259,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5410269498825073,
      "rewards/margins": 0.19129455089569092,
      "rewards/rejected": -0.732321560382843,
      "step": 5630
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4304763594659694e-06,
      "logits/chosen": -2.035388946533203,
      "logits/rejected": -1.5075829029083252,
      "logps/chosen": -302.56365966796875,
      "logps/rejected": -257.10601806640625,
      "loss": 0.1725,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.49816712737083435,
      "rewards/margins": 0.18971143662929535,
      "rewards/rejected": -0.6878786087036133,
      "step": 5640
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4210217359465483e-06,
      "logits/chosen": -2.001213312149048,
      "logits/rejected": -1.7480404376983643,
      "logps/chosen": -250.2290496826172,
      "logps/rejected": -263.33099365234375,
      "loss": 0.1514,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.43809619545936584,
      "rewards/margins": 0.09596999734640121,
      "rewards/rejected": -0.5340661406517029,
      "step": 5650
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4115860390651204e-06,
      "logits/chosen": -1.8610761165618896,
      "logits/rejected": -1.242117166519165,
      "logps/chosen": -272.94488525390625,
      "logps/rejected": -214.08584594726562,
      "loss": 0.1465,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.5517784953117371,
      "rewards/margins": 0.17574277520179749,
      "rewards/rejected": -0.7275213003158569,
      "step": 5660
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4021694343356992e-06,
      "logits/chosen": -2.0516114234924316,
      "logits/rejected": -1.6413084268569946,
      "logps/chosen": -243.1460723876953,
      "logps/rejected": -240.1009063720703,
      "loss": 0.1033,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.4688890874385834,
      "rewards/margins": 0.14948108792304993,
      "rewards/rejected": -0.6183701753616333,
      "step": 5670
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.3927720869373912e-06,
      "logits/chosen": -1.7400707006454468,
      "logits/rejected": -1.5164529085159302,
      "logps/chosen": -283.2667236328125,
      "logps/rejected": -289.87200927734375,
      "loss": 0.0951,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.46899938583374023,
      "rewards/margins": 0.15114405751228333,
      "rewards/rejected": -0.6201435327529907,
      "step": 5680
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.383394161711509e-06,
      "logits/chosen": -1.7821184396743774,
      "logits/rejected": -1.5143952369689941,
      "logps/chosen": -243.70639038085938,
      "logps/rejected": -248.849365234375,
      "loss": 0.0863,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.5013656616210938,
      "rewards/margins": 0.18036916851997375,
      "rewards/rejected": -0.6817347407341003,
      "step": 5690
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.3740358231586752e-06,
      "logits/chosen": -1.9248117208480835,
      "logits/rejected": -1.6384576559066772,
      "logps/chosen": -238.166015625,
      "logps/rejected": -213.8789825439453,
      "loss": 0.1689,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.5257433652877808,
      "rewards/margins": 0.1685757339000702,
      "rewards/rejected": -0.6943190693855286,
      "step": 5700
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3646972354359379e-06,
      "logits/chosen": -2.0671422481536865,
      "logits/rejected": -1.6382163763046265,
      "logps/chosen": -248.79116821289062,
      "logps/rejected": -239.56393432617188,
      "loss": 0.1352,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5227428674697876,
      "rewards/margins": 0.1494591385126114,
      "rewards/rejected": -0.672201931476593,
      "step": 5710
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3553785623538873e-06,
      "logits/chosen": -1.8637508153915405,
      "logits/rejected": -1.6675211191177368,
      "logps/chosen": -217.97286987304688,
      "logps/rejected": -246.8916015625,
      "loss": 0.1261,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5041235685348511,
      "rewards/margins": 0.16983681917190552,
      "rewards/rejected": -0.6739604473114014,
      "step": 5720
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.346079967373792e-06,
      "logits/chosen": -1.684739351272583,
      "logits/rejected": -1.6077110767364502,
      "logps/chosen": -205.7179718017578,
      "logps/rejected": -213.598388671875,
      "loss": 0.0877,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.4919508099555969,
      "rewards/margins": 0.15389500558376312,
      "rewards/rejected": -0.6458457708358765,
      "step": 5730
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3368016136047194e-06,
      "logits/chosen": -1.7886161804199219,
      "logits/rejected": -1.5077216625213623,
      "logps/chosen": -311.54486083984375,
      "logps/rejected": -265.2232971191406,
      "loss": 0.1852,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5703347325325012,
      "rewards/margins": 0.1437651813030243,
      "rewards/rejected": -0.7140999436378479,
      "step": 5740
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3275436638006838e-06,
      "logits/chosen": -1.9010818004608154,
      "logits/rejected": -1.6132938861846924,
      "logps/chosen": -279.0166320800781,
      "logps/rejected": -293.9725646972656,
      "loss": 0.1197,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5455999374389648,
      "rewards/margins": 0.1750425398349762,
      "rewards/rejected": -0.7206425070762634,
      "step": 5750
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3183062803577872e-06,
      "logits/chosen": -1.9034898281097412,
      "logits/rejected": -1.5641247034072876,
      "logps/chosen": -227.96151733398438,
      "logps/rejected": -215.01339721679688,
      "loss": 0.1401,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.5809696316719055,
      "rewards/margins": 0.2188103199005127,
      "rewards/rejected": -0.7997799515724182,
      "step": 5760
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3090896253113736e-06,
      "logits/chosen": -1.8766626119613647,
      "logits/rejected": -1.700510025024414,
      "logps/chosen": -258.7127380371094,
      "logps/rejected": -250.1974334716797,
      "loss": 0.1517,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.5055121183395386,
      "rewards/margins": 0.1273249089717865,
      "rewards/rejected": -0.6328369379043579,
      "step": 5770
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.2998938603331796e-06,
      "logits/chosen": -1.8572345972061157,
      "logits/rejected": -1.651341199874878,
      "logps/chosen": -255.7646026611328,
      "logps/rejected": -279.30694580078125,
      "loss": 0.0922,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6293990612030029,
      "rewards/margins": 0.12281368672847748,
      "rewards/rejected": -0.7522127032279968,
      "step": 5780
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.2907191467285118e-06,
      "logits/chosen": -1.9182488918304443,
      "logits/rejected": -1.6267467737197876,
      "logps/chosen": -266.6481018066406,
      "logps/rejected": -413.2779235839844,
      "loss": 5.1685,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.47287482023239136,
      "rewards/margins": 0.5273378491401672,
      "rewards/rejected": -1.0002126693725586,
      "step": 5790
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2815656454334013e-06,
      "logits/chosen": -1.9395920038223267,
      "logits/rejected": -1.8263431787490845,
      "logps/chosen": -256.23663330078125,
      "logps/rejected": -270.293701171875,
      "loss": 0.1378,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.4917454719543457,
      "rewards/margins": 0.15827788412570953,
      "rewards/rejected": -0.650023341178894,
      "step": 5800
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.272433517011793e-06,
      "logits/chosen": -1.9891746044158936,
      "logits/rejected": -1.7108840942382812,
      "logps/chosen": -311.8459167480469,
      "logps/rejected": -299.76654052734375,
      "loss": 0.0751,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.48035064339637756,
      "rewards/margins": 0.1297578364610672,
      "rewards/rejected": -0.6101084351539612,
      "step": 5810
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2633229216527235e-06,
      "logits/chosen": -1.9504735469818115,
      "logits/rejected": -1.4860206842422485,
      "logps/chosen": -230.38272094726562,
      "logps/rejected": -224.21981811523438,
      "loss": 0.1323,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.4245625138282776,
      "rewards/margins": 0.2091328203678131,
      "rewards/rejected": -0.6336953043937683,
      "step": 5820
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.254234019167514e-06,
      "logits/chosen": -1.9143810272216797,
      "logits/rejected": -1.5591933727264404,
      "logps/chosen": -283.6191711425781,
      "logps/rejected": -264.5260009765625,
      "loss": 0.0793,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.4440692067146301,
      "rewards/margins": 0.2215677946805954,
      "rewards/rejected": -0.6656370759010315,
      "step": 5830
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.24516696898696e-06,
      "logits/chosen": -1.9349048137664795,
      "logits/rejected": -1.555820345878601,
      "logps/chosen": -267.323486328125,
      "logps/rejected": -289.51947021484375,
      "loss": 0.1085,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5123074054718018,
      "rewards/margins": 0.20973484218120575,
      "rewards/rejected": -0.7220422029495239,
      "step": 5840
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2361219301585487e-06,
      "logits/chosen": -2.0896143913269043,
      "logits/rejected": -1.63728928565979,
      "logps/chosen": -277.4181213378906,
      "logps/rejected": -244.1571044921875,
      "loss": 0.111,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.4161832332611084,
      "rewards/margins": 0.1889437586069107,
      "rewards/rejected": -0.6051269769668579,
      "step": 5850
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2270990613436522e-06,
      "logits/chosen": -1.9406229257583618,
      "logits/rejected": -1.6228440999984741,
      "logps/chosen": -229.52969360351562,
      "logps/rejected": -256.2364196777344,
      "loss": 0.1576,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.4235209822654724,
      "rewards/margins": 0.12438831478357315,
      "rewards/rejected": -0.5479093194007874,
      "step": 5860
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2180985208147571e-06,
      "logits/chosen": -1.9510200023651123,
      "logits/rejected": -1.7511215209960938,
      "logps/chosen": -213.273681640625,
      "logps/rejected": -237.1942596435547,
      "loss": 0.0932,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.39561378955841064,
      "rewards/margins": 0.13690926134586334,
      "rewards/rejected": -0.5325230360031128,
      "step": 5870
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.2091204664526831e-06,
      "logits/chosen": -2.0162253379821777,
      "logits/rejected": -1.6138890981674194,
      "logps/chosen": -287.21746826171875,
      "logps/rejected": -234.35693359375,
      "loss": 0.184,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.4287000596523285,
      "rewards/margins": 0.1497940570116043,
      "rewards/rejected": -0.578494131565094,
      "step": 5880
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.2001650557438143e-06,
      "logits/chosen": -2.028672695159912,
      "logits/rejected": -1.7924978733062744,
      "logps/chosen": -294.2822265625,
      "logps/rejected": -278.37652587890625,
      "loss": 0.1596,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.44275110960006714,
      "rewards/margins": 0.11968086659908295,
      "rewards/rejected": -0.5624319911003113,
      "step": 5890
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1912324457773336e-06,
      "logits/chosen": -2.0378835201263428,
      "logits/rejected": -1.7147912979125977,
      "logps/chosen": -240.72216796875,
      "logps/rejected": -268.4493713378906,
      "loss": 0.0914,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.39333948493003845,
      "rewards/margins": 0.19647042453289032,
      "rewards/rejected": -0.5898098945617676,
      "step": 5900
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.182322793242476e-06,
      "logits/chosen": -2.0833020210266113,
      "logits/rejected": -1.8951669931411743,
      "logps/chosen": -225.5447540283203,
      "logps/rejected": -256.537109375,
      "loss": 0.1383,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.431450754404068,
      "rewards/margins": 0.16372133791446686,
      "rewards/rejected": -0.5951720476150513,
      "step": 5910
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1734362544257686e-06,
      "logits/chosen": -1.8667224645614624,
      "logits/rejected": -1.4377329349517822,
      "logps/chosen": -305.6443786621094,
      "logps/rejected": -238.24801635742188,
      "loss": 0.1093,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3690487742424011,
      "rewards/margins": 0.14427319169044495,
      "rewards/rejected": -0.5133219957351685,
      "step": 5920
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1645729852082977e-06,
      "logits/chosen": -2.2229387760162354,
      "logits/rejected": -1.7280595302581787,
      "logps/chosen": -246.1704559326172,
      "logps/rejected": -238.6482391357422,
      "loss": 0.1897,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.4612872004508972,
      "rewards/margins": 0.18152639269828796,
      "rewards/rejected": -0.6428135633468628,
      "step": 5930
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1557331410629708e-06,
      "logits/chosen": -2.273224115371704,
      "logits/rejected": -1.5956547260284424,
      "logps/chosen": -271.32366943359375,
      "logps/rejected": -216.1154327392578,
      "loss": 0.1183,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.39185529947280884,
      "rewards/margins": 0.1586187481880188,
      "rewards/rejected": -0.5504740476608276,
      "step": 5940
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1469168770517913e-06,
      "logits/chosen": -2.2409141063690186,
      "logits/rejected": -1.7641382217407227,
      "logps/chosen": -249.9724578857422,
      "logps/rejected": -213.88729858398438,
      "loss": 0.123,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4436149001121521,
      "rewards/margins": 0.15342697501182556,
      "rewards/rejected": -0.5970418453216553,
      "step": 5950
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.1381243478231336e-06,
      "logits/chosen": -2.1302855014801025,
      "logits/rejected": -1.7275043725967407,
      "logps/chosen": -295.96807861328125,
      "logps/rejected": -232.9789276123047,
      "loss": 0.1576,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3948201537132263,
      "rewards/margins": 0.1410309374332428,
      "rewards/rejected": -0.5358511209487915,
      "step": 5960
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.1293557076090403e-06,
      "logits/chosen": -2.055603504180908,
      "logits/rejected": -1.6448357105255127,
      "logps/chosen": -270.03485107421875,
      "logps/rejected": -257.23333740234375,
      "loss": 0.1255,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.3696318566799164,
      "rewards/margins": 0.16263523697853088,
      "rewards/rejected": -0.5322670936584473,
      "step": 5970
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.1206111102225043e-06,
      "logits/chosen": -2.016026020050049,
      "logits/rejected": -1.7732445001602173,
      "logps/chosen": -322.04571533203125,
      "logps/rejected": -314.3202819824219,
      "loss": 0.1256,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.43497371673583984,
      "rewards/margins": 0.15854512155056,
      "rewards/rejected": -0.593518853187561,
      "step": 5980
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.1118907090547805e-06,
      "logits/chosen": -2.1408379077911377,
      "logits/rejected": -1.6741759777069092,
      "logps/chosen": -290.5514221191406,
      "logps/rejected": -262.75262451171875,
      "loss": 0.099,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.45133018493652344,
      "rewards/margins": 0.22001805901527405,
      "rewards/rejected": -0.6713482737541199,
      "step": 5990
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.1031946570726912e-06,
      "logits/chosen": -1.9711778163909912,
      "logits/rejected": -1.855182409286499,
      "logps/chosen": -278.8828430175781,
      "logps/rejected": -290.8421325683594,
      "loss": 0.1164,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5591145157814026,
      "rewards/margins": 0.11836342513561249,
      "rewards/rejected": -0.6774778962135315,
      "step": 6000
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.094523106815944e-06,
      "logits/chosen": -1.855329155921936,
      "logits/rejected": -1.5584124326705933,
      "logps/chosen": -267.0565185546875,
      "logps/rejected": -273.24310302734375,
      "loss": 0.1452,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.44338005781173706,
      "rewards/margins": 0.17573294043540955,
      "rewards/rejected": -0.6191130876541138,
      "step": 6010
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.0858762103944511e-06,
      "logits/chosen": -1.87862229347229,
      "logits/rejected": -1.6584688425064087,
      "logps/chosen": -280.62738037109375,
      "logps/rejected": -262.09942626953125,
      "loss": 0.125,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5238662362098694,
      "rewards/margins": 0.09647830575704575,
      "rewards/rejected": -0.6203445196151733,
      "step": 6020
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.0772541194856732e-06,
      "logits/chosen": -2.123035430908203,
      "logits/rejected": -1.6155083179473877,
      "logps/chosen": -321.01776123046875,
      "logps/rejected": -279.3617858886719,
      "loss": 0.0745,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.42041999101638794,
      "rewards/margins": 0.16421575844287872,
      "rewards/rejected": -0.5846357941627502,
      "step": 6030
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.068656985331943e-06,
      "logits/chosen": -1.9696871042251587,
      "logits/rejected": -1.6894041299819946,
      "logps/chosen": -253.11233520507812,
      "logps/rejected": -267.6400146484375,
      "loss": 0.1156,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.48150572180747986,
      "rewards/margins": 0.17806780338287354,
      "rewards/rejected": -0.6595736145973206,
      "step": 6040
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.060084958737825e-06,
      "logits/chosen": -1.990724802017212,
      "logits/rejected": -1.446575403213501,
      "logps/chosen": -237.4898681640625,
      "logps/rejected": -233.758056640625,
      "loss": 0.1,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.3963894546031952,
      "rewards/margins": 0.20723167061805725,
      "rewards/rejected": -0.6036210656166077,
      "step": 6050
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0515381900674643e-06,
      "logits/chosen": -2.1221401691436768,
      "logits/rejected": -1.7900186777114868,
      "logps/chosen": -257.96722412109375,
      "logps/rejected": -280.8104553222656,
      "loss": 0.1199,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.4584527611732483,
      "rewards/margins": 0.16997918486595154,
      "rewards/rejected": -0.6284319162368774,
      "step": 6060
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.04301682924195e-06,
      "logits/chosen": -1.9626433849334717,
      "logits/rejected": -1.4727249145507812,
      "logps/chosen": -207.9431915283203,
      "logps/rejected": -195.52149963378906,
      "loss": 0.1507,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.4559662938117981,
      "rewards/margins": 0.22094134986400604,
      "rewards/rejected": -0.6769076585769653,
      "step": 6070
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.034521025736686e-06,
      "logits/chosen": -1.9479316473007202,
      "logits/rejected": -1.5621986389160156,
      "logps/chosen": -225.77102661132812,
      "logps/rejected": -238.942138671875,
      "loss": 0.1602,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.4776443839073181,
      "rewards/margins": 0.16333012282848358,
      "rewards/rejected": -0.6409745216369629,
      "step": 6080
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0260509285787694e-06,
      "logits/chosen": -2.129117250442505,
      "logits/rejected": -1.501849889755249,
      "logps/chosen": -253.2616729736328,
      "logps/rejected": -219.7965545654297,
      "loss": 0.1035,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.4091704785823822,
      "rewards/margins": 0.18820294737815857,
      "rewards/rejected": -0.597373366355896,
      "step": 6090
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0176066863443726e-06,
      "logits/chosen": -1.8864481449127197,
      "logits/rejected": -1.5883742570877075,
      "logps/chosen": -257.90185546875,
      "logps/rejected": -220.6878662109375,
      "loss": 0.1418,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.45451027154922485,
      "rewards/margins": 0.12341739982366562,
      "rewards/rejected": -0.5779277086257935,
      "step": 6100
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0091884471561424e-06,
      "logits/chosen": -1.8764444589614868,
      "logits/rejected": -1.63616943359375,
      "logps/chosen": -261.2283020019531,
      "logps/rejected": -244.85128784179688,
      "loss": 0.1435,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.4946800172328949,
      "rewards/margins": 0.1375671923160553,
      "rewards/rejected": -0.632247269153595,
      "step": 6110
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0007963586806e-06,
      "logits/chosen": -1.905747652053833,
      "logits/rejected": -1.4888708591461182,
      "logps/chosen": -284.759033203125,
      "logps/rejected": -269.97320556640625,
      "loss": 0.1243,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5422388315200806,
      "rewards/margins": 0.15825456380844116,
      "rewards/rejected": -0.700493335723877,
      "step": 6120
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.924305681255484e-07,
      "logits/chosen": -1.8924305438995361,
      "logits/rejected": -1.4120725393295288,
      "logps/chosen": -261.49359130859375,
      "logps/rejected": -246.7588653564453,
      "loss": 0.1309,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.4799385964870453,
      "rewards/margins": 0.22691066563129425,
      "rewards/rejected": -0.7068492770195007,
      "step": 6130
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.840912222374932e-07,
      "logits/chosen": -2.065091609954834,
      "logits/rejected": -1.551511287689209,
      "logps/chosen": -271.27764892578125,
      "logps/rejected": -250.86550903320312,
      "loss": 0.1203,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.465266615152359,
      "rewards/margins": 0.19768501818180084,
      "rewards/rejected": -0.6629515886306763,
      "step": 6140
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.757784672990668e-07,
      "logits/chosen": -1.8214833736419678,
      "logits/rejected": -1.3881083726882935,
      "logps/chosen": -256.01123046875,
      "logps/rejected": -230.39895629882812,
      "loss": 0.1763,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5507286787033081,
      "rewards/margins": 0.19404050707817078,
      "rewards/rejected": -0.7447691559791565,
      "step": 6150
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.674924491264632e-07,
      "logits/chosen": -1.8416248559951782,
      "logits/rejected": -1.6367276906967163,
      "logps/chosen": -219.26351928710938,
      "logps/rejected": -215.86849975585938,
      "loss": 0.1109,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.4397053122520447,
      "rewards/margins": 0.14263916015625,
      "rewards/rejected": -0.5823444724082947,
      "step": 6160
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.59233313066878e-07,
      "logits/chosen": -2.089197874069214,
      "logits/rejected": -1.6182596683502197,
      "logps/chosen": -260.54376220703125,
      "logps/rejected": -253.81912231445312,
      "loss": 0.1007,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.44585880637168884,
      "rewards/margins": 0.17450013756752014,
      "rewards/rejected": -0.620358943939209,
      "step": 6170
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.510012039959632e-07,
      "logits/chosen": -1.9944250583648682,
      "logits/rejected": -1.6469684839248657,
      "logps/chosen": -285.20440673828125,
      "logps/rejected": -260.31121826171875,
      "loss": 0.1213,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.4487648904323578,
      "rewards/margins": 0.14048054814338684,
      "rewards/rejected": -0.5892454981803894,
      "step": 6180
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.427962663152821e-07,
      "logits/chosen": -1.9396718740463257,
      "logits/rejected": -1.632127046585083,
      "logps/chosen": -305.9094543457031,
      "logps/rejected": -257.2548828125,
      "loss": 0.1065,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.46664246916770935,
      "rewards/margins": 0.15920694172382355,
      "rewards/rejected": -0.6258494257926941,
      "step": 6190
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.346186439497778e-07,
      "logits/chosen": -1.9716598987579346,
      "logits/rejected": -1.637711524963379,
      "logps/chosen": -238.27755737304688,
      "logps/rejected": -227.07540893554688,
      "loss": 0.1825,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.43801671266555786,
      "rewards/margins": 0.13967742025852203,
      "rewards/rejected": -0.5776941180229187,
      "step": 6200
    },
    {
      "epoch": 0.75,
      "learning_rate": 9.264684803452484e-07,
      "logits/chosen": -1.9573974609375,
      "logits/rejected": -1.6610110998153687,
      "logps/chosen": -292.9803771972656,
      "logps/rejected": -284.8642578125,
      "loss": 0.1376,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.4125700891017914,
      "rewards/margins": 0.1055794209241867,
      "rewards/rejected": -0.5181494951248169,
      "step": 6210
    },
    {
      "epoch": 0.75,
      "learning_rate": 9.183459184658317e-07,
      "logits/chosen": -1.8743131160736084,
      "logits/rejected": -1.5913054943084717,
      "logps/chosen": -259.7023010253906,
      "logps/rejected": -258.9094543457031,
      "loss": 0.105,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.45578956604003906,
      "rewards/margins": 0.11757218837738037,
      "rewards/rejected": -0.5733617544174194,
      "step": 6220
    },
    {
      "epoch": 0.75,
      "learning_rate": 9.102511007914924e-07,
      "logits/chosen": -1.9219213724136353,
      "logits/rejected": -1.4633852243423462,
      "logps/chosen": -210.48403930664062,
      "logps/rejected": -200.42410278320312,
      "loss": 0.126,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4389539659023285,
      "rewards/margins": 0.202097088098526,
      "rewards/rejected": -0.6410510540008545,
      "step": 6230
    },
    {
      "epoch": 0.75,
      "learning_rate": 9.021841693155343e-07,
      "logits/chosen": -2.061584234237671,
      "logits/rejected": -1.6734384298324585,
      "logps/chosen": -264.5237121582031,
      "logps/rejected": -252.5099334716797,
      "loss": 0.1233,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.39105328917503357,
      "rewards/margins": 0.18605628609657288,
      "rewards/rejected": -0.5771095752716064,
      "step": 6240
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.94145265542094e-07,
      "logits/chosen": -2.173652172088623,
      "logits/rejected": -1.8842623233795166,
      "logps/chosen": -311.49774169921875,
      "logps/rejected": -285.50506591796875,
      "loss": 0.0915,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.3695235252380371,
      "rewards/margins": 0.18801763653755188,
      "rewards/rejected": -0.5575411915779114,
      "step": 6250
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.861345304836727e-07,
      "logits/chosen": -1.937359094619751,
      "logits/rejected": -1.8544925451278687,
      "logps/chosen": -278.1236877441406,
      "logps/rejected": -304.86962890625,
      "loss": 0.1114,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5378960371017456,
      "rewards/margins": 0.09365091472864151,
      "rewards/rejected": -0.6315470933914185,
      "step": 6260
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.781521046586541e-07,
      "logits/chosen": -1.9989734888076782,
      "logits/rejected": -1.5364919900894165,
      "logps/chosen": -244.04989624023438,
      "logps/rejected": -233.1022491455078,
      "loss": 0.136,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.4168265461921692,
      "rewards/margins": 0.18471379578113556,
      "rewards/rejected": -0.6015402674674988,
      "step": 6270
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.701981280888444e-07,
      "logits/chosen": -1.8424322605133057,
      "logits/rejected": -1.6274335384368896,
      "logps/chosen": -247.665771484375,
      "logps/rejected": -262.58160400390625,
      "loss": 0.1512,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.3954276442527771,
      "rewards/margins": 0.15929386019706726,
      "rewards/rejected": -0.554721474647522,
      "step": 6280
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.622727402970097e-07,
      "logits/chosen": -1.7672450542449951,
      "logits/rejected": -1.7314002513885498,
      "logps/chosen": -255.74935913085938,
      "logps/rejected": -306.3473205566406,
      "loss": 0.0826,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.47890645265579224,
      "rewards/margins": 0.13192898035049438,
      "rewards/rejected": -0.6108353734016418,
      "step": 6290
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.543760803044393e-07,
      "logits/chosen": -1.9199352264404297,
      "logits/rejected": -1.465319275856018,
      "logps/chosen": -249.533447265625,
      "logps/rejected": -250.07919311523438,
      "loss": 0.1434,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.4657590389251709,
      "rewards/margins": 0.20879296958446503,
      "rewards/rejected": -0.6745520830154419,
      "step": 6300
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.465082866284951e-07,
      "logits/chosen": -2.023472547531128,
      "logits/rejected": -1.5475150346755981,
      "logps/chosen": -259.3255310058594,
      "logps/rejected": -234.91415405273438,
      "loss": 0.1315,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4470517635345459,
      "rewards/margins": 0.19469048082828522,
      "rewards/rejected": -0.6417423486709595,
      "step": 6310
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.386694972801904e-07,
      "logits/chosen": -1.8993467092514038,
      "logits/rejected": -1.540050745010376,
      "logps/chosen": -270.31182861328125,
      "logps/rejected": -249.36618041992188,
      "loss": 0.1409,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.4989251494407654,
      "rewards/margins": 0.16831564903259277,
      "rewards/rejected": -0.6672407984733582,
      "step": 6320
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.308598497617648e-07,
      "logits/chosen": -1.906795859336853,
      "logits/rejected": -1.6299690008163452,
      "logps/chosen": -175.07479858398438,
      "logps/rejected": -191.7357635498047,
      "loss": 0.0714,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.4141884446144104,
      "rewards/margins": 0.1404605209827423,
      "rewards/rejected": -0.5546489953994751,
      "step": 6330
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.230794810642753e-07,
      "logits/chosen": -1.9722293615341187,
      "logits/rejected": -1.5651670694351196,
      "logps/chosen": -290.1768798828125,
      "logps/rejected": -253.2939910888672,
      "loss": 0.1167,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.4297245442867279,
      "rewards/margins": 0.1225515827536583,
      "rewards/rejected": -0.5522761344909668,
      "step": 6340
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.153285276651876e-07,
      "logits/chosen": -2.1099610328674316,
      "logits/rejected": -1.8138777017593384,
      "logps/chosen": -228.72891235351562,
      "logps/rejected": -261.8756103515625,
      "loss": 0.0897,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.36999136209487915,
      "rewards/margins": 0.14173254370689392,
      "rewards/rejected": -0.5117239356040955,
      "step": 6350
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.076071255259918e-07,
      "logits/chosen": -1.9893842935562134,
      "logits/rejected": -1.562839150428772,
      "logps/chosen": -248.32656860351562,
      "logps/rejected": -230.19082641601562,
      "loss": 0.1012,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.4931401312351227,
      "rewards/margins": 0.16243189573287964,
      "rewards/rejected": -0.6555719971656799,
      "step": 6360
    },
    {
      "epoch": 0.76,
      "learning_rate": 7.999154100898063e-07,
      "logits/chosen": -1.8441355228424072,
      "logits/rejected": -1.6876119375228882,
      "logps/chosen": -210.8905792236328,
      "logps/rejected": -267.10882568359375,
      "loss": 0.0825,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.513385534286499,
      "rewards/margins": 0.17421071231365204,
      "rewards/rejected": -0.6875962018966675,
      "step": 6370
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.922535162790095e-07,
      "logits/chosen": -2.086892604827881,
      "logits/rejected": -1.8276185989379883,
      "logps/chosen": -234.5299072265625,
      "logps/rejected": -250.52059936523438,
      "loss": 0.1293,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.4303308129310608,
      "rewards/margins": 0.16687455773353577,
      "rewards/rejected": -0.5972053408622742,
      "step": 6380
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.846215784928721e-07,
      "logits/chosen": -2.0581459999084473,
      "logits/rejected": -1.7542043924331665,
      "logps/chosen": -229.604736328125,
      "logps/rejected": -245.43185424804688,
      "loss": 0.1118,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.43111807107925415,
      "rewards/margins": 0.1881767064332962,
      "rewards/rejected": -0.6192947626113892,
      "step": 6390
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.770197306051968e-07,
      "logits/chosen": -2.1675782203674316,
      "logits/rejected": -1.4943665266036987,
      "logps/chosen": -267.9649963378906,
      "logps/rejected": -236.1041259765625,
      "loss": 0.1182,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.42259639501571655,
      "rewards/margins": 0.24217908084392548,
      "rewards/rejected": -0.664775550365448,
      "step": 6400
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.694481059619705e-07,
      "logits/chosen": -1.932381272315979,
      "logits/rejected": -1.6388800144195557,
      "logps/chosen": -246.7991943359375,
      "logps/rejected": -249.91943359375,
      "loss": 0.1273,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.4872209429740906,
      "rewards/margins": 0.18630096316337585,
      "rewards/rejected": -0.6735219955444336,
      "step": 6410
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.619068373790306e-07,
      "logits/chosen": -2.089247226715088,
      "logits/rejected": -1.614682912826538,
      "logps/chosen": -282.35125732421875,
      "logps/rejected": -271.75067138671875,
      "loss": 0.129,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.44997042417526245,
      "rewards/margins": 0.1696648895740509,
      "rewards/rejected": -0.6196353435516357,
      "step": 6420
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.543960571397257e-07,
      "logits/chosen": -2.0346500873565674,
      "logits/rejected": -1.7321970462799072,
      "logps/chosen": -237.32608032226562,
      "logps/rejected": -245.86557006835938,
      "loss": 0.0754,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.46555739641189575,
      "rewards/margins": 0.16662751138210297,
      "rewards/rejected": -0.6321848630905151,
      "step": 6430
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.469158969926038e-07,
      "logits/chosen": -2.0735549926757812,
      "logits/rejected": -1.6884253025054932,
      "logps/chosen": -263.26898193359375,
      "logps/rejected": -264.8492126464844,
      "loss": 0.074,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.4788509011268616,
      "rewards/margins": 0.14806295931339264,
      "rewards/rejected": -0.6269139051437378,
      "step": 6440
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.39466488149097e-07,
      "logits/chosen": -2.0731894969940186,
      "logits/rejected": -1.5555330514907837,
      "logps/chosen": -243.86892700195312,
      "logps/rejected": -223.0145721435547,
      "loss": 0.1347,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.41140609979629517,
      "rewards/margins": 0.2104901522397995,
      "rewards/rejected": -0.6218962073326111,
      "step": 6450
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.320479612812218e-07,
      "logits/chosen": -2.017112970352173,
      "logits/rejected": -1.5632555484771729,
      "logps/chosen": -207.18191528320312,
      "logps/rejected": -199.11024475097656,
      "loss": 0.0694,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.3916897475719452,
      "rewards/margins": 0.14873406291007996,
      "rewards/rejected": -0.5404238700866699,
      "step": 6460
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.246604465192825e-07,
      "logits/chosen": -1.9492496252059937,
      "logits/rejected": -1.3995827436447144,
      "logps/chosen": -259.1231994628906,
      "logps/rejected": -208.67135620117188,
      "loss": 0.124,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.49213799834251404,
      "rewards/margins": 0.2363675832748413,
      "rewards/rejected": -0.7285054922103882,
      "step": 6470
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.173040734495973e-07,
      "logits/chosen": -1.8647918701171875,
      "logits/rejected": -1.5196516513824463,
      "logps/chosen": -291.6728210449219,
      "logps/rejected": -321.27923583984375,
      "loss": 0.134,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.504711925983429,
      "rewards/margins": 0.14177486300468445,
      "rewards/rejected": -0.646486759185791,
      "step": 6480
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.099789711122149e-07,
      "logits/chosen": -2.0234179496765137,
      "logits/rejected": -1.6390674114227295,
      "logps/chosen": -296.9944763183594,
      "logps/rejected": -274.94488525390625,
      "loss": 0.1205,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.4782020151615143,
      "rewards/margins": 0.15751081705093384,
      "rewards/rejected": -0.6357128024101257,
      "step": 6490
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.02685267998659e-07,
      "logits/chosen": -1.8039512634277344,
      "logits/rejected": -1.610396146774292,
      "logps/chosen": -217.3019256591797,
      "logps/rejected": -230.8792724609375,
      "loss": 0.1698,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.42607298493385315,
      "rewards/margins": 0.10820464789867401,
      "rewards/rejected": -0.534277617931366,
      "step": 6500
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.954230920496702e-07,
      "logits/chosen": -1.9449794292449951,
      "logits/rejected": -1.672254204750061,
      "logps/chosen": -209.4949188232422,
      "logps/rejected": -234.17465209960938,
      "loss": 0.0954,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5420662760734558,
      "rewards/margins": 0.2034389078617096,
      "rewards/rejected": -0.7455052733421326,
      "step": 6510
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.881925706529641e-07,
      "logits/chosen": -2.1921558380126953,
      "logits/rejected": -1.6133739948272705,
      "logps/chosen": -253.46450805664062,
      "logps/rejected": -225.8391571044922,
      "loss": 0.0943,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.47693949937820435,
      "rewards/margins": 0.17005819082260132,
      "rewards/rejected": -0.6469976305961609,
      "step": 6520
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.809938306409925e-07,
      "logits/chosen": -1.8478351831436157,
      "logits/rejected": -1.6184707880020142,
      "logps/chosen": -254.103515625,
      "logps/rejected": -244.87991333007812,
      "loss": 0.0911,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.474212646484375,
      "rewards/margins": 0.16678480803966522,
      "rewards/rejected": -0.6409973502159119,
      "step": 6530
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.738269982887266e-07,
      "logits/chosen": -2.0551493167877197,
      "logits/rejected": -1.6460390090942383,
      "logps/chosen": -320.0547790527344,
      "logps/rejected": -268.46875,
      "loss": 0.1129,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.4602360725402832,
      "rewards/margins": 0.18921080231666565,
      "rewards/rejected": -0.6494468450546265,
      "step": 6540
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.66692199311432e-07,
      "logits/chosen": -2.0328869819641113,
      "logits/rejected": -1.675079584121704,
      "logps/chosen": -289.8661193847656,
      "logps/rejected": -240.75991821289062,
      "loss": 0.1209,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.42221537232398987,
      "rewards/margins": 0.1419004648923874,
      "rewards/rejected": -0.5641158819198608,
      "step": 6550
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.595895588624717e-07,
      "logits/chosen": -2.185662269592285,
      "logits/rejected": -1.7335224151611328,
      "logps/chosen": -257.2197570800781,
      "logps/rejected": -243.74807739257812,
      "loss": 0.1239,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.36936455965042114,
      "rewards/margins": 0.1940496563911438,
      "rewards/rejected": -0.5634142756462097,
      "step": 6560
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.525192015311069e-07,
      "logits/chosen": -2.069929599761963,
      "logits/rejected": -1.7172218561172485,
      "logps/chosen": -269.12603759765625,
      "logps/rejected": -264.0863952636719,
      "loss": 0.085,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.456916481256485,
      "rewards/margins": 0.16084381937980652,
      "rewards/rejected": -0.6177603006362915,
      "step": 6570
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.454812513403127e-07,
      "logits/chosen": -2.1930034160614014,
      "logits/rejected": -1.7271572351455688,
      "logps/chosen": -229.4690399169922,
      "logps/rejected": -206.50888061523438,
      "loss": 0.105,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.4114112854003906,
      "rewards/margins": 0.1248263344168663,
      "rewards/rejected": -0.5362376570701599,
      "step": 6580
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.384758317445991e-07,
      "logits/chosen": -1.9955850839614868,
      "logits/rejected": -1.3841392993927002,
      "logps/chosen": -287.55853271484375,
      "logps/rejected": -227.1190948486328,
      "loss": 0.1237,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.39280954003334045,
      "rewards/margins": 0.25686582922935486,
      "rewards/rejected": -0.6496754288673401,
      "step": 6590
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.31503065627854e-07,
      "logits/chosen": -1.863050103187561,
      "logits/rejected": -1.605564832687378,
      "logps/chosen": -265.7858581542969,
      "logps/rejected": -288.51629638671875,
      "loss": 0.1061,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.48526960611343384,
      "rewards/margins": 0.19922366738319397,
      "rewards/rejected": -0.684493362903595,
      "step": 6600
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.245630753011767e-07,
      "logits/chosen": -2.018514633178711,
      "logits/rejected": -1.6095269918441772,
      "logps/chosen": -293.40185546875,
      "logps/rejected": -261.62066650390625,
      "loss": 0.1041,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.39809244871139526,
      "rewards/margins": 0.2280418872833252,
      "rewards/rejected": -0.6261343955993652,
      "step": 6610
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.176559825007408e-07,
      "logits/chosen": -2.115142345428467,
      "logits/rejected": -1.8222726583480835,
      "logps/chosen": -286.07147216796875,
      "logps/rejected": -268.15667724609375,
      "loss": 0.0929,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.4131897985935211,
      "rewards/margins": 0.17083369195461273,
      "rewards/rejected": -0.5840234756469727,
      "step": 6620
    },
    {
      "epoch": 0.8,
      "learning_rate": 6.107819083856559e-07,
      "logits/chosen": -2.1033377647399902,
      "logits/rejected": -1.6299479007720947,
      "logps/chosen": -292.8658142089844,
      "logps/rejected": -253.60104370117188,
      "loss": 0.1869,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.4031642973423004,
      "rewards/margins": 0.14742298424243927,
      "rewards/rejected": -0.5505872964859009,
      "step": 6630
    },
    {
      "epoch": 0.8,
      "learning_rate": 6.039409735358418e-07,
      "logits/chosen": -1.938940405845642,
      "logits/rejected": -1.6774669885635376,
      "logps/chosen": -264.48992919921875,
      "logps/rejected": -243.3132781982422,
      "loss": 0.0917,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.4583476483821869,
      "rewards/margins": 0.227961927652359,
      "rewards/rejected": -0.6863095164299011,
      "step": 6640
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.971332979499112e-07,
      "logits/chosen": -1.9997894763946533,
      "logits/rejected": -1.6866439580917358,
      "logps/chosen": -228.4779815673828,
      "logps/rejected": -202.27304077148438,
      "loss": 0.0927,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.4338861405849457,
      "rewards/margins": 0.186149001121521,
      "rewards/rejected": -0.6200351715087891,
      "step": 6650
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.903590010430732e-07,
      "logits/chosen": -1.9610633850097656,
      "logits/rejected": -1.4865853786468506,
      "logps/chosen": -247.3853302001953,
      "logps/rejected": -217.017333984375,
      "loss": 0.1559,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4397171139717102,
      "rewards/margins": 0.16036757826805115,
      "rewards/rejected": -0.600084662437439,
      "step": 6660
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.836182016450273e-07,
      "logits/chosen": -1.881838083267212,
      "logits/rejected": -1.6120822429656982,
      "logps/chosen": -277.1618957519531,
      "logps/rejected": -234.37094116210938,
      "loss": 0.1946,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.4851130545139313,
      "rewards/margins": 0.14165827631950378,
      "rewards/rejected": -0.6267713308334351,
      "step": 6670
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.769110179978874e-07,
      "logits/chosen": -2.084548234939575,
      "logits/rejected": -1.870194435119629,
      "logps/chosen": -213.6473388671875,
      "logps/rejected": -241.52255249023438,
      "loss": 0.1835,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.41156840324401855,
      "rewards/margins": 0.1030372828245163,
      "rewards/rejected": -0.514605700969696,
      "step": 6680
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.702375677541037e-07,
      "logits/chosen": -1.8728317022323608,
      "logits/rejected": -1.7525784969329834,
      "logps/chosen": -246.52294921875,
      "logps/rejected": -250.93368530273438,
      "loss": 0.0981,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.4282185435295105,
      "rewards/margins": 0.15069648623466492,
      "rewards/rejected": -0.578914999961853,
      "step": 6690
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.635979679744006e-07,
      "logits/chosen": -1.756136178970337,
      "logits/rejected": -1.4906980991363525,
      "logps/chosen": -233.78359985351562,
      "logps/rejected": -230.1853485107422,
      "loss": 0.0827,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4131496548652649,
      "rewards/margins": 0.11998526751995087,
      "rewards/rejected": -0.533134937286377,
      "step": 6700
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.569923351257223e-07,
      "logits/chosen": -1.9852508306503296,
      "logits/rejected": -1.7096633911132812,
      "logps/chosen": -241.21292114257812,
      "logps/rejected": -279.4994201660156,
      "loss": 0.1367,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.38888391852378845,
      "rewards/margins": 0.18084433674812317,
      "rewards/rejected": -0.5697282552719116,
      "step": 6710
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.504207850791912e-07,
      "logits/chosen": -1.9846687316894531,
      "logits/rejected": -1.4948934316635132,
      "logps/chosen": -277.4078674316406,
      "logps/rejected": -225.47042846679688,
      "loss": 0.1489,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.4744594097137451,
      "rewards/margins": 0.17791931331157684,
      "rewards/rejected": -0.6523788571357727,
      "step": 6720
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.438834331080725e-07,
      "logits/chosen": -2.0223140716552734,
      "logits/rejected": -1.8239043951034546,
      "logps/chosen": -246.28005981445312,
      "logps/rejected": -258.4651794433594,
      "loss": 0.145,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.43237772583961487,
      "rewards/margins": 0.14295579493045807,
      "rewards/rejected": -0.5753334760665894,
      "step": 6730
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.373803938857558e-07,
      "logits/chosen": -1.9347299337387085,
      "logits/rejected": -1.681318998336792,
      "logps/chosen": -266.93133544921875,
      "logps/rejected": -230.0156707763672,
      "loss": 0.2025,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5939790606498718,
      "rewards/margins": 0.11487730592489243,
      "rewards/rejected": -0.708856463432312,
      "step": 6740
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.309117814837409e-07,
      "logits/chosen": -2.084141731262207,
      "logits/rejected": -1.590496301651001,
      "logps/chosen": -245.8157501220703,
      "logps/rejected": -212.6662139892578,
      "loss": 0.1119,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.33361050486564636,
      "rewards/margins": 0.17130649089813232,
      "rewards/rejected": -0.5049170255661011,
      "step": 6750
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.244777093696385e-07,
      "logits/chosen": -2.146206855773926,
      "logits/rejected": -1.6665055751800537,
      "logps/chosen": -225.45962524414062,
      "logps/rejected": -228.45425415039062,
      "loss": 0.0773,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.4145995080471039,
      "rewards/margins": 0.2185206413269043,
      "rewards/rejected": -0.6331201791763306,
      "step": 6760
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.180782904051787e-07,
      "logits/chosen": -1.8693435192108154,
      "logits/rejected": -1.7655032873153687,
      "logps/chosen": -248.69705200195312,
      "logps/rejected": -256.6230773925781,
      "loss": 0.1171,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.44969773292541504,
      "rewards/margins": 0.13754042983055115,
      "rewards/rejected": -0.5872381925582886,
      "step": 6770
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.117136368442322e-07,
      "logits/chosen": -1.9687871932983398,
      "logits/rejected": -1.5913883447647095,
      "logps/chosen": -216.0896759033203,
      "logps/rejected": -212.8240203857422,
      "loss": 0.1145,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.47843655943870544,
      "rewards/margins": 0.14962342381477356,
      "rewards/rejected": -0.628059983253479,
      "step": 6780
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.053838603308403e-07,
      "logits/chosen": -2.2426624298095703,
      "logits/rejected": -1.8825994729995728,
      "logps/chosen": -305.18011474609375,
      "logps/rejected": -311.623046875,
      "loss": 0.1362,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.4912681579589844,
      "rewards/margins": 0.1363973617553711,
      "rewards/rejected": -0.6276654601097107,
      "step": 6790
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.99089071897256e-07,
      "logits/chosen": -1.974311113357544,
      "logits/rejected": -1.5755977630615234,
      "logps/chosen": -247.4668731689453,
      "logps/rejected": -246.3483123779297,
      "loss": 0.1313,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.4030587077140808,
      "rewards/margins": 0.2043962925672531,
      "rewards/rejected": -0.6074550151824951,
      "step": 6800
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.92829381961999e-07,
      "logits/chosen": -1.7325855493545532,
      "logits/rejected": -1.5258713960647583,
      "logps/chosen": -243.26107788085938,
      "logps/rejected": -243.32833862304688,
      "loss": 0.0918,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.47821131348609924,
      "rewards/margins": 0.18177253007888794,
      "rewards/rejected": -0.6599838733673096,
      "step": 6810
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.866049003279163e-07,
      "logits/chosen": -1.9763206243515015,
      "logits/rejected": -1.603864073753357,
      "logps/chosen": -245.575439453125,
      "logps/rejected": -217.13046264648438,
      "loss": 0.1529,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.4021136164665222,
      "rewards/margins": 0.12271346896886826,
      "rewards/rejected": -0.5248271226882935,
      "step": 6820
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.80415736180257e-07,
      "logits/chosen": -1.7185128927230835,
      "logits/rejected": -1.5524407625198364,
      "logps/chosen": -206.218017578125,
      "logps/rejected": -224.42819213867188,
      "loss": 0.1003,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.502429187297821,
      "rewards/margins": 0.15189214050769806,
      "rewards/rejected": -0.6543213129043579,
      "step": 6830
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.7426199808475735e-07,
      "logits/chosen": -1.9034755229949951,
      "logits/rejected": -1.6419847011566162,
      "logps/chosen": -266.4277648925781,
      "logps/rejected": -252.0305633544922,
      "loss": 0.1589,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.46889528632164,
      "rewards/margins": 0.16565118730068207,
      "rewards/rejected": -0.6345464587211609,
      "step": 6840
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.6814379398573613e-07,
      "logits/chosen": -1.9863262176513672,
      "logits/rejected": -1.6904399394989014,
      "logps/chosen": -288.7919921875,
      "logps/rejected": -317.627197265625,
      "loss": 0.1441,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4666666090488434,
      "rewards/margins": 0.15177717804908752,
      "rewards/rejected": -0.6184438467025757,
      "step": 6850
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.6206123120419944e-07,
      "logits/chosen": -1.7895174026489258,
      "logits/rejected": -1.514021635055542,
      "logps/chosen": -262.0445251464844,
      "logps/rejected": -283.6744384765625,
      "loss": 0.1085,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.49015116691589355,
      "rewards/margins": 0.18592293560504913,
      "rewards/rejected": -0.6760741472244263,
      "step": 6860
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.5601441643596145e-07,
      "logits/chosen": -1.9775257110595703,
      "logits/rejected": -1.5629953145980835,
      "logps/chosen": -260.10284423828125,
      "logps/rejected": -243.96127319335938,
      "loss": 0.126,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.4571099877357483,
      "rewards/margins": 0.18302378058433533,
      "rewards/rejected": -0.6401337385177612,
      "step": 6870
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.500034557497709e-07,
      "logits/chosen": -1.863673448562622,
      "logits/rejected": -1.4204473495483398,
      "logps/chosen": -292.68621826171875,
      "logps/rejected": -247.7226104736328,
      "loss": 0.1288,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5655902624130249,
      "rewards/margins": 0.15709365904331207,
      "rewards/rejected": -0.7226839661598206,
      "step": 6880
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.4402845458545037e-07,
      "logits/chosen": -1.9163280725479126,
      "logits/rejected": -1.6178480386734009,
      "logps/chosen": -260.6136169433594,
      "logps/rejected": -262.7823181152344,
      "loss": 0.0979,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5299848318099976,
      "rewards/margins": 0.15895573794841766,
      "rewards/rejected": -0.6889406442642212,
      "step": 6890
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.380895177520475e-07,
      "logits/chosen": -2.119663953781128,
      "logits/rejected": -1.4375776052474976,
      "logps/chosen": -320.13934326171875,
      "logps/rejected": -282.3840026855469,
      "loss": 0.1025,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.48634710907936096,
      "rewards/margins": 0.1542474776506424,
      "rewards/rejected": -0.6405946612358093,
      "step": 6900
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.3218674942599655e-07,
      "logits/chosen": -1.989381194114685,
      "logits/rejected": -1.6404857635498047,
      "logps/chosen": -266.8316955566406,
      "logps/rejected": -250.7163848876953,
      "loss": 0.1453,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3762025237083435,
      "rewards/margins": 0.18678084015846252,
      "rewards/rejected": -0.5629833936691284,
      "step": 6910
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.263202531492877e-07,
      "logits/chosen": -1.9917621612548828,
      "logits/rejected": -1.7141485214233398,
      "logps/chosen": -256.47882080078125,
      "logps/rejected": -231.6102752685547,
      "loss": 0.1329,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5046517252922058,
      "rewards/margins": 0.1110520213842392,
      "rewards/rejected": -0.6157038807868958,
      "step": 6920
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.204901318276586e-07,
      "logits/chosen": -2.0761396884918213,
      "logits/rejected": -1.6558525562286377,
      "logps/chosen": -301.8915710449219,
      "logps/rejected": -294.1426086425781,
      "loss": 0.4635,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.49290648102760315,
      "rewards/margins": 0.2808048725128174,
      "rewards/rejected": -0.7737113237380981,
      "step": 6930
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.146964877287804e-07,
      "logits/chosen": -2.0482866764068604,
      "logits/rejected": -1.5608055591583252,
      "logps/chosen": -353.6920166015625,
      "logps/rejected": -292.26177978515625,
      "loss": 0.1426,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.44764071702957153,
      "rewards/margins": 0.165061354637146,
      "rewards/rejected": -0.6127020716667175,
      "step": 6940
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.089394224804691e-07,
      "logits/chosen": -2.0461716651916504,
      "logits/rejected": -1.643463373184204,
      "logps/chosen": -236.15139770507812,
      "logps/rejected": -208.002685546875,
      "loss": 0.1336,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.4394107460975647,
      "rewards/margins": 0.15235844254493713,
      "rewards/rejected": -0.5917690992355347,
      "step": 6950
    },
    {
      "epoch": 0.84,
      "learning_rate": 4.032190370689018e-07,
      "logits/chosen": -2.036041021347046,
      "logits/rejected": -1.5846761465072632,
      "logps/chosen": -270.18865966796875,
      "logps/rejected": -221.50357055664062,
      "loss": 0.1346,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.48520708084106445,
      "rewards/margins": 0.1285451054573059,
      "rewards/rejected": -0.6137521862983704,
      "step": 6960
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.9753543183684573e-07,
      "logits/chosen": -1.8880681991577148,
      "logits/rejected": -1.741532564163208,
      "logps/chosen": -290.96209716796875,
      "logps/rejected": -323.4405212402344,
      "loss": 0.1471,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5335294008255005,
      "rewards/margins": 0.1267957240343094,
      "rewards/rejected": -0.6603251099586487,
      "step": 6970
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.9188870648189437e-07,
      "logits/chosen": -2.0555896759033203,
      "logits/rejected": -1.7752418518066406,
      "logps/chosen": -273.6790466308594,
      "logps/rejected": -279.67437744140625,
      "loss": 0.1747,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.5123944282531738,
      "rewards/margins": 0.10983245074748993,
      "rewards/rejected": -0.622226893901825,
      "step": 6980
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.862789600547268e-07,
      "logits/chosen": -2.072603702545166,
      "logits/rejected": -1.5208203792572021,
      "logps/chosen": -228.24551391601562,
      "logps/rejected": -198.6292724609375,
      "loss": 0.1493,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.46002787351608276,
      "rewards/margins": 0.15952149033546448,
      "rewards/rejected": -0.6195493936538696,
      "step": 6990
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.8070629095736e-07,
      "logits/chosen": -2.0098140239715576,
      "logits/rejected": -1.8628854751586914,
      "logps/chosen": -278.3236389160156,
      "logps/rejected": -290.70135498046875,
      "loss": 0.1144,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.47310394048690796,
      "rewards/margins": 0.1412026733160019,
      "rewards/rejected": -0.6143065690994263,
      "step": 7000
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.7517079694143145e-07,
      "logits/chosen": -1.8572998046875,
      "logits/rejected": -1.6057090759277344,
      "logps/chosen": -219.03427124023438,
      "logps/rejected": -238.3125762939453,
      "loss": 0.1321,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.3572883903980255,
      "rewards/margins": 0.18927468359470367,
      "rewards/rejected": -0.5465630292892456,
      "step": 7010
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.696725751064778e-07,
      "logits/chosen": -1.8692944049835205,
      "logits/rejected": -1.6396774053573608,
      "logps/chosen": -249.4277801513672,
      "logps/rejected": -236.60397338867188,
      "loss": 0.1562,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.41915005445480347,
      "rewards/margins": 0.17370714247226715,
      "rewards/rejected": -0.5928572416305542,
      "step": 7020
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.6421172189823884e-07,
      "logits/chosen": -2.1776063442230225,
      "logits/rejected": -1.8611905574798584,
      "logps/chosen": -308.03680419921875,
      "logps/rejected": -264.7606506347656,
      "loss": 0.1133,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.44754162430763245,
      "rewards/margins": 0.1368330419063568,
      "rewards/rejected": -0.5843747854232788,
      "step": 7030
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.587883331069575e-07,
      "logits/chosen": -1.870141625404358,
      "logits/rejected": -1.6269527673721313,
      "logps/chosen": -300.21435546875,
      "logps/rejected": -270.22515869140625,
      "loss": 0.1038,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5208837389945984,
      "rewards/margins": 0.10919564962387085,
      "rewards/rejected": -0.630079448223114,
      "step": 7040
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.5340250386570547e-07,
      "logits/chosen": -1.9647839069366455,
      "logits/rejected": -1.6906397342681885,
      "logps/chosen": -289.46002197265625,
      "logps/rejected": -272.2466735839844,
      "loss": 0.1131,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5417642593383789,
      "rewards/margins": 0.15753653645515442,
      "rewards/rejected": -0.6993007063865662,
      "step": 7050
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.480543286487126e-07,
      "logits/chosen": -2.036736249923706,
      "logits/rejected": -1.7443698644638062,
      "logps/chosen": -250.82235717773438,
      "logps/rejected": -267.0654602050781,
      "loss": 0.111,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.4719162881374359,
      "rewards/margins": 0.19385971128940582,
      "rewards/rejected": -0.6657760143280029,
      "step": 7060
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.4274390126971035e-07,
      "logits/chosen": -1.9663559198379517,
      "logits/rejected": -1.701615571975708,
      "logps/chosen": -236.70419311523438,
      "logps/rejected": -215.01290893554688,
      "loss": 0.2167,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.4362650513648987,
      "rewards/margins": 0.10641946643590927,
      "rewards/rejected": -0.5426844358444214,
      "step": 7070
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.374713148802827e-07,
      "logits/chosen": -2.056093692779541,
      "logits/rejected": -1.5538482666015625,
      "logps/chosen": -264.41412353515625,
      "logps/rejected": -251.5968780517578,
      "loss": 0.1848,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.46888089179992676,
      "rewards/margins": 0.1688876450061798,
      "rewards/rejected": -0.6377686262130737,
      "step": 7080
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.3223666196823963e-07,
      "logits/chosen": -2.1422367095947266,
      "logits/rejected": -1.6432521343231201,
      "logps/chosen": -332.3525695800781,
      "logps/rejected": -257.8883361816406,
      "loss": 0.139,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.5275644063949585,
      "rewards/margins": 0.13949260115623474,
      "rewards/rejected": -0.6670569181442261,
      "step": 7090
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.27040034355986e-07,
      "logits/chosen": -1.8350646495819092,
      "logits/rejected": -1.7812795639038086,
      "logps/chosen": -248.4701690673828,
      "logps/rejected": -259.24176025390625,
      "loss": 0.1682,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.523715615272522,
      "rewards/margins": 0.1881420910358429,
      "rewards/rejected": -0.7118576765060425,
      "step": 7100
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.218815231989167e-07,
      "logits/chosen": -2.013810157775879,
      "logits/rejected": -1.7800830602645874,
      "logps/chosen": -269.4825744628906,
      "logps/rejected": -250.37014770507812,
      "loss": 0.1172,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.47187668085098267,
      "rewards/margins": 0.1158447265625,
      "rewards/rejected": -0.5877213478088379,
      "step": 7110
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.1676121898381597e-07,
      "logits/chosen": -1.7077372074127197,
      "logits/rejected": -1.537479281425476,
      "logps/chosen": -283.3507385253906,
      "logps/rejected": -296.3389892578125,
      "loss": 0.0994,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.4422314167022705,
      "rewards/margins": 0.15675051510334015,
      "rewards/rejected": -0.5989819765090942,
      "step": 7120
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.1167921152727096e-07,
      "logits/chosen": -1.9554319381713867,
      "logits/rejected": -1.6862990856170654,
      "logps/chosen": -265.57684326171875,
      "logps/rejected": -236.5033416748047,
      "loss": 0.1686,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.41017451882362366,
      "rewards/margins": 0.1357969343662262,
      "rewards/rejected": -0.5459714531898499,
      "step": 7130
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.066355899740925e-07,
      "logits/chosen": -1.9464342594146729,
      "logits/rejected": -1.6717723608016968,
      "logps/chosen": -255.412841796875,
      "logps/rejected": -277.73638916015625,
      "loss": 0.0925,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.48311883211135864,
      "rewards/margins": 0.16044536232948303,
      "rewards/rejected": -0.6435642242431641,
      "step": 7140
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.0163044279575865e-07,
      "logits/chosen": -2.0800702571868896,
      "logits/rejected": -1.6157634258270264,
      "logps/chosen": -273.6175231933594,
      "logps/rejected": -198.8623809814453,
      "loss": 0.1335,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.4554726481437683,
      "rewards/margins": 0.1640358418226242,
      "rewards/rejected": -0.6195084452629089,
      "step": 7150
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.966638577888548e-07,
      "logits/chosen": -1.9913661479949951,
      "logits/rejected": -1.7240318059921265,
      "logps/chosen": -269.86920166015625,
      "logps/rejected": -281.88836669921875,
      "loss": 0.1086,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.5008378028869629,
      "rewards/margins": 0.14693915843963623,
      "rewards/rejected": -0.6477769613265991,
      "step": 7160
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.917359220735386e-07,
      "logits/chosen": -1.7666356563568115,
      "logits/rejected": -1.6807903051376343,
      "logps/chosen": -212.5333709716797,
      "logps/rejected": -204.54331970214844,
      "loss": 0.1653,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.5168853998184204,
      "rewards/margins": 0.08568285405635834,
      "rewards/rejected": -0.6025682687759399,
      "step": 7170
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.8684672209201067e-07,
      "logits/chosen": -1.8947250843048096,
      "logits/rejected": -1.5195536613464355,
      "logps/chosen": -257.7271423339844,
      "logps/rejected": -226.953125,
      "loss": 0.1413,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.48005929589271545,
      "rewards/margins": 0.11954379081726074,
      "rewards/rejected": -0.5996031165122986,
      "step": 7180
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.819963436069986e-07,
      "logits/chosen": -2.003467321395874,
      "logits/rejected": -1.6604106426239014,
      "logps/chosen": -301.46466064453125,
      "logps/rejected": -248.64669799804688,
      "loss": 0.1251,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.47957324981689453,
      "rewards/margins": 0.09554027020931244,
      "rewards/rejected": -0.5751134753227234,
      "step": 7190
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.771848717002498e-07,
      "logits/chosen": -1.7738151550292969,
      "logits/rejected": -1.6896775960922241,
      "logps/chosen": -236.6964569091797,
      "logps/rejected": -270.410888671875,
      "loss": 0.1098,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5385271310806274,
      "rewards/margins": 0.1548021137714386,
      "rewards/rejected": -0.6933292746543884,
      "step": 7200
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.724123907710444e-07,
      "logits/chosen": -1.7751373052597046,
      "logits/rejected": -1.565288782119751,
      "logps/chosen": -195.97508239746094,
      "logps/rejected": -213.2833251953125,
      "loss": 0.1312,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.466492235660553,
      "rewards/margins": 0.1685236245393753,
      "rewards/rejected": -0.6350158452987671,
      "step": 7210
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.6767898453470886e-07,
      "logits/chosen": -2.038952350616455,
      "logits/rejected": -1.636182427406311,
      "logps/chosen": -237.19473266601562,
      "logps/rejected": -223.7322235107422,
      "loss": 0.1162,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.383628785610199,
      "rewards/margins": 0.1849764585494995,
      "rewards/rejected": -0.5686052441596985,
      "step": 7220
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.629847360211518e-07,
      "logits/chosen": -1.9980814456939697,
      "logits/rejected": -1.6197658777236938,
      "logps/chosen": -247.9432830810547,
      "logps/rejected": -251.46630859375,
      "loss": 0.0962,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.4834599494934082,
      "rewards/margins": 0.15811513364315033,
      "rewards/rejected": -0.6415750980377197,
      "step": 7230
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.5832972757340565e-07,
      "logits/chosen": -2.032080888748169,
      "logits/rejected": -1.8052698373794556,
      "logps/chosen": -242.31698608398438,
      "logps/rejected": -257.6308898925781,
      "loss": 0.142,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.44113120436668396,
      "rewards/margins": 0.11361245810985565,
      "rewards/rejected": -0.5547436475753784,
      "step": 7240
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.53714040846183e-07,
      "logits/chosen": -1.9440562725067139,
      "logits/rejected": -1.5986840724945068,
      "logps/chosen": -284.2532958984375,
      "logps/rejected": -236.9690704345703,
      "loss": 0.1171,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3770473003387451,
      "rewards/margins": 0.23315231502056122,
      "rewards/rejected": -0.6101996302604675,
      "step": 7250
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.491377568044434e-07,
      "logits/chosen": -2.0843589305877686,
      "logits/rejected": -1.6826503276824951,
      "logps/chosen": -314.3594665527344,
      "logps/rejected": -272.8559875488281,
      "loss": 0.0734,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5402888059616089,
      "rewards/margins": 0.14350660145282745,
      "rewards/rejected": -0.6837953925132751,
      "step": 7260
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.4460095572197476e-07,
      "logits/chosen": -2.0028257369995117,
      "logits/rejected": -1.6907081604003906,
      "logps/chosen": -238.27001953125,
      "logps/rejected": -250.3758544921875,
      "loss": 0.1027,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.4421817660331726,
      "rewards/margins": 0.1625339388847351,
      "rewards/rejected": -0.6047157049179077,
      "step": 7270
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.401037171799819e-07,
      "logits/chosen": -1.9249871969223022,
      "logits/rejected": -1.5126729011535645,
      "logps/chosen": -267.6981506347656,
      "logps/rejected": -230.08407592773438,
      "loss": 0.1672,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.3991939127445221,
      "rewards/margins": 0.1651068925857544,
      "rewards/rejected": -0.5643008351325989,
      "step": 7280
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.3564612006569482e-07,
      "logits/chosen": -2.02858304977417,
      "logits/rejected": -1.7860488891601562,
      "logps/chosen": -264.84002685546875,
      "logps/rejected": -264.8438415527344,
      "loss": 0.1607,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.45592936873435974,
      "rewards/margins": 0.1130853146314621,
      "rewards/rejected": -0.5690146684646606,
      "step": 7290
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.3122824257098275e-07,
      "logits/chosen": -1.7619152069091797,
      "logits/rejected": -1.4626259803771973,
      "logps/chosen": -230.26889038085938,
      "logps/rejected": -224.6363525390625,
      "loss": 0.0667,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.4507187008857727,
      "rewards/margins": 0.16845285892486572,
      "rewards/rejected": -0.6191716194152832,
      "step": 7300
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.2685016219098187e-07,
      "logits/chosen": -2.1385879516601562,
      "logits/rejected": -1.5403960943222046,
      "logps/chosen": -266.4300231933594,
      "logps/rejected": -216.80722045898438,
      "loss": 0.1237,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.39975208044052124,
      "rewards/margins": 0.22581128776073456,
      "rewards/rejected": -0.625563383102417,
      "step": 7310
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.2251195572273758e-07,
      "logits/chosen": -2.1694719791412354,
      "logits/rejected": -1.607452154159546,
      "logps/chosen": -341.0927734375,
      "logps/rejected": -272.94464111328125,
      "loss": 0.0914,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4034740924835205,
      "rewards/margins": 0.19167360663414001,
      "rewards/rejected": -0.5951477289199829,
      "step": 7320
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.18213699263857e-07,
      "logits/chosen": -1.9197025299072266,
      "logits/rejected": -1.6683180332183838,
      "logps/chosen": -291.236572265625,
      "logps/rejected": -316.86737060546875,
      "loss": 0.0876,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.47617173194885254,
      "rewards/margins": 0.2178315371274948,
      "rewards/rejected": -0.6940032839775085,
      "step": 7330
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.1395546821117192e-07,
      "logits/chosen": -1.8608232736587524,
      "logits/rejected": -1.577675461769104,
      "logps/chosen": -279.8750915527344,
      "logps/rejected": -252.7456817626953,
      "loss": 0.1519,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4677848815917969,
      "rewards/margins": 0.1569843590259552,
      "rewards/rejected": -0.6247692108154297,
      "step": 7340
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.097373372594197e-07,
      "logits/chosen": -2.017251968383789,
      "logits/rejected": -1.649173378944397,
      "logps/chosen": -284.208984375,
      "logps/rejected": -264.468505859375,
      "loss": 0.1668,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.47412624955177307,
      "rewards/margins": 0.12772853672504425,
      "rewards/rejected": -0.6018548011779785,
      "step": 7350
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.0555938039993145e-07,
      "logits/chosen": -2.207703113555908,
      "logits/rejected": -1.7608064413070679,
      "logps/chosen": -317.61566162109375,
      "logps/rejected": -273.2945861816406,
      "loss": 0.1079,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.3838236629962921,
      "rewards/margins": 0.15046364068984985,
      "rewards/rejected": -0.5342873334884644,
      "step": 7360
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.0142167091933368e-07,
      "logits/chosen": -1.8200185298919678,
      "logits/rejected": -1.7389323711395264,
      "logps/chosen": -233.045654296875,
      "logps/rejected": -263.55535888671875,
      "loss": 0.1277,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.4523780941963196,
      "rewards/margins": 0.1272934377193451,
      "rewards/rejected": -0.5796715617179871,
      "step": 7370
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.973242813982626e-07,
      "logits/chosen": -1.7422631978988647,
      "logits/rejected": -1.4749271869659424,
      "logps/chosen": -223.44418334960938,
      "logps/rejected": -217.0834503173828,
      "loss": 0.1286,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.490098237991333,
      "rewards/margins": 0.15991918742656708,
      "rewards/rejected": -0.6500174403190613,
      "step": 7380
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.932672837100924e-07,
      "logits/chosen": -2.1760799884796143,
      "logits/rejected": -1.4984136819839478,
      "logps/chosen": -262.04400634765625,
      "logps/rejected": -240.68911743164062,
      "loss": 0.1252,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.49469202756881714,
      "rewards/margins": 0.19546754658222198,
      "rewards/rejected": -0.6901595592498779,
      "step": 7390
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.8925074901967406e-07,
      "logits/chosen": -2.032710313796997,
      "logits/rejected": -1.4763513803482056,
      "logps/chosen": -284.44317626953125,
      "logps/rejected": -255.60302734375,
      "loss": 0.0695,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4748278558254242,
      "rewards/margins": 0.16811513900756836,
      "rewards/rejected": -0.6429430246353149,
      "step": 7400
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.8527474778208458e-07,
      "logits/chosen": -1.841803789138794,
      "logits/rejected": -1.7014901638031006,
      "logps/chosen": -180.16915893554688,
      "logps/rejected": -192.54147338867188,
      "loss": 0.1502,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.42819100618362427,
      "rewards/margins": 0.07826290279626846,
      "rewards/rejected": -0.5064539313316345,
      "step": 7410
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.813393497413951e-07,
      "logits/chosen": -1.9294426441192627,
      "logits/rejected": -1.611358880996704,
      "logps/chosen": -286.4609680175781,
      "logps/rejected": -255.0117645263672,
      "loss": 0.0994,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.4472865164279938,
      "rewards/margins": 0.13087594509124756,
      "rewards/rejected": -0.5781623721122742,
      "step": 7420
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.7744462392944472e-07,
      "logits/chosen": -2.1310625076293945,
      "logits/rejected": -1.6776885986328125,
      "logps/chosen": -288.6611328125,
      "logps/rejected": -262.71453857421875,
      "loss": 0.1262,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.47246164083480835,
      "rewards/margins": 0.1253913938999176,
      "rewards/rejected": -0.5978530049324036,
      "step": 7430
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.7359063866463048e-07,
      "logits/chosen": -2.0324885845184326,
      "logits/rejected": -1.6409807205200195,
      "logps/chosen": -239.01052856445312,
      "logps/rejected": -202.0733642578125,
      "loss": 0.1716,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.3990083932876587,
      "rewards/margins": 0.19792206585407257,
      "rewards/rejected": -0.5969304442405701,
      "step": 7440
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.6977746155070946e-07,
      "logits/chosen": -1.877323865890503,
      "logits/rejected": -1.8759233951568604,
      "logps/chosen": -244.84945678710938,
      "logps/rejected": -282.86456298828125,
      "loss": 0.1464,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.4934001564979553,
      "rewards/margins": 0.12719208002090454,
      "rewards/rejected": -0.6205921769142151,
      "step": 7450
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.6600515947561207e-07,
      "logits/chosen": -2.076573133468628,
      "logits/rejected": -1.5323445796966553,
      "logps/chosen": -282.5911560058594,
      "logps/rejected": -220.7271728515625,
      "loss": 0.1678,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4402221739292145,
      "rewards/margins": 0.16551920771598816,
      "rewards/rejected": -0.6057413220405579,
      "step": 7460
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.6227379861026738e-07,
      "logits/chosen": -2.02341365814209,
      "logits/rejected": -1.6644586324691772,
      "logps/chosen": -255.2810821533203,
      "logps/rejected": -251.2646484375,
      "loss": 0.151,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.4583802819252014,
      "rewards/margins": 0.15048515796661377,
      "rewards/rejected": -0.60886549949646,
      "step": 7470
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.5858344440744745e-07,
      "logits/chosen": -2.074061632156372,
      "logits/rejected": -1.6793878078460693,
      "logps/chosen": -275.97467041015625,
      "logps/rejected": -275.2462463378906,
      "loss": 0.128,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.4571126103401184,
      "rewards/margins": 0.14766040444374084,
      "rewards/rejected": -0.6047729253768921,
      "step": 7480
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.5493416160061254e-07,
      "logits/chosen": -2.1556544303894043,
      "logits/rejected": -1.7006380558013916,
      "logps/chosen": -289.0265197753906,
      "logps/rejected": -254.0576171875,
      "loss": 0.179,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.4663164019584656,
      "rewards/margins": 0.13743355870246887,
      "rewards/rejected": -0.6037499904632568,
      "step": 7490
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.5132601420278086e-07,
      "logits/chosen": -1.9497236013412476,
      "logits/rejected": -1.6325147151947021,
      "logps/chosen": -267.0816955566406,
      "logps/rejected": -226.15576171875,
      "loss": 0.0762,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.47710466384887695,
      "rewards/margins": 0.14392545819282532,
      "rewards/rejected": -0.6210300326347351,
      "step": 7500
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.4775906550540287e-07,
      "logits/chosen": -1.7985035181045532,
      "logits/rejected": -1.484886884689331,
      "logps/chosen": -217.06076049804688,
      "logps/rejected": -206.60055541992188,
      "loss": 0.1064,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.443693071603775,
      "rewards/margins": 0.13600948452949524,
      "rewards/rejected": -0.5797025561332703,
      "step": 7510
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.4423337807725286e-07,
      "logits/chosen": -1.957148790359497,
      "logits/rejected": -1.8867321014404297,
      "logps/chosen": -207.6150360107422,
      "logps/rejected": -233.46658325195312,
      "loss": 0.0856,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.4386165142059326,
      "rewards/margins": 0.1362169086933136,
      "rewards/rejected": -0.5748334527015686,
      "step": 7520
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.4074901376332855e-07,
      "logits/chosen": -1.9299287796020508,
      "logits/rejected": -1.9098894596099854,
      "logps/chosen": -274.15850830078125,
      "logps/rejected": -299.91265869140625,
      "loss": 0.0883,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.47814321517944336,
      "rewards/margins": 0.13489031791687012,
      "rewards/rejected": -0.6130335927009583,
      "step": 7530
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.3730603368377088e-07,
      "logits/chosen": -1.915826439857483,
      "logits/rejected": -1.7198493480682373,
      "logps/chosen": -296.31182861328125,
      "logps/rejected": -286.13592529296875,
      "loss": 0.0992,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.4671846926212311,
      "rewards/margins": 0.1536525934934616,
      "rewards/rejected": -0.6208373308181763,
      "step": 7540
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.3390449823278666e-07,
      "logits/chosen": -1.976479172706604,
      "logits/rejected": -1.7112070322036743,
      "logps/chosen": -309.96136474609375,
      "logps/rejected": -279.93597412109375,
      "loss": 0.2023,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4522860646247864,
      "rewards/margins": 0.08242306113243103,
      "rewards/rejected": -0.5347092151641846,
      "step": 7550
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.3054446707759323e-07,
      "logits/chosen": -2.088073968887329,
      "logits/rejected": -1.6631252765655518,
      "logps/chosen": -269.19268798828125,
      "logps/rejected": -219.1171112060547,
      "loss": 0.0938,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.4191514551639557,
      "rewards/margins": 0.1780991554260254,
      "rewards/rejected": -0.5972505807876587,
      "step": 7560
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.2722599915736962e-07,
      "logits/chosen": -1.9217841625213623,
      "logits/rejected": -1.4964293241500854,
      "logps/chosen": -215.7314910888672,
      "logps/rejected": -203.1638641357422,
      "loss": 0.161,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.4369204640388489,
      "rewards/margins": 0.19439806044101715,
      "rewards/rejected": -0.6313184499740601,
      "step": 7570
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.2394915268222423e-07,
      "logits/chosen": -1.7211532592773438,
      "logits/rejected": -1.4849998950958252,
      "logps/chosen": -253.0435028076172,
      "logps/rejected": -240.583740234375,
      "loss": 0.1213,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.45572876930236816,
      "rewards/margins": 0.12647785246372223,
      "rewards/rejected": -0.5822066068649292,
      "step": 7580
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.2071398513217118e-07,
      "logits/chosen": -1.8722648620605469,
      "logits/rejected": -1.4593003988265991,
      "logps/chosen": -299.0888366699219,
      "logps/rejected": -245.06686401367188,
      "loss": 0.1898,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.45327168703079224,
      "rewards/margins": 0.16469185054302216,
      "rewards/rejected": -0.6179635524749756,
      "step": 7590
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1752055325612605e-07,
      "logits/chosen": -2.0962700843811035,
      "logits/rejected": -1.614005446434021,
      "logps/chosen": -284.02618408203125,
      "logps/rejected": -245.89785766601562,
      "loss": 0.1521,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.3982982933521271,
      "rewards/margins": 0.14497140049934387,
      "rewards/rejected": -0.5432697534561157,
      "step": 7600
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.143689130709058e-07,
      "logits/chosen": -1.713621735572815,
      "logits/rejected": -1.4907618761062622,
      "logps/chosen": -288.0763854980469,
      "logps/rejected": -314.97869873046875,
      "loss": 0.067,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.4883494973182678,
      "rewards/margins": 0.1738799810409546,
      "rewards/rejected": -0.6622294187545776,
      "step": 7610
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1125911986025001e-07,
      "logits/chosen": -1.7698646783828735,
      "logits/rejected": -1.6841932535171509,
      "logps/chosen": -350.5254211425781,
      "logps/rejected": -300.12103271484375,
      "loss": 0.1248,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.5203927755355835,
      "rewards/margins": 0.08340970426797867,
      "rewards/rejected": -0.603802502155304,
      "step": 7620
    },
    {
      "epoch": 0.92,
      "learning_rate": 1.0819122817384897e-07,
      "logits/chosen": -1.9861576557159424,
      "logits/rejected": -1.5860307216644287,
      "logps/chosen": -254.307861328125,
      "logps/rejected": -279.30615234375,
      "loss": 0.1191,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5575435757637024,
      "rewards/margins": 0.1310313642024994,
      "rewards/rejected": -0.6885749101638794,
      "step": 7630
    },
    {
      "epoch": 0.92,
      "learning_rate": 1.0516529182638819e-07,
      "logits/chosen": -1.825350046157837,
      "logits/rejected": -1.5633935928344727,
      "logps/chosen": -293.57757568359375,
      "logps/rejected": -316.1748962402344,
      "loss": 0.1375,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5138299465179443,
      "rewards/margins": 0.13412019610404968,
      "rewards/rejected": -0.6479502320289612,
      "step": 7640
    },
    {
      "epoch": 0.92,
      "learning_rate": 1.0218136389660211e-07,
      "logits/chosen": -1.9528892040252686,
      "logits/rejected": -1.6495583057403564,
      "logps/chosen": -300.08294677734375,
      "logps/rejected": -279.8624572753906,
      "loss": 0.1515,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.441044420003891,
      "rewards/margins": 0.08407802134752274,
      "rewards/rejected": -0.5251224040985107,
      "step": 7650
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.923949672634714e-08,
      "logits/chosen": -1.7535518407821655,
      "logits/rejected": -1.5248991250991821,
      "logps/chosen": -286.5433654785156,
      "logps/rejected": -296.0802001953125,
      "loss": 0.148,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.519503116607666,
      "rewards/margins": 0.18411189317703247,
      "rewards/rejected": -0.7036150097846985,
      "step": 7660
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.633974191967794e-08,
      "logits/chosen": -1.9656978845596313,
      "logits/rejected": -1.546311616897583,
      "logps/chosen": -251.5748748779297,
      "logps/rejected": -249.95162963867188,
      "loss": 0.1335,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3971438705921173,
      "rewards/margins": 0.2471657246351242,
      "rewards/rejected": -0.6443095207214355,
      "step": 7670
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.348215034194752e-08,
      "logits/chosen": -1.918678879737854,
      "logits/rejected": -1.3947335481643677,
      "logps/chosen": -295.56622314453125,
      "logps/rejected": -266.44805908203125,
      "loss": 0.103,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.43143653869628906,
      "rewards/margins": 0.18379929661750793,
      "rewards/rejected": -0.6152359247207642,
      "step": 7680
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.066677211891195e-08,
      "logits/chosen": -1.9028263092041016,
      "logits/rejected": -1.4752349853515625,
      "logps/chosen": -219.3140869140625,
      "logps/rejected": -224.39013671875,
      "loss": 0.1199,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.47012224793434143,
      "rewards/margins": 0.2023288756608963,
      "rewards/rejected": -0.6724511384963989,
      "step": 7690
    },
    {
      "epoch": 0.92,
      "learning_rate": 8.789365663585208e-08,
      "logits/chosen": -2.116147518157959,
      "logits/rejected": -1.8864881992340088,
      "logps/chosen": -283.5893249511719,
      "logps/rejected": -234.08224487304688,
      "loss": 0.1225,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.4223853051662445,
      "rewards/margins": 0.1264767050743103,
      "rewards/rejected": -0.5488620400428772,
      "step": 7700
    },
    {
      "epoch": 0.93,
      "learning_rate": 8.516285253670597e-08,
      "logits/chosen": -1.9596683979034424,
      "logits/rejected": -1.6173160076141357,
      "logps/chosen": -230.27188110351562,
      "logps/rejected": -205.488525390625,
      "loss": 0.1671,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5056071281433105,
      "rewards/margins": 0.17759716510772705,
      "rewards/rejected": -0.6832043528556824,
      "step": 7710
    },
    {
      "epoch": 0.93,
      "learning_rate": 8.247440772321924e-08,
      "logits/chosen": -1.8863664865493774,
      "logits/rejected": -1.7962907552719116,
      "logps/chosen": -265.54937744140625,
      "logps/rejected": -279.07379150390625,
      "loss": 0.1479,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5093497037887573,
      "rewards/margins": 0.14122812449932098,
      "rewards/rejected": -0.6505778431892395,
      "step": 7720
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.982836935409938e-08,
      "logits/chosen": -1.9433815479278564,
      "logits/rejected": -1.6725490093231201,
      "logps/chosen": -274.528564453125,
      "logps/rejected": -256.54766845703125,
      "loss": 0.1345,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5039865970611572,
      "rewards/margins": 0.16849592328071594,
      "rewards/rejected": -0.6724825501441956,
      "step": 7730
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.722478384419335e-08,
      "logits/chosen": -1.8644914627075195,
      "logits/rejected": -1.4965304136276245,
      "logps/chosen": -278.33746337890625,
      "logps/rejected": -243.71817016601562,
      "loss": 0.1328,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.49178391695022583,
      "rewards/margins": 0.12286220490932465,
      "rewards/rejected": -0.6146460771560669,
      "step": 7740
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.466369686367075e-08,
      "logits/chosen": -2.0346133708953857,
      "logits/rejected": -1.6774375438690186,
      "logps/chosen": -293.4043273925781,
      "logps/rejected": -247.53579711914062,
      "loss": 0.1475,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.46793246269226074,
      "rewards/margins": 0.09048617631196976,
      "rewards/rejected": -0.5584186315536499,
      "step": 7750
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.21451533372236e-08,
      "logits/chosen": -1.9626423120498657,
      "logits/rejected": -1.580739974975586,
      "logps/chosen": -258.93060302734375,
      "logps/rejected": -241.1245574951172,
      "loss": 0.1536,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.47752898931503296,
      "rewards/margins": 0.1563757061958313,
      "rewards/rejected": -0.6339046955108643,
      "step": 7760
    },
    {
      "epoch": 0.93,
      "learning_rate": 6.966919744327783e-08,
      "logits/chosen": -2.0937347412109375,
      "logits/rejected": -1.5341440439224243,
      "logps/chosen": -341.6336669921875,
      "logps/rejected": -260.72589111328125,
      "loss": 0.1603,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.42314642667770386,
      "rewards/margins": 0.19263425469398499,
      "rewards/rejected": -0.6157806515693665,
      "step": 7770
    },
    {
      "epoch": 0.93,
      "learning_rate": 6.723587261321912e-08,
      "logits/chosen": -1.9547611474990845,
      "logits/rejected": -1.642380714416504,
      "logps/chosen": -297.21649169921875,
      "logps/rejected": -273.52459716796875,
      "loss": 0.1617,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.4811842441558838,
      "rewards/margins": 0.11549937725067139,
      "rewards/rejected": -0.5966835618019104,
      "step": 7780
    },
    {
      "epoch": 0.93,
      "learning_rate": 6.484522153063056e-08,
      "logits/chosen": -1.9168577194213867,
      "logits/rejected": -1.5155632495880127,
      "logps/chosen": -225.7365264892578,
      "logps/rejected": -204.46615600585938,
      "loss": 0.1319,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.4766554832458496,
      "rewards/margins": 0.1555982381105423,
      "rewards/rejected": -0.6322537064552307,
      "step": 7790
    },
    {
      "epoch": 0.94,
      "learning_rate": 6.249728613054313e-08,
      "logits/chosen": -1.7597625255584717,
      "logits/rejected": -1.5485173463821411,
      "logps/chosen": -264.35321044921875,
      "logps/rejected": -273.6183166503906,
      "loss": 0.1479,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.48368602991104126,
      "rewards/margins": 0.07144041359424591,
      "rewards/rejected": -0.5551263689994812,
      "step": 7800
    },
    {
      "epoch": 0.94,
      "learning_rate": 6.01921075987022e-08,
      "logits/chosen": -1.9097301959991455,
      "logits/rejected": -1.186971664428711,
      "logps/chosen": -246.1496124267578,
      "logps/rejected": -194.6399688720703,
      "loss": 0.1488,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.4605187773704529,
      "rewards/margins": 0.23739011585712433,
      "rewards/rejected": -0.697908878326416,
      "step": 7810
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.7929726370843096e-08,
      "logits/chosen": -2.1597225666046143,
      "logits/rejected": -1.8278968334197998,
      "logps/chosen": -254.0515594482422,
      "logps/rejected": -271.48236083984375,
      "loss": 0.1766,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.4439367651939392,
      "rewards/margins": 0.2024269998073578,
      "rewards/rejected": -0.6463637351989746,
      "step": 7820
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.5710182131981927e-08,
      "logits/chosen": -1.99951171875,
      "logits/rejected": -1.7067577838897705,
      "logps/chosen": -238.32467651367188,
      "logps/rejected": -256.7763671875,
      "loss": 0.1304,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.39582788944244385,
      "rewards/margins": 0.21840229630470276,
      "rewards/rejected": -0.614230215549469,
      "step": 7830
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.3533513815721694e-08,
      "logits/chosen": -2.118350028991699,
      "logits/rejected": -1.6194887161254883,
      "logps/chosen": -272.92449951171875,
      "logps/rejected": -227.6458282470703,
      "loss": 0.0858,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.43067464232444763,
      "rewards/margins": 0.21112823486328125,
      "rewards/rejected": -0.6418029069900513,
      "step": 7840
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.1399759603565916e-08,
      "logits/chosen": -2.176056385040283,
      "logits/rejected": -1.72799551486969,
      "logps/chosen": -259.24176025390625,
      "logps/rejected": -220.6403350830078,
      "loss": 0.1577,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.41245096921920776,
      "rewards/margins": 0.16166391968727112,
      "rewards/rejected": -0.5741148591041565,
      "step": 7850
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.930895692425192e-08,
      "logits/chosen": -2.08243989944458,
      "logits/rejected": -1.6198593378067017,
      "logps/chosen": -313.93475341796875,
      "logps/rejected": -302.1040344238281,
      "loss": 0.0815,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.48940593004226685,
      "rewards/margins": 0.1875011920928955,
      "rewards/rejected": -0.6769071817398071,
      "step": 7860
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.726114245309249e-08,
      "logits/chosen": -2.1306631565093994,
      "logits/rejected": -1.8443044424057007,
      "logps/chosen": -260.3841552734375,
      "logps/rejected": -240.7478790283203,
      "loss": 0.084,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.4020455777645111,
      "rewards/margins": 0.11921534687280655,
      "rewards/rejected": -0.5212609171867371,
      "step": 7870
    },
    {
      "epoch": 0.95,
      "learning_rate": 4.5256352111333334e-08,
      "logits/chosen": -2.197538137435913,
      "logits/rejected": -1.9828275442123413,
      "logps/chosen": -265.1056213378906,
      "logps/rejected": -240.36581420898438,
      "loss": 0.1341,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.41930776834487915,
      "rewards/margins": 0.10385145992040634,
      "rewards/rejected": -0.5231592059135437,
      "step": 7880
    },
    {
      "epoch": 0.95,
      "learning_rate": 4.32946210655219e-08,
      "logits/chosen": -1.879839539527893,
      "logits/rejected": -1.6344287395477295,
      "logps/chosen": -289.47540283203125,
      "logps/rejected": -303.8387756347656,
      "loss": 0.0844,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.4271796643733978,
      "rewards/margins": 0.12638108432292938,
      "rewards/rejected": -0.553560733795166,
      "step": 7890
    },
    {
      "epoch": 0.95,
      "learning_rate": 4.137598372689289e-08,
      "logits/chosen": -1.9711477756500244,
      "logits/rejected": -1.640442132949829,
      "logps/chosen": -289.0892639160156,
      "logps/rejected": -256.19390869140625,
      "loss": 0.1124,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.5559543967247009,
      "rewards/margins": 0.06960447132587433,
      "rewards/rejected": -0.6255587935447693,
      "step": 7900
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.950047375076177e-08,
      "logits/chosen": -1.9874846935272217,
      "logits/rejected": -1.746363639831543,
      "logps/chosen": -253.0285186767578,
      "logps/rejected": -267.914306640625,
      "loss": 0.151,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.4185236096382141,
      "rewards/margins": 0.170128732919693,
      "rewards/rejected": -0.5886522531509399,
      "step": 7910
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.7668124035936395e-08,
      "logits/chosen": -1.8806092739105225,
      "logits/rejected": -1.7465565204620361,
      "logps/chosen": -281.9869384765625,
      "logps/rejected": -293.6903381347656,
      "loss": 0.1098,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.48278847336769104,
      "rewards/margins": 0.16966548562049866,
      "rewards/rejected": -0.6524539589881897,
      "step": 7920
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.587896672413882e-08,
      "logits/chosen": -1.954755187034607,
      "logits/rejected": -1.6159816980361938,
      "logps/chosen": -330.97698974609375,
      "logps/rejected": -257.1192932128906,
      "loss": 0.1392,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4463822841644287,
      "rewards/margins": 0.12165029346942902,
      "rewards/rejected": -0.5680325031280518,
      "step": 7930
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.413303319944244e-08,
      "logits/chosen": -1.8800256252288818,
      "logits/rejected": -1.666426420211792,
      "logps/chosen": -255.54055786132812,
      "logps/rejected": -272.89056396484375,
      "loss": 0.1096,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5043514370918274,
      "rewards/margins": 0.13485528528690338,
      "rewards/rejected": -0.6392067670822144,
      "step": 7940
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.243035408772077e-08,
      "logits/chosen": -1.8116118907928467,
      "logits/rejected": -1.465587854385376,
      "logps/chosen": -263.7826232910156,
      "logps/rejected": -215.6946563720703,
      "loss": 0.1744,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.47774848341941833,
      "rewards/margins": 0.16809609532356262,
      "rewards/rejected": -0.645844578742981,
      "step": 7950
    },
    {
      "epoch": 0.96,
      "learning_rate": 3.077095925611007e-08,
      "logits/chosen": -1.8181393146514893,
      "logits/rejected": -1.7243306636810303,
      "logps/chosen": -255.312255859375,
      "logps/rejected": -261.9159851074219,
      "loss": 0.1284,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.49375930428504944,
      "rewards/margins": 0.1126602292060852,
      "rewards/rejected": -0.6064194440841675,
      "step": 7960
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.915487781248616e-08,
      "logits/chosen": -2.0126254558563232,
      "logits/rejected": -1.7640451192855835,
      "logps/chosen": -258.76007080078125,
      "logps/rejected": -300.79937744140625,
      "loss": 0.0698,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.43722277879714966,
      "rewards/margins": 0.21826669573783875,
      "rewards/rejected": -0.6554895043373108,
      "step": 7970
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.7582138104953748e-08,
      "logits/chosen": -1.9624278545379639,
      "logits/rejected": -1.6752490997314453,
      "logps/chosen": -219.3885955810547,
      "logps/rejected": -218.8914794921875,
      "loss": 0.1453,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.4231085181236267,
      "rewards/margins": 0.1884673833847046,
      "rewards/rejected": -0.6115759015083313,
      "step": 7980
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.6052767721348184e-08,
      "logits/chosen": -2.087956666946411,
      "logits/rejected": -1.7118041515350342,
      "logps/chosen": -260.5646667480469,
      "logps/rejected": -232.9169921875,
      "loss": 0.1097,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.46867817640304565,
      "rewards/margins": 0.12946224212646484,
      "rewards/rejected": -0.5981403589248657,
      "step": 7990
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.4566793488752795e-08,
      "logits/chosen": -1.9538524150848389,
      "logits/rejected": -1.9380991458892822,
      "logps/chosen": -240.32583618164062,
      "logps/rejected": -280.9888000488281,
      "loss": 0.1325,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.5332067608833313,
      "rewards/margins": 0.08598224818706512,
      "rewards/rejected": -0.6191889643669128,
      "step": 8000
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.3124241473027333e-08,
      "logits/chosen": -1.9650394916534424,
      "logits/rejected": -1.666481375694275,
      "logps/chosen": -229.7511444091797,
      "logps/rejected": -257.80889892578125,
      "loss": 0.1788,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.44220954179763794,
      "rewards/margins": 0.14231547713279724,
      "rewards/rejected": -0.5845250487327576,
      "step": 8010
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.1725136978351934e-08,
      "logits/chosen": -2.1228089332580566,
      "logits/rejected": -1.5950334072113037,
      "logps/chosen": -295.29400634765625,
      "logps/rejected": -244.79330444335938,
      "loss": 0.1059,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.36449581384658813,
      "rewards/margins": 0.2231093943119049,
      "rewards/rejected": -0.5876051783561707,
      "step": 8020
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.036950454678166e-08,
      "logits/chosen": -2.112056016921997,
      "logits/rejected": -1.595947504043579,
      "logps/chosen": -281.60601806640625,
      "logps/rejected": -243.0486297607422,
      "loss": 0.1566,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.3312150835990906,
      "rewards/margins": 0.19925157725811005,
      "rewards/rejected": -0.530466616153717,
      "step": 8030
    },
    {
      "epoch": 0.96,
      "learning_rate": 1.9057367957817096e-08,
      "logits/chosen": -1.9058849811553955,
      "logits/rejected": -1.4319086074829102,
      "logps/chosen": -244.7132568359375,
      "logps/rejected": -223.8097686767578,
      "loss": 0.09,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.38957375288009644,
      "rewards/margins": 0.19439074397087097,
      "rewards/rejected": -0.583964467048645,
      "step": 8040
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.778875022798693e-08,
      "logits/chosen": -1.5648739337921143,
      "logits/rejected": -1.51466965675354,
      "logps/chosen": -189.83966064453125,
      "logps/rejected": -232.4983367919922,
      "loss": 0.1424,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.5467423796653748,
      "rewards/margins": 0.11828476190567017,
      "rewards/rejected": -0.6650272011756897,
      "step": 8050
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.6563673610444363e-08,
      "logits/chosen": -2.011043071746826,
      "logits/rejected": -1.761691689491272,
      "logps/chosen": -261.29498291015625,
      "logps/rejected": -261.8739929199219,
      "loss": 0.0942,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.5379740595817566,
      "rewards/margins": 0.1529974639415741,
      "rewards/rejected": -0.6909714937210083,
      "step": 8060
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.5382159594576616e-08,
      "logits/chosen": -1.802074670791626,
      "logits/rejected": -1.4913840293884277,
      "logps/chosen": -231.8621368408203,
      "logps/rejected": -263.0806884765625,
      "loss": 0.1387,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.5238322019577026,
      "rewards/margins": 0.1918632835149765,
      "rewards/rejected": -0.7156955003738403,
      "step": 8070
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.424422890562771e-08,
      "logits/chosen": -2.1378629207611084,
      "logits/rejected": -1.9434757232666016,
      "logps/chosen": -232.4508514404297,
      "logps/rejected": -239.05911254882812,
      "loss": 0.1501,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.42561620473861694,
      "rewards/margins": 0.15531761944293976,
      "rewards/rejected": -0.5809338688850403,
      "step": 8080
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.3149901504335706e-08,
      "logits/chosen": -1.9773696660995483,
      "logits/rejected": -1.4441204071044922,
      "logps/chosen": -237.9873504638672,
      "logps/rejected": -219.8182373046875,
      "loss": 0.1467,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.43333473801612854,
      "rewards/margins": 0.14528748393058777,
      "rewards/rejected": -0.5786222219467163,
      "step": 8090
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.2099196586581596e-08,
      "logits/chosen": -1.9610668420791626,
      "logits/rejected": -1.8130521774291992,
      "logps/chosen": -229.5089569091797,
      "logps/rejected": -231.4257354736328,
      "loss": 0.1316,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.501410722732544,
      "rewards/margins": 0.13243083655834198,
      "rewards/rejected": -0.6338415741920471,
      "step": 8100
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.1092132583053472e-08,
      "logits/chosen": -1.984513521194458,
      "logits/rejected": -1.6125694513320923,
      "logps/chosen": -297.69390869140625,
      "logps/rejected": -272.7558898925781,
      "loss": 0.0944,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.4865299165248871,
      "rewards/margins": 0.19219791889190674,
      "rewards/rejected": -0.6787278056144714,
      "step": 8110
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.0128727158922603e-08,
      "logits/chosen": -1.8715741634368896,
      "logits/rejected": -1.7751662731170654,
      "logps/chosen": -242.9435577392578,
      "logps/rejected": -240.58901977539062,
      "loss": 0.1389,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4711694121360779,
      "rewards/margins": 0.08951371163129807,
      "rewards/rejected": -0.5606831312179565,
      "step": 8120
    },
    {
      "epoch": 0.98,
      "learning_rate": 9.20899721353341e-09,
      "logits/chosen": -2.031369924545288,
      "logits/rejected": -1.8035614490509033,
      "logps/chosen": -258.4107971191406,
      "logps/rejected": -323.7535705566406,
      "loss": 0.1405,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.4729674458503723,
      "rewards/margins": 0.15468671917915344,
      "rewards/rejected": -0.6276541948318481,
      "step": 8130
    },
    {
      "epoch": 0.98,
      "learning_rate": 8.332958880108155e-09,
      "logits/chosen": -2.0815842151641846,
      "logits/rejected": -1.6333885192871094,
      "logps/chosen": -278.9797668457031,
      "logps/rejected": -250.55886840820312,
      "loss": 0.1568,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.3721701502799988,
      "rewards/margins": 0.18021699786186218,
      "rewards/rejected": -0.5523871183395386,
      "step": 8140
    },
    {
      "epoch": 0.98,
      "learning_rate": 7.500627525462711e-09,
      "logits/chosen": -2.008918285369873,
      "logits/rejected": -1.5942274332046509,
      "logps/chosen": -286.1038513183594,
      "logps/rejected": -229.10812377929688,
      "loss": 0.1275,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.4887969493865967,
      "rewards/margins": 0.1278802454471588,
      "rewards/rejected": -0.6166771650314331,
      "step": 8150
    },
    {
      "epoch": 0.98,
      "learning_rate": 6.712017749737343e-09,
      "logits/chosen": -2.0941195487976074,
      "logits/rejected": -1.3795114755630493,
      "logps/chosen": -307.4806823730469,
      "logps/rejected": -249.28445434570312,
      "loss": 0.1405,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.4613245129585266,
      "rewards/margins": 0.1985275000333786,
      "rewards/rejected": -0.6598520278930664,
      "step": 8160
    },
    {
      "epoch": 0.98,
      "learning_rate": 5.96714338614135e-09,
      "logits/chosen": -2.215099334716797,
      "logits/rejected": -1.6631901264190674,
      "logps/chosen": -375.0947265625,
      "logps/rejected": -313.10205078125,
      "loss": 0.1627,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.39094269275665283,
      "rewards/margins": 0.1836312711238861,
      "rewards/rejected": -0.5745739936828613,
      "step": 8170
    },
    {
      "epoch": 0.98,
      "learning_rate": 5.266017500709098e-09,
      "logits/chosen": -2.128441095352173,
      "logits/rejected": -1.8798637390136719,
      "logps/chosen": -252.12783813476562,
      "logps/rejected": -264.87506103515625,
      "loss": 0.1501,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4046745300292969,
      "rewards/margins": 0.11516742408275604,
      "rewards/rejected": -0.5198420286178589,
      "step": 8180
    },
    {
      "epoch": 0.98,
      "learning_rate": 4.608652392072144e-09,
      "logits/chosen": -2.095583438873291,
      "logits/rejected": -1.7360079288482666,
      "logps/chosen": -267.0264587402344,
      "logps/rejected": -242.8651885986328,
      "loss": 0.1365,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.4499017596244812,
      "rewards/margins": 0.15220007300376892,
      "rewards/rejected": -0.6021018028259277,
      "step": 8190
    },
    {
      "epoch": 0.98,
      "learning_rate": 3.995059591242467e-09,
      "logits/chosen": -2.0483319759368896,
      "logits/rejected": -1.7100093364715576,
      "logps/chosen": -344.25360107421875,
      "logps/rejected": -320.6583251953125,
      "loss": 0.113,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4792884886264801,
      "rewards/margins": 0.13656087219715118,
      "rewards/rejected": -0.6158494353294373,
      "step": 8200
    },
    {
      "epoch": 0.99,
      "learning_rate": 3.4252498614106843e-09,
      "logits/chosen": -1.996120810508728,
      "logits/rejected": -1.5770938396453857,
      "logps/chosen": -301.490478515625,
      "logps/rejected": -247.4650421142578,
      "loss": 0.1302,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.49081936478614807,
      "rewards/margins": 0.14099135994911194,
      "rewards/rejected": -0.6318107843399048,
      "step": 8210
    },
    {
      "epoch": 0.99,
      "learning_rate": 2.8992331977570343e-09,
      "logits/chosen": -2.1005160808563232,
      "logits/rejected": -1.9152495861053467,
      "logps/chosen": -269.256591796875,
      "logps/rejected": -259.864501953125,
      "loss": 0.1759,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.4604567885398865,
      "rewards/margins": 0.09413363039493561,
      "rewards/rejected": -0.5545904636383057,
      "step": 8220
    },
    {
      "epoch": 0.99,
      "learning_rate": 2.4170188272770736e-09,
      "logits/chosen": -2.0771050453186035,
      "logits/rejected": -1.7211967706680298,
      "logps/chosen": -322.2476501464844,
      "logps/rejected": -295.37115478515625,
      "loss": 0.1513,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.380420446395874,
      "rewards/margins": 0.18110953271389008,
      "rewards/rejected": -0.5615299940109253,
      "step": 8230
    },
    {
      "epoch": 0.99,
      "learning_rate": 1.9786152086181955e-09,
      "logits/chosen": -2.000828266143799,
      "logits/rejected": -1.55103600025177,
      "logps/chosen": -267.5838928222656,
      "logps/rejected": -248.3422393798828,
      "loss": 0.1141,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.4607185423374176,
      "rewards/margins": 0.19947698712348938,
      "rewards/rejected": -0.660195529460907,
      "step": 8240
    },
    {
      "epoch": 0.99,
      "learning_rate": 1.5840300319316937e-09,
      "logits/chosen": -1.786924123764038,
      "logits/rejected": -1.3054568767547607,
      "logps/chosen": -270.92205810546875,
      "logps/rejected": -260.0880126953125,
      "loss": 0.1442,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.541989266872406,
      "rewards/margins": 0.18230721354484558,
      "rewards/rejected": -0.7242964506149292,
      "step": 8250
    },
    {
      "epoch": 0.99,
      "learning_rate": 1.23327021873898e-09,
      "logits/chosen": -2.0959393978118896,
      "logits/rejected": -1.521923303604126,
      "logps/chosen": -264.62353515625,
      "logps/rejected": -225.8861846923828,
      "loss": 0.1201,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.37499696016311646,
      "rewards/margins": 0.20748789608478546,
      "rewards/rejected": -0.5824848413467407,
      "step": 8260
    },
    {
      "epoch": 0.99,
      "learning_rate": 9.263419218089042e-10,
      "logits/chosen": -1.8493993282318115,
      "logits/rejected": -1.476485013961792,
      "logps/chosen": -253.7661895751953,
      "logps/rejected": -263.26422119140625,
      "loss": 0.1452,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.4299536645412445,
      "rewards/margins": 0.20787473022937775,
      "rewards/rejected": -0.6378284692764282,
      "step": 8270
    },
    {
      "epoch": 0.99,
      "learning_rate": 6.632505250506183e-10,
      "logits/chosen": -2.12581205368042,
      "logits/rejected": -1.7464358806610107,
      "logps/chosen": -305.6679382324219,
      "logps/rejected": -250.2874298095703,
      "loss": 0.1467,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.44572052359580994,
      "rewards/margins": 0.15005187690258026,
      "rewards/rejected": -0.595772385597229,
      "step": 8280
    },
    {
      "epoch": 0.99,
      "learning_rate": 4.440006434183741e-10,
      "logits/chosen": -1.8823049068450928,
      "logits/rejected": -1.5557067394256592,
      "logps/chosen": -312.54608154296875,
      "logps/rejected": -316.3292541503906,
      "loss": 0.1481,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.5499744415283203,
      "rewards/margins": 0.11392368376255035,
      "rewards/rejected": -0.663898229598999,
      "step": 8290
    },
    {
      "epoch": 1.0,
      "learning_rate": 2.6859612283186567e-10,
      "logits/chosen": -2.018319845199585,
      "logits/rejected": -1.6919094324111938,
      "logps/chosen": -225.6822052001953,
      "logps/rejected": -243.1230926513672,
      "loss": 0.1243,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.4279976785182953,
      "rewards/margins": 0.1937953531742096,
      "rewards/rejected": -0.6217929720878601,
      "step": 8300
    },
    {
      "epoch": 1.0,
      "learning_rate": 1.370400401065619e-10,
      "logits/chosen": -1.9666255712509155,
      "logits/rejected": -1.7945621013641357,
      "logps/chosen": -199.1581573486328,
      "logps/rejected": -203.54124450683594,
      "loss": 0.1544,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5227402448654175,
      "rewards/margins": 0.12387014925479889,
      "rewards/rejected": -0.6466103792190552,
      "step": 8310
    },
    {
      "epoch": 1.0,
      "learning_rate": 4.933470290263698e-11,
      "logits/chosen": -2.083996295928955,
      "logits/rejected": -1.7333097457885742,
      "logps/chosen": -277.2462463378906,
      "logps/rejected": -264.87225341796875,
      "loss": 0.155,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.4118651747703552,
      "rewards/margins": 0.1753673553466797,
      "rewards/rejected": -0.5872325301170349,
      "step": 8320
    },
    {
      "epoch": 1.0,
      "learning_rate": 5.481649681671197e-12,
      "logits/chosen": -1.8581920862197876,
      "logits/rejected": -1.672580361366272,
      "logps/chosen": -255.0128173828125,
      "logps/rejected": -287.12408447265625,
      "loss": 0.1253,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.5130528211593628,
      "rewards/margins": 0.1296093761920929,
      "rewards/rejected": -0.6426622867584229,
      "step": 8330
    },
    {
      "epoch": 1.0,
      "step": 8335,
      "total_flos": 0.0,
      "train_loss": 0.14340321629899808,
      "train_runtime": 34860.3009,
      "train_samples_per_second": 0.956,
      "train_steps_per_second": 0.239
    }
  ],
  "logging_steps": 10,
  "max_steps": 8335,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}