{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 1875,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 2.6595744680851065e-08,
      "logits/chosen": 0.09963524341583252,
      "logits/rejected": 0.17411324381828308,
      "logps/chosen": -378.2606506347656,
      "logps/rejected": -386.1599426269531,
      "loss": 0.21,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "learning_rate": 2.6595744680851066e-07,
      "logits/chosen": 0.1553082913160324,
      "logits/rejected": 0.24615353345870972,
      "logps/chosen": -397.8062438964844,
      "logps/rejected": -394.25341796875,
      "loss": 0.2045,
      "rewards/accuracies": 0.2777777910232544,
      "rewards/chosen": -0.0005843756953254342,
      "rewards/margins": -3.1117440812522545e-05,
      "rewards/rejected": -0.0005532581708393991,
      "step": 10
    },
    {
      "epoch": 0.01,
      "learning_rate": 5.319148936170213e-07,
      "logits/chosen": 0.3212467432022095,
      "logits/rejected": 0.24689868092536926,
      "logps/chosen": -409.90167236328125,
      "logps/rejected": -409.5082092285156,
      "loss": 0.2087,
      "rewards/accuracies": 0.3687500059604645,
      "rewards/chosen": -0.0009676044574007392,
      "rewards/margins": 5.4393574828282e-05,
      "rewards/rejected": -0.0010219980031251907,
      "step": 20
    },
    {
      "epoch": 0.02,
      "learning_rate": 7.97872340425532e-07,
      "logits/chosen": 0.22952334582805634,
      "logits/rejected": 0.2672143876552582,
      "logps/chosen": -385.3333740234375,
      "logps/rejected": -392.8493957519531,
      "loss": 0.2137,
      "rewards/accuracies": 0.28125,
      "rewards/chosen": -0.0009394015069119632,
      "rewards/margins": -8.656540012452751e-05,
      "rewards/rejected": -0.0008528361213393509,
      "step": 30
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.0638297872340427e-06,
      "logits/chosen": 0.24455244839191437,
      "logits/rejected": 0.14430980384349823,
      "logps/chosen": -414.5653381347656,
      "logps/rejected": -410.3203125,
      "loss": 0.2186,
      "rewards/accuracies": 0.35624998807907104,
      "rewards/chosen": -0.0006429333006963134,
      "rewards/margins": 7.077780173858628e-05,
      "rewards/rejected": -0.0007137111388146877,
      "step": 40
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.3297872340425533e-06,
      "logits/chosen": 0.16719885170459747,
      "logits/rejected": 0.19559958577156067,
      "logps/chosen": -388.90740966796875,
      "logps/rejected": -379.8285827636719,
      "loss": 0.2026,
      "rewards/accuracies": 0.35624998807907104,
      "rewards/chosen": -3.721771645359695e-05,
      "rewards/margins": -9.795947335078381e-06,
      "rewards/rejected": -2.742178003245499e-05,
      "step": 50
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.595744680851064e-06,
      "logits/chosen": 0.2627559304237366,
      "logits/rejected": 0.2265670746564865,
      "logps/chosen": -355.0706787109375,
      "logps/rejected": -372.27508544921875,
      "loss": 0.2185,
      "rewards/accuracies": 0.39375001192092896,
      "rewards/chosen": 0.00125337322242558,
      "rewards/margins": 0.00012534942652564496,
      "rewards/rejected": 0.0011280238395556808,
      "step": 60
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.8617021276595745e-06,
      "logits/chosen": 0.15168386697769165,
      "logits/rejected": 0.3290417194366455,
      "logps/chosen": -432.84197998046875,
      "logps/rejected": -418.5240783691406,
      "loss": 0.2123,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": 0.0038177906535565853,
      "rewards/margins": 0.00015494157560169697,
      "rewards/rejected": 0.003662849310785532,
      "step": 70
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.1276595744680853e-06,
      "logits/chosen": 0.150368332862854,
      "logits/rejected": 0.2564127445220947,
      "logps/chosen": -402.29449462890625,
      "logps/rejected": -372.3401184082031,
      "loss": 0.2282,
      "rewards/accuracies": 0.4312500059604645,
      "rewards/chosen": 0.006463122554123402,
      "rewards/margins": 0.00028264717548154294,
      "rewards/rejected": 0.00618047546595335,
      "step": 80
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.393617021276596e-06,
      "logits/chosen": 0.19279547035694122,
      "logits/rejected": 0.3653843104839325,
      "logps/chosen": -433.8819885253906,
      "logps/rejected": -433.3250427246094,
      "loss": 0.2121,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": 0.008460921235382557,
      "rewards/margins": 0.0003720732347574085,
      "rewards/rejected": 0.008088847622275352,
      "step": 90
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.6595744680851065e-06,
      "logits/chosen": 0.187942236661911,
      "logits/rejected": 0.30413854122161865,
      "logps/chosen": -415.242431640625,
      "logps/rejected": -411.3409118652344,
      "loss": 0.2096,
      "rewards/accuracies": 0.4312500059604645,
      "rewards/chosen": 0.006011855788528919,
      "rewards/margins": 0.0004307249910198152,
      "rewards/rejected": 0.005581130273640156,
      "step": 100
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.9255319148936174e-06,
      "logits/chosen": 0.18410541117191315,
      "logits/rejected": 0.22288016974925995,
      "logps/chosen": -402.7239685058594,
      "logps/rejected": -401.08685302734375,
      "loss": 0.2072,
      "rewards/accuracies": 0.4312500059604645,
      "rewards/chosen": 0.002768724225461483,
      "rewards/margins": 0.00032299422309733927,
      "rewards/rejected": 0.0024457299150526524,
      "step": 110
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.191489361702128e-06,
      "logits/chosen": 0.23630130290985107,
      "logits/rejected": 0.2844993472099304,
      "logps/chosen": -436.46624755859375,
      "logps/rejected": -418.0640563964844,
      "loss": 0.2094,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": 0.0015008236514404416,
      "rewards/margins": 0.000703630386851728,
      "rewards/rejected": 0.0007971933227963746,
      "step": 120
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.457446808510639e-06,
      "logits/chosen": 0.1779193878173828,
      "logits/rejected": 0.2234630137681961,
      "logps/chosen": -445.2749938964844,
      "logps/rejected": -395.0121154785156,
      "loss": 0.218,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": 0.0037746974267065525,
      "rewards/margins": 0.000721289194189012,
      "rewards/rejected": 0.0030534083489328623,
      "step": 130
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.723404255319149e-06,
      "logits/chosen": 0.21873219311237335,
      "logits/rejected": 0.28560131788253784,
      "logps/chosen": -386.23992919921875,
      "logps/rejected": -392.177978515625,
      "loss": 0.2085,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.005661582108587027,
      "rewards/margins": 0.001634161570109427,
      "rewards/rejected": 0.004027421120554209,
      "step": 140
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.98936170212766e-06,
      "logits/chosen": 0.14144310355186462,
      "logits/rejected": 0.21909043192863464,
      "logps/chosen": -410.2815856933594,
      "logps/rejected": -409.62420654296875,
      "loss": 0.2042,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.002802160568535328,
      "rewards/margins": 0.002337489277124405,
      "rewards/rejected": 0.0004646713496185839,
      "step": 150
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.255319148936171e-06,
      "logits/chosen": 0.15275654196739197,
      "logits/rejected": 0.18240560591220856,
      "logps/chosen": -368.1477966308594,
      "logps/rejected": -379.97052001953125,
      "loss": 0.2084,
      "rewards/accuracies": 0.4312500059604645,
      "rewards/chosen": -0.0024238377809524536,
      "rewards/margins": 0.0018925836775451899,
      "rewards/rejected": -0.0043164207600057125,
      "step": 160
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.521276595744681e-06,
      "logits/chosen": 0.16948816180229187,
      "logits/rejected": 0.27625542879104614,
      "logps/chosen": -439.19580078125,
      "logps/rejected": -423.7747497558594,
      "loss": 0.1964,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.010004468262195587,
      "rewards/margins": 0.0029372868593782187,
      "rewards/rejected": -0.01294175535440445,
      "step": 170
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.787234042553192e-06,
      "logits/chosen": 0.14220403134822845,
      "logits/rejected": 0.17016319930553436,
      "logps/chosen": -402.444091796875,
      "logps/rejected": -399.7784729003906,
      "loss": 0.2215,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.017413243651390076,
      "rewards/margins": 0.005264014936983585,
      "rewards/rejected": -0.022677261382341385,
      "step": 180
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999982660399688e-06,
      "logits/chosen": 0.12951701879501343,
      "logits/rejected": 0.24623069167137146,
      "logps/chosen": -468.02325439453125,
      "logps/rejected": -468.9815979003906,
      "loss": 0.1973,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.03261370584368706,
      "rewards/margins": 0.006805351935327053,
      "rewards/rejected": -0.039419058710336685,
      "step": 190
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.99937579964398e-06,
      "logits/chosen": 0.052798859775066376,
      "logits/rejected": 0.23226705193519592,
      "logps/chosen": -469.90753173828125,
      "logps/rejected": -448.4300842285156,
      "loss": 0.1876,
      "rewards/accuracies": 0.4437499940395355,
      "rewards/chosen": -0.05150008201599121,
      "rewards/margins": 0.006041003856807947,
      "rewards/rejected": -0.05754108354449272,
      "step": 200
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.9979021993870645e-06,
      "logits/chosen": 0.08435344696044922,
      "logits/rejected": 0.16326232254505157,
      "logps/chosen": -446.05584716796875,
      "logps/rejected": -458.54217529296875,
      "loss": 0.2059,
      "rewards/accuracies": 0.41874998807907104,
      "rewards/chosen": -0.06337223947048187,
      "rewards/margins": 0.008098254911601543,
      "rewards/rejected": -0.07147050648927689,
      "step": 210
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.995562370647553e-06,
      "logits/chosen": 0.14515891671180725,
      "logits/rejected": 0.1300167292356491,
      "logps/chosen": -490.72271728515625,
      "logps/rejected": -501.8885803222656,
      "loss": 0.2165,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.06726966798305511,
      "rewards/margins": 0.010882614180445671,
      "rewards/rejected": -0.07815226912498474,
      "step": 220
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.992357124836838e-06,
      "logits/chosen": 0.14549218118190765,
      "logits/rejected": 0.11253325641155243,
      "logps/chosen": -450.588623046875,
      "logps/rejected": -453.873291015625,
      "loss": 0.2008,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.0649234801530838,
      "rewards/margins": 0.012732243165373802,
      "rewards/rejected": -0.07765571773052216,
      "step": 230
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.9882875734777044e-06,
      "logits/chosen": 0.17049750685691833,
      "logits/rejected": 0.08599407970905304,
      "logps/chosen": -452.00152587890625,
      "logps/rejected": -478.6534118652344,
      "loss": 0.2,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.07021967321634293,
      "rewards/margins": 0.01245578657835722,
      "rewards/rejected": -0.08267545700073242,
      "step": 240
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.983355127818882e-06,
      "logits/chosen": 0.08762475103139877,
      "logits/rejected": 0.15931396186351776,
      "logps/chosen": -472.1898498535156,
      "logps/rejected": -486.1856994628906,
      "loss": 0.1904,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.08154710382223129,
      "rewards/margins": 0.02140663005411625,
      "rewards/rejected": -0.10295373201370239,
      "step": 250
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.977561498345639e-06,
      "logits/chosen": 0.052127450704574585,
      "logits/rejected": 0.011202364228665829,
      "logps/chosen": -450.86712646484375,
      "logps/rejected": -529.0736083984375,
      "loss": 0.1943,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0922573059797287,
      "rewards/margins": 0.03915147855877876,
      "rewards/rejected": -0.13140878081321716,
      "step": 260
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.970908694186624e-06,
      "logits/chosen": 0.02654929831624031,
      "logits/rejected": 0.14042030274868011,
      "logps/chosen": -490.01922607421875,
      "logps/rejected": -497.9236755371094,
      "loss": 0.2082,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.10826190561056137,
      "rewards/margins": 0.021642591804265976,
      "rewards/rejected": -0.12990449368953705,
      "step": 270
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.9633990224171305e-06,
      "logits/chosen": 0.09107808768749237,
      "logits/rejected": 0.08074366301298141,
      "logps/chosen": -493.1717834472656,
      "logps/rejected": -526.7948608398438,
      "loss": 0.1922,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.09491624683141708,
      "rewards/margins": 0.02360902726650238,
      "rewards/rejected": -0.11852526664733887,
      "step": 280
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.955035087259046e-06,
      "logits/chosen": -0.01798112317919731,
      "logits/rejected": 0.14514566957950592,
      "logps/chosen": -473.822021484375,
      "logps/rejected": -492.75079345703125,
      "loss": 0.1917,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.0761452168226242,
      "rewards/margins": 0.02504543401300907,
      "rewards/rejected": -0.10119066387414932,
      "step": 290
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.945819789177756e-06,
      "logits/chosen": 0.012573105283081532,
      "logits/rejected": 0.08077313005924225,
      "logps/chosen": -461.69134521484375,
      "logps/rejected": -452.34393310546875,
      "loss": 0.2008,
      "rewards/accuracies": 0.4312500059604645,
      "rewards/chosen": -0.07385671138763428,
      "rewards/margins": 0.022219108417630196,
      "rewards/rejected": -0.09607584029436111,
      "step": 300
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.935756323876306e-06,
      "logits/chosen": 0.0107968058437109,
      "logits/rejected": 0.017728794366121292,
      "logps/chosen": -512.6489868164062,
      "logps/rejected": -566.363525390625,
      "loss": 0.1829,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.09047379344701767,
      "rewards/margins": 0.04145172983407974,
      "rewards/rejected": -0.1319255381822586,
      "step": 310
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.924848181187199e-06,
      "logits/chosen": -0.0678163543343544,
      "logits/rejected": 0.10408651828765869,
      "logps/chosen": -535.5982055664062,
      "logps/rejected": -556.8341064453125,
      "loss": 0.1803,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.1051037460565567,
      "rewards/margins": 0.029761528596282005,
      "rewards/rejected": -0.13486526906490326,
      "step": 320
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.913099143862173e-06,
      "logits/chosen": -0.04091095179319382,
      "logits/rejected": -0.0030039071571081877,
      "logps/chosen": -553.9273681640625,
      "logps/rejected": -610.8055419921875,
      "loss": 0.1908,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.12315535545349121,
      "rewards/margins": 0.04520713910460472,
      "rewards/rejected": -0.16836249828338623,
      "step": 330
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.900513286260416e-06,
      "logits/chosen": -0.1246240958571434,
      "logits/rejected": -0.03937938064336777,
      "logps/chosen": -490.50030517578125,
      "logps/rejected": -532.2816162109375,
      "loss": 0.1887,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -0.11805293709039688,
      "rewards/margins": 0.02471679449081421,
      "rewards/rejected": -0.1427697390317917,
      "step": 340
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.887094972935645e-06,
      "logits/chosen": -0.08421911299228668,
      "logits/rejected": 0.008182203397154808,
      "logps/chosen": -535.670654296875,
      "logps/rejected": -536.1793212890625,
      "loss": 0.1949,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.10835033655166626,
      "rewards/margins": 0.03422056883573532,
      "rewards/rejected": -0.1425708830356598,
      "step": 350
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.87284885712256e-06,
      "logits/chosen": -0.07014905661344528,
      "logits/rejected": -0.006586342118680477,
      "logps/chosen": -494.0985412597656,
      "logps/rejected": -525.9044189453125,
      "loss": 0.2065,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.10610511153936386,
      "rewards/margins": 0.0382779985666275,
      "rewards/rejected": -0.14438311755657196,
      "step": 360
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.857779879123181e-06,
      "logits/chosen": -0.0900832936167717,
      "logits/rejected": -0.10245800018310547,
      "logps/chosen": -572.3137817382812,
      "logps/rejected": -617.9874267578125,
      "loss": 0.2156,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.12338846921920776,
      "rewards/margins": 0.036996591836214066,
      "rewards/rejected": -0.16038507223129272,
      "step": 370
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.841893264593643e-06,
      "logits/chosen": -0.051192570477724075,
      "logits/rejected": -0.02001297101378441,
      "logps/chosen": -547.9758911132812,
      "logps/rejected": -576.4624633789062,
      "loss": 0.1939,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.10065815597772598,
      "rewards/margins": 0.02802097424864769,
      "rewards/rejected": -0.12867912650108337,
      "step": 380
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.825194522732023e-06,
      "logits/chosen": -0.14838102459907532,
      "logits/rejected": 0.1484626680612564,
      "logps/chosen": -494.1825256347656,
      "logps/rejected": -524.7642822265625,
      "loss": 0.1967,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.08151207864284515,
      "rewards/margins": 0.032111816108226776,
      "rewards/rejected": -0.11362389475107193,
      "step": 390
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.807689444367853e-06,
      "logits/chosen": -0.046164363622665405,
      "logits/rejected": -0.03798365592956543,
      "logps/chosen": -472.62677001953125,
      "logps/rejected": -500.4376525878906,
      "loss": 0.1883,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.07655288279056549,
      "rewards/margins": 0.039138298481702805,
      "rewards/rejected": -0.1156911849975586,
      "step": 400
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.78938409995396e-06,
      "logits/chosen": -0.1458251178264618,
      "logits/rejected": -0.0393032506108284,
      "logps/chosen": -436.66033935546875,
      "logps/rejected": -447.3544006347656,
      "loss": 0.2138,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.08541758358478546,
      "rewards/margins": 0.011176596395671368,
      "rewards/rejected": -0.0965941846370697,
      "step": 410
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.770284837461342e-06,
      "logits/chosen": -0.15639075636863708,
      "logits/rejected": -0.010832506231963634,
      "logps/chosen": -489.99151611328125,
      "logps/rejected": -543.11767578125,
      "loss": 0.1843,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.09636662900447845,
      "rewards/margins": 0.04293360933661461,
      "rewards/rejected": -0.13930022716522217,
      "step": 420
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.7503982801778015e-06,
      "logits/chosen": -0.22076280415058136,
      "logits/rejected": -0.018700700253248215,
      "logps/chosen": -517.91455078125,
      "logps/rejected": -551.0287475585938,
      "loss": 0.1901,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -0.11141178756952286,
      "rewards/margins": 0.033164333552122116,
      "rewards/rejected": -0.14457611739635468,
      "step": 430
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.729731324411104e-06,
      "logits/chosen": -0.15695324540138245,
      "logits/rejected": -0.08840291202068329,
      "logps/chosen": -491.2373046875,
      "logps/rejected": -511.56781005859375,
      "loss": 0.1891,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.10444734245538712,
      "rewards/margins": 0.04566046595573425,
      "rewards/rejected": -0.15010783076286316,
      "step": 440
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.7082911370974645e-06,
      "logits/chosen": -0.11595731973648071,
      "logits/rejected": -0.128843754529953,
      "logps/chosen": -557.0236206054688,
      "logps/rejected": -633.2085571289062,
      "loss": 0.2007,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.12254992872476578,
      "rewards/margins": 0.05843643471598625,
      "rewards/rejected": -0.18098635971546173,
      "step": 450
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.68608515331618e-06,
      "logits/chosen": -0.08469126373529434,
      "logits/rejected": -0.03279350325465202,
      "logps/chosen": -486.62078857421875,
      "logps/rejected": -515.3419799804688,
      "loss": 0.1855,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.0999387800693512,
      "rewards/margins": 0.04227910935878754,
      "rewards/rejected": -0.14221788942813873,
      "step": 460
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.663121073711269e-06,
      "logits/chosen": -0.1471957415342331,
      "logits/rejected": -0.05932598561048508,
      "logps/chosen": -474.94915771484375,
      "logps/rejected": -536.3035888671875,
      "loss": 0.1819,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.0976475328207016,
      "rewards/margins": 0.04680894687771797,
      "rewards/rejected": -0.14445647597312927,
      "step": 470
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.63940686182103e-06,
      "logits/chosen": -0.17284898459911346,
      "logits/rejected": -0.1464369297027588,
      "logps/chosen": -502.36578369140625,
      "logps/rejected": -583.7872314453125,
      "loss": 0.1724,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.10467234998941422,
      "rewards/margins": 0.052542489022016525,
      "rewards/rejected": -0.15721485018730164,
      "step": 480
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.614950741316425e-06,
      "logits/chosen": -0.21725276112556458,
      "logits/rejected": -0.14681020379066467,
      "logps/chosen": -523.1455078125,
      "logps/rejected": -577.9979248046875,
      "loss": 0.184,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.09476028382778168,
      "rewards/margins": 0.056237511336803436,
      "rewards/rejected": -0.15099778771400452,
      "step": 490
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.589761193149254e-06,
      "logits/chosen": -0.2188795506954193,
      "logits/rejected": -0.24383120238780975,
      "logps/chosen": -541.2039794921875,
      "logps/rejected": -610.45166015625,
      "loss": 0.1864,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.11351511627435684,
      "rewards/margins": 0.044019803404808044,
      "rewards/rejected": -0.1575349122285843,
      "step": 500
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.563846952611112e-06,
      "logits/chosen": -0.2326229363679886,
      "logits/rejected": -0.15911969542503357,
      "logps/chosen": -494.03302001953125,
      "logps/rejected": -544.4476318359375,
      "loss": 0.1756,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.09819264709949493,
      "rewards/margins": 0.056823331862688065,
      "rewards/rejected": -0.1550159752368927,
      "step": 510
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.537217006304141e-06,
      "logits/chosen": -0.16317172348499298,
      "logits/rejected": -0.1998225897550583,
      "logps/chosen": -530.8056640625,
      "logps/rejected": -605.7437133789062,
      "loss": 0.1588,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.09498073905706406,
      "rewards/margins": 0.07860372215509415,
      "rewards/rejected": -0.1735844761133194,
      "step": 520
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.50988058902464e-06,
      "logits/chosen": -0.17353829741477966,
      "logits/rejected": -0.20073547959327698,
      "logps/chosen": -539.7728271484375,
      "logps/rejected": -576.4498901367188,
      "loss": 0.1832,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.10605832189321518,
      "rewards/margins": 0.05749889463186264,
      "rewards/rejected": -0.16355721652507782,
      "step": 530
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.481847180560593e-06,
      "logits/chosen": -0.17195938527584076,
      "logits/rejected": -0.15436215698719025,
      "logps/chosen": -502.6675720214844,
      "logps/rejected": -537.3475341796875,
      "loss": 0.2008,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -0.10046552121639252,
      "rewards/margins": 0.028822993859648705,
      "rewards/rejected": -0.12928852438926697,
      "step": 540
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.453126502404253e-06,
      "logits/chosen": -0.21576860547065735,
      "logits/rejected": -0.22100479900836945,
      "logps/chosen": -528.4920654296875,
      "logps/rejected": -600.3697509765625,
      "loss": 0.1657,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.11002079397439957,
      "rewards/margins": 0.06445158272981644,
      "rewards/rejected": -0.1744723618030548,
      "step": 550
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.423728514380892e-06,
      "logits/chosen": -0.2724868655204773,
      "logits/rejected": -0.18154220283031464,
      "logps/chosen": -534.0877075195312,
      "logps/rejected": -585.4654541015625,
      "loss": 0.1745,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.11277395486831665,
      "rewards/margins": 0.05171126872301102,
      "rewards/rejected": -0.16448521614074707,
      "step": 560
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.393663411194918e-06,
      "logits/chosen": -0.29137545824050903,
      "logits/rejected": -0.1960664540529251,
      "logps/chosen": -543.6236572265625,
      "logps/rejected": -613.0497436523438,
      "loss": 0.1896,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.11756434291601181,
      "rewards/margins": 0.049765445291996,
      "rewards/rejected": -0.16732977330684662,
      "step": 570
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.362941618894523e-06,
      "logits/chosen": -0.24844829738140106,
      "logits/rejected": -0.17800265550613403,
      "logps/chosen": -483.45697021484375,
      "logps/rejected": -549.9342041015625,
      "loss": 0.1642,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.10112373530864716,
      "rewards/margins": 0.06612999737262726,
      "rewards/rejected": -0.16725373268127441,
      "step": 580
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.331573791256116e-06,
      "logits/chosen": -0.35919591784477234,
      "logits/rejected": -0.2761411964893341,
      "logps/chosen": -489.17852783203125,
      "logps/rejected": -502.04022216796875,
      "loss": 0.1868,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -0.09544913470745087,
      "rewards/margins": 0.05491846054792404,
      "rewards/rejected": -0.1503675878047943,
      "step": 590
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.299570806089786e-06,
      "logits/chosen": -0.254047155380249,
      "logits/rejected": -0.24359670281410217,
      "logps/chosen": -517.5626220703125,
      "logps/rejected": -570.8585205078125,
      "loss": 0.1936,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.10336007177829742,
      "rewards/margins": 0.06461663544178009,
      "rewards/rejected": -0.16797669231891632,
      "step": 600
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.266943761467057e-06,
      "logits/chosen": -0.3495100736618042,
      "logits/rejected": -0.22493663430213928,
      "logps/chosen": -526.2481079101562,
      "logps/rejected": -611.931396484375,
      "loss": 0.1684,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.10717520862817764,
      "rewards/margins": 0.07210040837526321,
      "rewards/rejected": -0.17927563190460205,
      "step": 610
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.233703971872287e-06,
      "logits/chosen": -0.28394874930381775,
      "logits/rejected": -0.2587239146232605,
      "logps/chosen": -567.471435546875,
      "logps/rejected": -639.43310546875,
      "loss": 0.1944,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.11128894239664078,
      "rewards/margins": 0.06409231573343277,
      "rewards/rejected": -0.17538125813007355,
      "step": 620
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.1998629642789925e-06,
      "logits/chosen": -0.28350967168807983,
      "logits/rejected": -0.277797669172287,
      "logps/chosen": -476.7294921875,
      "logps/rejected": -528.3859252929688,
      "loss": 0.1746,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.07711435854434967,
      "rewards/margins": 0.0645236223936081,
      "rewards/rejected": -0.14163798093795776,
      "step": 630
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.165432474152505e-06,
      "logits/chosen": -0.3298090100288391,
      "logits/rejected": -0.27090325951576233,
      "logps/chosen": -543.4595947265625,
      "logps/rejected": -586.7649536132812,
      "loss": 0.1706,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.08748660981655121,
      "rewards/margins": 0.06710480898618698,
      "rewards/rejected": -0.1545914113521576,
      "step": 640
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.130424441380308e-06,
      "logits/chosen": -0.3356090188026428,
      "logits/rejected": -0.14076504111289978,
      "logps/chosen": -532.180908203125,
      "logps/rejected": -585.5838623046875,
      "loss": 0.1865,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.09366744011640549,
      "rewards/margins": 0.046517182141542435,
      "rewards/rejected": -0.14018462598323822,
      "step": 650
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.09485100613151e-06,
      "logits/chosen": -0.2658724784851074,
      "logits/rejected": -0.2869165539741516,
      "logps/chosen": -479.1338806152344,
      "logps/rejected": -522.8956298828125,
      "loss": 0.1859,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.07211341708898544,
      "rewards/margins": 0.050303440541028976,
      "rewards/rejected": -0.12241685390472412,
      "step": 660
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.058724504646834e-06,
      "logits/chosen": -0.3326638340950012,
      "logits/rejected": -0.3534831404685974,
      "logps/chosen": -465.98046875,
      "logps/rejected": -551.1593017578125,
      "loss": 0.1817,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.06464243680238724,
      "rewards/margins": 0.06546784937381744,
      "rewards/rejected": -0.13011027872562408,
      "step": 670
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.022057464960632e-06,
      "logits/chosen": -0.2847468852996826,
      "logits/rejected": -0.3084865212440491,
      "logps/chosen": -466.17431640625,
      "logps/rejected": -591.4989624023438,
      "loss": 0.169,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.06827010214328766,
      "rewards/margins": 0.08749465644359589,
      "rewards/rejected": -0.15576477348804474,
      "step": 680
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.984862602556383e-06,
      "logits/chosen": -0.29794925451278687,
      "logits/rejected": -0.2838256359100342,
      "logps/chosen": -505.8924255371094,
      "logps/rejected": -569.5626831054688,
      "loss": 0.173,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.08569585531949997,
      "rewards/margins": 0.058602042496204376,
      "rewards/rejected": -0.14429791271686554,
      "step": 690
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.947152815957187e-06,
      "logits/chosen": -0.4137333333492279,
      "logits/rejected": -0.201189324259758,
      "logps/chosen": -520.5211181640625,
      "logps/rejected": -567.9163208007812,
      "loss": 0.1771,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.07455627620220184,
      "rewards/margins": 0.06876102834939957,
      "rewards/rejected": -0.1433172971010208,
      "step": 700
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.908941182252785e-06,
      "logits/chosen": -0.3251053988933563,
      "logits/rejected": -0.21724073588848114,
      "logps/chosen": -457.90692138671875,
      "logps/rejected": -512.7808837890625,
      "loss": 0.1777,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.0669245719909668,
      "rewards/margins": 0.05744165927171707,
      "rewards/rejected": -0.12436623871326447,
      "step": 710
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.8702409525646535e-06,
      "logits/chosen": -0.20694856345653534,
      "logits/rejected": -0.2958626449108124,
      "logps/chosen": -455.8350524902344,
      "logps/rejected": -536.2254638671875,
      "loss": 0.1935,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0645083338022232,
      "rewards/margins": 0.06103675439953804,
      "rewards/rejected": -0.12554509937763214,
      "step": 720
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.8310655474507495e-06,
      "logits/chosen": -0.3800198435783386,
      "logits/rejected": -0.2641103267669678,
      "logps/chosen": -480.8030700683594,
      "logps/rejected": -521.5682373046875,
      "loss": 0.2025,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.07129710167646408,
      "rewards/margins": 0.04021110758185387,
      "rewards/rejected": -0.11150821298360825,
      "step": 730
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.7914285522515002e-06,
      "logits/chosen": -0.35266560316085815,
      "logits/rejected": -0.21721109747886658,
      "logps/chosen": -503.35357666015625,
      "logps/rejected": -518.2500610351562,
      "loss": 0.1928,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.07125665247440338,
      "rewards/margins": 0.034123968333005905,
      "rewards/rejected": -0.10538061708211899,
      "step": 740
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.751343712378639e-06,
      "logits/chosen": -0.303008496761322,
      "logits/rejected": -0.3047345280647278,
      "logps/chosen": -447.089599609375,
      "logps/rejected": -469.61590576171875,
      "loss": 0.1992,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.06738699972629547,
      "rewards/margins": 0.027302343398332596,
      "rewards/rejected": -0.09468933939933777,
      "step": 750
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.710824928548546e-06,
      "logits/chosen": -0.29965728521347046,
      "logits/rejected": -0.2571870982646942,
      "logps/chosen": -450.4271545410156,
      "logps/rejected": -482.21502685546875,
      "loss": 0.1935,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.055190593004226685,
      "rewards/margins": 0.04364421218633652,
      "rewards/rejected": -0.0988348051905632,
      "step": 760
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6698862519617225e-06,
      "logits/chosen": -0.290757954120636,
      "logits/rejected": -0.3262009024620056,
      "logps/chosen": -449.7730407714844,
      "logps/rejected": -520.2115478515625,
      "loss": 0.1599,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.05544726923108101,
      "rewards/margins": 0.06180447340011597,
      "rewards/rejected": -0.11725173890590668,
      "step": 770
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.6285418794300793e-06,
      "logits/chosen": -0.3281220495700836,
      "logits/rejected": -0.2547357678413391,
      "logps/chosen": -441.2623596191406,
      "logps/rejected": -495.96368408203125,
      "loss": 0.1915,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.0586274154484272,
      "rewards/margins": 0.03878774121403694,
      "rewards/rejected": -0.09741516411304474,
      "step": 780
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.5868061484537365e-06,
      "logits/chosen": -0.41046142578125,
      "logits/rejected": -0.29820680618286133,
      "logps/chosen": -447.24151611328125,
      "logps/rejected": -473.2769470214844,
      "loss": 0.1898,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.07239285856485367,
      "rewards/margins": 0.03644294664263725,
      "rewards/rejected": -0.10883580148220062,
      "step": 790
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5446935322490285e-06,
      "logits/chosen": -0.35609519481658936,
      "logits/rejected": -0.2902910113334656,
      "logps/chosen": -469.17706298828125,
      "logps/rejected": -546.7158203125,
      "loss": 0.1788,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.05905740335583687,
      "rewards/margins": 0.06316393613815308,
      "rewards/rejected": -0.12222135066986084,
      "step": 800
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.502218634729447e-06,
      "logits/chosen": -0.2686077356338501,
      "logits/rejected": -0.2707546055316925,
      "logps/chosen": -522.1178588867188,
      "logps/rejected": -570.613525390625,
      "loss": 0.1825,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07459238916635513,
      "rewards/margins": 0.061357058584690094,
      "rewards/rejected": -0.13594944775104523,
      "step": 810
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.459396185441265e-06,
      "logits/chosen": -0.38130688667297363,
      "logits/rejected": -0.14519771933555603,
      "logps/chosen": -474.2879943847656,
      "logps/rejected": -492.1015625,
      "loss": 0.1804,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.049582771956920624,
      "rewards/margins": 0.04935313016176224,
      "rewards/rejected": -0.09893589466810226,
      "step": 820
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4162410344555834e-06,
      "logits/chosen": -0.2872348427772522,
      "logits/rejected": -0.2791721224784851,
      "logps/chosen": -485.4851989746094,
      "logps/rejected": -571.0114135742188,
      "loss": 0.1743,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.060776274651288986,
      "rewards/margins": 0.06890632957220078,
      "rewards/rejected": -0.12968261539936066,
      "step": 830
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3727681472185937e-06,
      "logits/chosen": -0.3343026041984558,
      "logits/rejected": -0.2967159152030945,
      "logps/chosen": -441.74713134765625,
      "logps/rejected": -455.7685546875,
      "loss": 0.1898,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.05669083446264267,
      "rewards/margins": 0.04281865060329437,
      "rewards/rejected": -0.09950948506593704,
      "step": 840
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3289925993618217e-06,
      "logits/chosen": -0.31694555282592773,
      "logits/rejected": -0.23448070883750916,
      "logps/chosen": -505.189208984375,
      "logps/rejected": -574.4694213867188,
      "loss": 0.1641,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.06701242923736572,
      "rewards/margins": 0.07507143914699554,
      "rewards/rejected": -0.14208386838436127,
      "step": 850
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2849295714741643e-06,
      "logits/chosen": -0.36678022146224976,
      "logits/rejected": -0.22024087607860565,
      "logps/chosen": -481.8533630371094,
      "logps/rejected": -517.580078125,
      "loss": 0.1795,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.08131915330886841,
      "rewards/margins": 0.054040372371673584,
      "rewards/rejected": -0.135359525680542,
      "step": 860
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2405943438375287e-06,
      "logits/chosen": -0.31432998180389404,
      "logits/rejected": -0.3275941014289856,
      "logps/chosen": -486.9906311035156,
      "logps/rejected": -556.7380981445312,
      "loss": 0.1659,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0830111876130104,
      "rewards/margins": 0.08041650056838989,
      "rewards/rejected": -0.1634276807308197,
      "step": 870
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.1960022911279036e-06,
      "logits/chosen": -0.3552636504173279,
      "logits/rejected": -0.12550072371959686,
      "logps/chosen": -442.874267578125,
      "logps/rejected": -486.37457275390625,
      "loss": 0.1946,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.06523702293634415,
      "rewards/margins": 0.0638941302895546,
      "rewards/rejected": -0.12913115322589874,
      "step": 880
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.1511688770836844e-06,
      "logits/chosen": -0.44293370842933655,
      "logits/rejected": -0.206680566072464,
      "logps/chosen": -426.76861572265625,
      "logps/rejected": -496.85565185546875,
      "loss": 0.1725,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.06692390143871307,
      "rewards/margins": 0.07236063480377197,
      "rewards/rejected": -0.13928455114364624,
      "step": 890
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1061096491431307e-06,
      "logits/chosen": -0.35239410400390625,
      "logits/rejected": -0.22438523173332214,
      "logps/chosen": -493.1455078125,
      "logps/rejected": -538.7332153320312,
      "loss": 0.189,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.08153922855854034,
      "rewards/margins": 0.05397678166627884,
      "rewards/rejected": -0.13551601767539978,
      "step": 900
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0608402330527796e-06,
      "logits/chosen": -0.28043216466903687,
      "logits/rejected": -0.32117584347724915,
      "logps/chosen": -489.95855712890625,
      "logps/rejected": -528.5255737304688,
      "loss": 0.1694,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.07012003660202026,
      "rewards/margins": 0.05748220160603523,
      "rewards/rejected": -0.1276022493839264,
      "step": 910
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0153763274487176e-06,
      "logits/chosen": -0.30975884199142456,
      "logits/rejected": -0.28235483169555664,
      "logps/chosen": -523.9342041015625,
      "logps/rejected": -557.6265869140625,
      "loss": 0.1987,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.08475781232118607,
      "rewards/margins": 0.04346821457147598,
      "rewards/rejected": -0.12822601199150085,
      "step": 920
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9697336984125683e-06,
      "logits/chosen": -0.3483017385005951,
      "logits/rejected": -0.30833274126052856,
      "logps/chosen": -446.2549743652344,
      "logps/rejected": -472.79736328125,
      "loss": 0.1947,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -0.07903579622507095,
      "rewards/margins": 0.04975394904613495,
      "rewards/rejected": -0.1287897378206253,
      "step": 930
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.923928174004094e-06,
      "logits/chosen": -0.34173694252967834,
      "logits/rejected": -0.2428359091281891,
      "logps/chosen": -514.5177612304688,
      "logps/rejected": -543.485595703125,
      "loss": 0.174,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.06802085787057877,
      "rewards/margins": 0.06197448447346687,
      "rewards/rejected": -0.12999534606933594,
      "step": 940
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8779756387723036e-06,
      "logits/chosen": -0.2911175787448883,
      "logits/rejected": -0.26822999119758606,
      "logps/chosen": -492.79412841796875,
      "logps/rejected": -546.5845947265625,
      "loss": 0.1747,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.068579763174057,
      "rewards/margins": 0.06541167199611664,
      "rewards/rejected": -0.13399145007133484,
      "step": 950
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.831892028246968e-06,
      "logits/chosen": -0.3150990307331085,
      "logits/rejected": -0.25644177198410034,
      "logps/chosen": -497.47216796875,
      "logps/rejected": -579.6931762695312,
      "loss": 0.1752,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.059874940663576126,
      "rewards/margins": 0.06721685081720352,
      "rewards/rejected": -0.12709179520606995,
      "step": 960
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7856933234124617e-06,
      "logits/chosen": -0.31079426407814026,
      "logits/rejected": -0.3151572346687317,
      "logps/chosen": -463.4168395996094,
      "logps/rejected": -555.4398193359375,
      "loss": 0.1846,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0642518401145935,
      "rewards/margins": 0.05405072122812271,
      "rewards/rejected": -0.11830256134271622,
      "step": 970
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7393955451658387e-06,
      "logits/chosen": -0.22878186404705048,
      "logits/rejected": -0.21153822541236877,
      "logps/chosen": -510.89837646484375,
      "logps/rejected": -539.2826538085938,
      "loss": 0.175,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.05998819321393967,
      "rewards/margins": 0.06107606738805771,
      "rewards/rejected": -0.12106426060199738,
      "step": 980
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6930147487610667e-06,
      "logits/chosen": -0.34622281789779663,
      "logits/rejected": -0.25016504526138306,
      "logps/chosen": -429.02178955078125,
      "logps/rejected": -502.2627868652344,
      "loss": 0.1741,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.05254564806818962,
      "rewards/margins": 0.06378427147865295,
      "rewards/rejected": -0.11632990837097168,
      "step": 990
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6465670182413487e-06,
      "logits/chosen": -0.37311479449272156,
      "logits/rejected": -0.20994921028614044,
      "logps/chosen": -451.9130859375,
      "logps/rejected": -485.57061767578125,
      "loss": 0.1822,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.05379350110888481,
      "rewards/margins": 0.04739636927843094,
      "rewards/rejected": -0.10118986666202545,
      "step": 1000
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.6000684608614594e-06,
      "logits/chosen": -0.34606754779815674,
      "logits/rejected": -0.33781155943870544,
      "logps/chosen": -458.77392578125,
      "logps/rejected": -538.665283203125,
      "loss": 0.1762,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.04491495341062546,
      "rewards/margins": 0.07754696160554886,
      "rewards/rejected": -0.12246191501617432,
      "step": 1010
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5535352015020338e-06,
      "logits/chosen": -0.3478602468967438,
      "logits/rejected": -0.3592594563961029,
      "logps/chosen": -426.89324951171875,
      "logps/rejected": -497.1231994628906,
      "loss": 0.1811,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.05246494337916374,
      "rewards/margins": 0.06296879798173904,
      "rewards/rejected": -0.11543373763561249,
      "step": 1020
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.506983377077741e-06,
      "logits/chosen": -0.30938345193862915,
      "logits/rejected": -0.36706987023353577,
      "logps/chosen": -438.70477294921875,
      "logps/rejected": -479.84930419921875,
      "loss": 0.185,
      "rewards/accuracies": 0.38749998807907104,
      "rewards/chosen": -0.06825192272663116,
      "rewards/margins": 0.051073990762233734,
      "rewards/rejected": -0.1193259134888649,
      "step": 1030
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.460429130941289e-06,
      "logits/chosen": -0.30941054224967957,
      "logits/rejected": -0.3541705310344696,
      "logps/chosen": -500.3165588378906,
      "logps/rejected": -547.2388916015625,
      "loss": 0.1648,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.0632004514336586,
      "rewards/margins": 0.0647406131029129,
      "rewards/rejected": -0.1279410421848297,
      "step": 1040
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.413888607285192e-06,
      "logits/chosen": -0.369188129901886,
      "logits/rejected": -0.2474605143070221,
      "logps/chosen": -482.4578552246094,
      "logps/rejected": -470.77178955078125,
      "loss": 0.2005,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -0.06637545675039291,
      "rewards/margins": 0.03153007850050926,
      "rewards/rejected": -0.09790553897619247,
      "step": 1050
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.367377945543249e-06,
      "logits/chosen": -0.323347270488739,
      "logits/rejected": -0.30056172609329224,
      "logps/chosen": -450.601318359375,
      "logps/rejected": -498.0372619628906,
      "loss": 0.1822,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.06100805476307869,
      "rewards/margins": 0.048852551728487015,
      "rewards/rejected": -0.1098606139421463,
      "step": 1060
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.320913274793676e-06,
      "logits/chosen": -0.37067005038261414,
      "logits/rejected": -0.28159070014953613,
      "logps/chosen": -450.1123046875,
      "logps/rejected": -492.94903564453125,
      "loss": 0.177,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.054400503635406494,
      "rewards/margins": 0.05063963681459427,
      "rewards/rejected": -0.10504014790058136,
      "step": 1070
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.27451070816582e-06,
      "logits/chosen": -0.3574444651603699,
      "logits/rejected": -0.26874804496765137,
      "logps/chosen": -471.46002197265625,
      "logps/rejected": -542.9146728515625,
      "loss": 0.1728,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.06453348696231842,
      "rewards/margins": 0.07580892741680145,
      "rewards/rejected": -0.14034239947795868,
      "step": 1080
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.228186337252414e-06,
      "logits/chosen": -0.2806636393070221,
      "logits/rejected": -0.29087376594543457,
      "logps/chosen": -486.2430725097656,
      "logps/rejected": -514.122314453125,
      "loss": 0.1749,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.07113119959831238,
      "rewards/margins": 0.05178775265812874,
      "rewards/rejected": -0.12291894853115082,
      "step": 1090
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1819562265292946e-06,
      "logits/chosen": -0.4109742045402527,
      "logits/rejected": -0.32353344559669495,
      "logps/chosen": -539.9480590820312,
      "logps/rejected": -556.8568725585938,
      "loss": 0.1739,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.07678203284740448,
      "rewards/margins": 0.04964344576001167,
      "rewards/rejected": -0.12642547488212585,
      "step": 1100
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1358364077845236e-06,
      "logits/chosen": -0.33106154203414917,
      "logits/rejected": -0.22853222489356995,
      "logps/chosen": -463.4998474121094,
      "logps/rejected": -510.4764709472656,
      "loss": 0.1836,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.0651940405368805,
      "rewards/margins": 0.05113544315099716,
      "rewards/rejected": -0.11632947623729706,
      "step": 1110
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.089842874558849e-06,
      "logits/chosen": -0.4000775218009949,
      "logits/rejected": -0.16853252053260803,
      "logps/chosen": -520.6731567382812,
      "logps/rejected": -535.2789306640625,
      "loss": 0.1777,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.06823070347309113,
      "rewards/margins": 0.046103235334157944,
      "rewards/rejected": -0.11433394253253937,
      "step": 1120
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0439915765994242e-06,
      "logits/chosen": -0.3381574749946594,
      "logits/rejected": -0.22323736548423767,
      "logps/chosen": -483.09814453125,
      "logps/rejected": -513.6597900390625,
      "loss": 0.2054,
      "rewards/accuracies": 0.4437499940395355,
      "rewards/chosen": -0.06595513969659805,
      "rewards/margins": 0.043463800102472305,
      "rewards/rejected": -0.10941894352436066,
      "step": 1130
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.9982984143287186e-06,
      "logits/chosen": -0.34239357709884644,
      "logits/rejected": -0.3451228439807892,
      "logps/chosen": -495.82403564453125,
      "logps/rejected": -536.5828857421875,
      "loss": 0.1641,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.05730292201042175,
      "rewards/margins": 0.05639542266726494,
      "rewards/rejected": -0.11369834840297699,
      "step": 1140
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.95277923333053e-06,
      "logits/chosen": -0.23302578926086426,
      "logits/rejected": -0.3353923261165619,
      "logps/chosen": -467.14642333984375,
      "logps/rejected": -511.3815002441406,
      "loss": 0.1702,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.05319993942975998,
      "rewards/margins": 0.055631499737501144,
      "rewards/rejected": -0.10883144289255142,
      "step": 1150
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.9074498188550156e-06,
      "logits/chosen": -0.3332996964454651,
      "logits/rejected": -0.34316807985305786,
      "logps/chosen": -486.32373046875,
      "logps/rejected": -549.6057739257812,
      "loss": 0.1644,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.06252233684062958,
      "rewards/margins": 0.06708262860774994,
      "rewards/rejected": -0.1296049803495407,
      "step": 1160
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.862325890344643e-06,
      "logits/chosen": -0.34824302792549133,
      "logits/rejected": -0.19805127382278442,
      "logps/chosen": -435.487060546875,
      "logps/rejected": -514.4375610351562,
      "loss": 0.1957,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.06748778373003006,
      "rewards/margins": 0.052836887538433075,
      "rewards/rejected": -0.12032465636730194,
      "step": 1170
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.817423095982972e-06,
      "logits/chosen": -0.4363935589790344,
      "logits/rejected": -0.2699471712112427,
      "logps/chosen": -517.750732421875,
      "logps/rejected": -548.2755126953125,
      "loss": 0.1871,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.08836901187896729,
      "rewards/margins": 0.04405100643634796,
      "rewards/rejected": -0.13242003321647644,
      "step": 1180
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.7727570072681293e-06,
      "logits/chosen": -0.3290655016899109,
      "logits/rejected": -0.3122135400772095,
      "logps/chosen": -466.1502990722656,
      "logps/rejected": -535.735107421875,
      "loss": 0.1699,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.07871656864881516,
      "rewards/margins": 0.06162920594215393,
      "rewards/rejected": -0.14034578204154968,
      "step": 1190
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7283431136128961e-06,
      "logits/chosen": -0.35683125257492065,
      "logits/rejected": -0.28000420331954956,
      "logps/chosen": -494.61016845703125,
      "logps/rejected": -513.5484619140625,
      "loss": 0.1875,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.07970693707466125,
      "rewards/margins": 0.04146048426628113,
      "rewards/rejected": -0.12116742134094238,
      "step": 1200
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6841968169732478e-06,
      "logits/chosen": -0.3199581801891327,
      "logits/rejected": -0.30323344469070435,
      "logps/chosen": -484.8779296875,
      "logps/rejected": -546.8536987304688,
      "loss": 0.1711,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.07513175904750824,
      "rewards/margins": 0.05645768716931343,
      "rewards/rejected": -0.13158944249153137,
      "step": 1210
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6403334265072284e-06,
      "logits/chosen": -0.2838456928730011,
      "logits/rejected": -0.31488656997680664,
      "logps/chosen": -470.95025634765625,
      "logps/rejected": -502.6404724121094,
      "loss": 0.1864,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.07776617258787155,
      "rewards/margins": 0.044505417346954346,
      "rewards/rejected": -0.1222715824842453,
      "step": 1220
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5967681532660066e-06,
      "logits/chosen": -0.2862758934497833,
      "logits/rejected": -0.38872987031936646,
      "logps/chosen": -515.7428588867188,
      "logps/rejected": -601.1578369140625,
      "loss": 0.1767,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.08725148439407349,
      "rewards/margins": 0.07090319693088531,
      "rewards/rejected": -0.15815469622612,
      "step": 1230
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5535161049189463e-06,
      "logits/chosen": -0.35917767882347107,
      "logits/rejected": -0.39498478174209595,
      "logps/chosen": -512.1259765625,
      "logps/rejected": -557.6096801757812,
      "loss": 0.1919,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.08681392669677734,
      "rewards/margins": 0.055614955723285675,
      "rewards/rejected": -0.14242888987064362,
      "step": 1240
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.5105922805145356e-06,
      "logits/chosen": -0.3822699189186096,
      "logits/rejected": -0.2847387194633484,
      "logps/chosen": -531.32861328125,
      "logps/rejected": -538.8329467773438,
      "loss": 0.1865,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.09042377769947052,
      "rewards/margins": 0.047544240951538086,
      "rewards/rejected": -0.1379680335521698,
      "step": 1250
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4680115652789823e-06,
      "logits/chosen": -0.2868042588233948,
      "logits/rejected": -0.39659756422042847,
      "logps/chosen": -440.13201904296875,
      "logps/rejected": -505.8741760253906,
      "loss": 0.168,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.06821772456169128,
      "rewards/margins": 0.07103266566991806,
      "rewards/rejected": -0.13925038278102875,
      "step": 1260
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4257887254542767e-06,
      "logits/chosen": -0.267185777425766,
      "logits/rejected": -0.3715210556983948,
      "logps/chosen": -508.6048278808594,
      "logps/rejected": -551.8297119140625,
      "loss": 0.1861,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.08317936956882477,
      "rewards/margins": 0.04879546910524368,
      "rewards/rejected": -0.13197484612464905,
      "step": 1270
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.3839384031775227e-06,
      "logits/chosen": -0.40115490555763245,
      "logits/rejected": -0.3220266103744507,
      "logps/chosen": -511.22564697265625,
      "logps/rejected": -581.5658569335938,
      "loss": 0.1907,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.07654689252376556,
      "rewards/margins": 0.05669323727488518,
      "rewards/rejected": -0.13324013352394104,
      "step": 1280
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.342475111403298e-06,
      "logits/chosen": -0.4519384801387787,
      "logits/rejected": -0.3560634255409241,
      "logps/chosen": -482.3561096191406,
      "logps/rejected": -541.8218383789062,
      "loss": 0.1587,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.07603658735752106,
      "rewards/margins": 0.062384773045778275,
      "rewards/rejected": -0.13842137157917023,
      "step": 1290
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3014132288708209e-06,
      "logits/chosen": -0.45893383026123047,
      "logits/rejected": -0.34163400530815125,
      "logps/chosen": -470.9329528808594,
      "logps/rejected": -551.7818603515625,
      "loss": 0.1698,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.07042945921421051,
      "rewards/margins": 0.07146573066711426,
      "rewards/rejected": -0.14189518988132477,
      "step": 1300
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2607669951176549e-06,
      "logits/chosen": -0.30523496866226196,
      "logits/rejected": -0.35816168785095215,
      "logps/chosen": -522.6578979492188,
      "logps/rejected": -601.38818359375,
      "loss": 0.1701,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.08582588285207748,
      "rewards/margins": 0.06938043236732483,
      "rewards/rejected": -0.15520630776882172,
      "step": 1310
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2205505055416891e-06,
      "logits/chosen": -0.4220711290836334,
      "logits/rejected": -0.37211546301841736,
      "logps/chosen": -488.38916015625,
      "logps/rejected": -604.0801391601562,
      "loss": 0.1738,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07974353432655334,
      "rewards/margins": 0.086810402572155,
      "rewards/rejected": -0.16655394434928894,
      "step": 1320
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1807777065131002e-06,
      "logits/chosen": -0.4147316515445709,
      "logits/rejected": -0.3836427628993988,
      "logps/chosen": -502.61083984375,
      "logps/rejected": -578.9619140625,
      "loss": 0.1864,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.08877801150083542,
      "rewards/margins": 0.07549368590116501,
      "rewards/rejected": -0.16427169740200043,
      "step": 1330
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1414623905380012e-06,
      "logits/chosen": -0.38667914271354675,
      "logits/rejected": -0.432734876871109,
      "logps/chosen": -504.5563049316406,
      "logps/rejected": -590.4224853515625,
      "loss": 0.1893,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.08736555278301239,
      "rewards/margins": 0.06142817810177803,
      "rewards/rejected": -0.14879372715950012,
      "step": 1340
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.1026181914754388e-06,
      "logits/chosen": -0.44079136848449707,
      "logits/rejected": -0.32765600085258484,
      "logps/chosen": -426.31103515625,
      "logps/rejected": -458.25,
      "loss": 0.1883,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.06336107105016708,
      "rewards/margins": 0.05088551715016365,
      "rewards/rejected": -0.11424658447504044,
      "step": 1350
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0642585798094136e-06,
      "logits/chosen": -0.41225147247314453,
      "logits/rejected": -0.35755234956741333,
      "logps/chosen": -410.5465393066406,
      "logps/rejected": -444.13201904296875,
      "loss": 0.1893,
      "rewards/accuracies": 0.38749998807907104,
      "rewards/chosen": -0.07661779224872589,
      "rewards/margins": 0.043170757591724396,
      "rewards/rejected": -0.11978854984045029,
      "step": 1360
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0263968579775522e-06,
      "logits/chosen": -0.43255501985549927,
      "logits/rejected": -0.3273727297782898,
      "logps/chosen": -493.48895263671875,
      "logps/rejected": -553.6163330078125,
      "loss": 0.1807,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.0860995277762413,
      "rewards/margins": 0.05855479836463928,
      "rewards/rejected": -0.1446543186903,
      "step": 1370
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.89046155758058e-07,
      "logits/chosen": -0.49861329793930054,
      "logits/rejected": -0.3107123374938965,
      "logps/chosen": -458.6407775878906,
      "logps/rejected": -507.7791442871094,
      "loss": 0.1735,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.06812812387943268,
      "rewards/margins": 0.05930028110742569,
      "rewards/rejected": -0.12742839753627777,
      "step": 1380
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.52219425716534e-07,
      "logits/chosen": -0.43581169843673706,
      "logits/rejected": -0.3831300139427185,
      "logps/chosen": -505.4359436035156,
      "logps/rejected": -514.2116088867188,
      "loss": 0.191,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.08728592097759247,
      "rewards/margins": 0.043969061225652695,
      "rewards/rejected": -0.13125498592853546,
      "step": 1390
    },
    {
      "epoch": 0.75,
      "learning_rate": 9.15929438714262e-07,
      "logits/chosen": -0.4287847876548767,
      "logits/rejected": -0.3385041356086731,
      "logps/chosen": -453.57763671875,
      "logps/rejected": -497.10394287109375,
      "loss": 0.1644,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.06731972098350525,
      "rewards/margins": 0.06951483339071274,
      "rewards/rejected": -0.13683456182479858,
      "step": 1400
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.801887794794911e-07,
      "logits/chosen": -0.45681437849998474,
      "logits/rejected": -0.33320799469947815,
      "logps/chosen": -503.00701904296875,
      "logps/rejected": -537.3406982421875,
      "loss": 0.186,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.08495952188968658,
      "rewards/margins": 0.056953202933073044,
      "rewards/rejected": -0.14191272854804993,
      "step": 1410
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.450098422432787e-07,
      "logits/chosen": -0.3651468753814697,
      "logits/rejected": -0.4669302999973297,
      "logps/chosen": -472.07421875,
      "logps/rejected": -524.6736450195312,
      "loss": 0.1901,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.0767470970749855,
      "rewards/margins": 0.049060117453336716,
      "rewards/rejected": -0.12580721080303192,
      "step": 1420
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.104048264413858e-07,
      "logits/chosen": -0.42758917808532715,
      "logits/rejected": -0.3809109330177307,
      "logps/chosen": -458.9986877441406,
      "logps/rejected": -511.10076904296875,
      "loss": 0.1656,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.061626553535461426,
      "rewards/margins": 0.05785534530878067,
      "rewards/rejected": -0.11948190629482269,
      "step": 1430
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.763857324837321e-07,
      "logits/chosen": -0.45241889357566833,
      "logits/rejected": -0.48053082823753357,
      "logps/chosen": -479.00628662109375,
      "logps/rejected": -570.8079223632812,
      "loss": 0.1688,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.07606343924999237,
      "rewards/margins": 0.077184297144413,
      "rewards/rejected": -0.15324774384498596,
      "step": 1440
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.429643575928605e-07,
      "logits/chosen": -0.4044586718082428,
      "logits/rejected": -0.4395798146724701,
      "logps/chosen": -458.71307373046875,
      "logps/rejected": -535.9743041992188,
      "loss": 0.1802,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.07682407647371292,
      "rewards/margins": 0.062133777886629105,
      "rewards/rejected": -0.13895784318447113,
      "step": 1450
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.101522917128709e-07,
      "logits/chosen": -0.5003517866134644,
      "logits/rejected": -0.33631640672683716,
      "logps/chosen": -469.9927673339844,
      "logps/rejected": -499.16119384765625,
      "loss": 0.1919,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.08875006437301636,
      "rewards/margins": 0.04381968826055527,
      "rewards/rejected": -0.13256976008415222,
      "step": 1460
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.779609134902312e-07,
      "logits/chosen": -0.4493805766105652,
      "logits/rejected": -0.3728955388069153,
      "logps/chosen": -463.2948303222656,
      "logps/rejected": -491.2294006347656,
      "loss": 0.1858,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.07361346483230591,
      "rewards/margins": 0.06247217208147049,
      "rewards/rejected": -0.1360856294631958,
      "step": 1470
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.464013863278629e-07,
      "logits/chosen": -0.4490404725074768,
      "logits/rejected": -0.4013081192970276,
      "logps/chosen": -473.97869873046875,
      "logps/rejected": -497.76239013671875,
      "loss": 0.1927,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0837077870965004,
      "rewards/margins": 0.04747476428747177,
      "rewards/rejected": -0.13118253648281097,
      "step": 1480
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.154846545138696e-07,
      "logits/chosen": -0.4343733787536621,
      "logits/rejected": -0.32414108514785767,
      "logps/chosen": -459.42059326171875,
      "logps/rejected": -485.1878967285156,
      "loss": 0.1714,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.08275729417800903,
      "rewards/margins": 0.050310492515563965,
      "rewards/rejected": -0.133067786693573,
      "step": 1490
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.852214394262515e-07,
      "logits/chosen": -0.38409170508384705,
      "logits/rejected": -0.36326664686203003,
      "logps/chosen": -479.1509704589844,
      "logps/rejected": -543.0540161132812,
      "loss": 0.1888,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.08453266322612762,
      "rewards/margins": 0.05342210456728935,
      "rewards/rejected": -0.13795478641986847,
      "step": 1500
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.556222358149191e-07,
      "logits/chosen": -0.3777012526988983,
      "logits/rejected": -0.4088537096977234,
      "logps/chosen": -484.7447204589844,
      "logps/rejected": -536.444091796875,
      "loss": 0.1752,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.08025570213794708,
      "rewards/margins": 0.061404865235090256,
      "rewards/rejected": -0.14166057109832764,
      "step": 1510
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.266973081622992e-07,
      "logits/chosen": -0.4726890027523041,
      "logits/rejected": -0.31746143102645874,
      "logps/chosen": -519.90869140625,
      "logps/rejected": -538.6578369140625,
      "loss": 0.1785,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07591833174228668,
      "rewards/margins": 0.06365291029214859,
      "rewards/rejected": -0.13957123458385468,
      "step": 1520
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.984566871237942e-07,
      "logits/chosen": -0.5020835399627686,
      "logits/rejected": -0.37848636507987976,
      "logps/chosen": -493.8661193847656,
      "logps/rejected": -555.9498901367188,
      "loss": 0.1738,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.08122778683900833,
      "rewards/margins": 0.06830001622438431,
      "rewards/rejected": -0.14952778816223145,
      "step": 1530
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.709101660493251e-07,
      "logits/chosen": -0.44509878754615784,
      "logits/rejected": -0.3348652720451355,
      "logps/chosen": -458.56353759765625,
      "logps/rejected": -519.80029296875,
      "loss": 0.1837,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.07414525747299194,
      "rewards/margins": 0.07978249341249466,
      "rewards/rejected": -0.153927743434906,
      "step": 1540
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.440672975871743e-07,
      "logits/chosen": -0.4507269859313965,
      "logits/rejected": -0.28932178020477295,
      "logps/chosen": -517.053955078125,
      "logps/rejected": -568.24658203125,
      "loss": 0.1837,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.08871571719646454,
      "rewards/margins": 0.0724785327911377,
      "rewards/rejected": -0.16119424998760223,
      "step": 1550
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.1793739037129134e-07,
      "logits/chosen": -0.44717317819595337,
      "logits/rejected": -0.3539753556251526,
      "logps/chosen": -513.8165283203125,
      "logps/rejected": -546.3040771484375,
      "loss": 0.183,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.08804842829704285,
      "rewards/margins": 0.05240340158343315,
      "rewards/rejected": -0.1404518187046051,
      "step": 1560
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.9252950579322405e-07,
      "logits/chosen": -0.4437643885612488,
      "logits/rejected": -0.43425798416137695,
      "logps/chosen": -429.33642578125,
      "logps/rejected": -503.93695068359375,
      "loss": 0.1803,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.06738166511058807,
      "rewards/margins": 0.07205136865377426,
      "rewards/rejected": -0.13943304121494293,
      "step": 1570
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.6785245485978864e-07,
      "logits/chosen": -0.42230138182640076,
      "logits/rejected": -0.33158788084983826,
      "logps/chosen": -474.5384826660156,
      "logps/rejected": -532.5901489257812,
      "loss": 0.1876,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.08723129332065582,
      "rewards/margins": 0.050881870090961456,
      "rewards/rejected": -0.13811317086219788,
      "step": 1580
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.43914795137566e-07,
      "logits/chosen": -0.4339612126350403,
      "logits/rejected": -0.4076845645904541,
      "logps/chosen": -514.6712036132812,
      "logps/rejected": -599.03955078125,
      "loss": 0.1866,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.09284873306751251,
      "rewards/margins": 0.05939553305506706,
      "rewards/rejected": -0.15224425494670868,
      "step": 1590
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.207248277852901e-07,
      "logits/chosen": -0.35802894830703735,
      "logits/rejected": -0.42679715156555176,
      "logps/chosen": -499.3904724121094,
      "logps/rejected": -601.59619140625,
      "loss": 0.1633,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.08157524466514587,
      "rewards/margins": 0.08413099497556686,
      "rewards/rejected": -0.16570623219013214,
      "step": 1600
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.9829059467515074e-07,
      "logits/chosen": -0.47917836904525757,
      "logits/rejected": -0.3484811782836914,
      "logps/chosen": -485.848388671875,
      "logps/rejected": -538.3430786132812,
      "loss": 0.1783,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.08013377338647842,
      "rewards/margins": 0.0631137266755104,
      "rewards/rejected": -0.14324750006198883,
      "step": 1610
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.766198756040153e-07,
      "logits/chosen": -0.4456513524055481,
      "logits/rejected": -0.32365337014198303,
      "logps/chosen": -411.17755126953125,
      "logps/rejected": -458.21923828125,
      "loss": 0.1884,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.06910572201013565,
      "rewards/margins": 0.057366736233234406,
      "rewards/rejected": -0.12647245824337006,
      "step": 1620
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.5572018559553155e-07,
      "logits/chosen": -0.39375001192092896,
      "logits/rejected": -0.37620794773101807,
      "logps/chosen": -486.57952880859375,
      "logps/rejected": -546.0811157226562,
      "loss": 0.1909,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.07672485709190369,
      "rewards/margins": 0.0658622533082962,
      "rewards/rejected": -0.1425871104001999,
      "step": 1630
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.3559877229404864e-07,
      "logits/chosen": -0.4211618900299072,
      "logits/rejected": -0.3744751513004303,
      "logps/chosen": -487.8899841308594,
      "logps/rejected": -532.0867309570312,
      "loss": 0.1696,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.08219683170318604,
      "rewards/margins": 0.05911056324839592,
      "rewards/rejected": -0.14130739867687225,
      "step": 1640
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.1626261345126576e-07,
      "logits/chosen": -0.44197311997413635,
      "logits/rejected": -0.36461585760116577,
      "logps/chosen": -481.8468322753906,
      "logps/rejected": -576.697021484375,
      "loss": 0.173,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06508458405733109,
      "rewards/margins": 0.08507412672042847,
      "rewards/rejected": -0.15015868842601776,
      "step": 1650
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.9771841450646505e-07,
      "logits/chosen": -0.5129082798957825,
      "logits/rejected": -0.3829159140586853,
      "logps/chosen": -495.10162353515625,
      "logps/rejected": -520.402587890625,
      "loss": 0.184,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.08100192248821259,
      "rewards/margins": 0.04889564588665962,
      "rewards/rejected": -0.1298975646495819,
      "step": 1660
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.7997260626118758e-07,
      "logits/chosen": -0.308317631483078,
      "logits/rejected": -0.4445367455482483,
      "logps/chosen": -456.50860595703125,
      "logps/rejected": -507.1785583496094,
      "loss": 0.1696,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.06736288219690323,
      "rewards/margins": 0.06645634770393372,
      "rewards/rejected": -0.13381922245025635,
      "step": 1670
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.6303134264914365e-07,
      "logits/chosen": -0.4731658399105072,
      "logits/rejected": -0.40087780356407166,
      "logps/chosen": -472.3287048339844,
      "logps/rejected": -530.8348388671875,
      "loss": 0.1823,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.07341121137142181,
      "rewards/margins": 0.05679051950573921,
      "rewards/rejected": -0.13020172715187073,
      "step": 1680
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.469004986021355e-07,
      "logits/chosen": -0.43905097246170044,
      "logits/rejected": -0.4049570560455322,
      "logps/chosen": -467.10504150390625,
      "logps/rejected": -543.2198486328125,
      "loss": 0.1844,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.09198932349681854,
      "rewards/margins": 0.05702856183052063,
      "rewards/rejected": -0.14901788532733917,
      "step": 1690
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.315856680127367e-07,
      "logits/chosen": -0.4321065843105316,
      "logits/rejected": -0.3883914649486542,
      "logps/chosen": -507.26641845703125,
      "logps/rejected": -551.1337280273438,
      "loss": 0.1953,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.08602117002010345,
      "rewards/margins": 0.05450272560119629,
      "rewards/rejected": -0.14052391052246094,
      "step": 1700
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1709216179442817e-07,
      "logits/chosen": -0.42082422971725464,
      "logits/rejected": -0.41699376702308655,
      "logps/chosen": -464.18939208984375,
      "logps/rejected": -534.9678344726562,
      "loss": 0.1756,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.07923159748315811,
      "rewards/margins": 0.06845531612634659,
      "rewards/rejected": -0.1476869136095047,
      "step": 1710
    },
    {
      "epoch": 0.92,
      "learning_rate": 1.0342500603986421e-07,
      "logits/chosen": -0.45704323053359985,
      "logits/rejected": -0.3829982876777649,
      "logps/chosen": -516.6412353515625,
      "logps/rejected": -563.4931640625,
      "loss": 0.1887,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.08942435681819916,
      "rewards/margins": 0.07117379456758499,
      "rewards/rejected": -0.16059814393520355,
      "step": 1720
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.058894027791643e-08,
      "logits/chosen": -0.4678065776824951,
      "logits/rejected": -0.39768505096435547,
      "logps/chosen": -384.6977233886719,
      "logps/rejected": -438.79754638671875,
      "loss": 0.1838,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.05616065859794617,
      "rewards/margins": 0.054783742874860764,
      "rewards/rejected": -0.11094440519809723,
      "step": 1730
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.858841583008592e-08,
      "logits/chosen": -0.45986437797546387,
      "logits/rejected": -0.3251883089542389,
      "logps/chosen": -523.9485473632812,
      "logps/rejected": -555.4884033203125,
      "loss": 0.1823,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.0797450989484787,
      "rewards/margins": 0.05564265325665474,
      "rewards/rejected": -0.13538774847984314,
      "step": 1740
    },
    {
      "epoch": 0.93,
      "learning_rate": 6.742759426686313e-08,
      "logits/chosen": -0.4108913540840149,
      "logits/rejected": -0.2947922945022583,
      "logps/chosen": -538.8464965820312,
      "logps/rejected": -580.0211791992188,
      "loss": 0.192,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.09397675096988678,
      "rewards/margins": 0.05662484094500542,
      "rewards/rejected": -0.1506015956401825,
      "step": 1750
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.7110345964571104e-08,
      "logits/chosen": -0.31575465202331543,
      "logits/rejected": -0.376641184091568,
      "logps/chosen": -487.80633544921875,
      "logps/rejected": -575.765625,
      "loss": 0.1692,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.0703391507267952,
      "rewards/margins": 0.08664790540933609,
      "rewards/rejected": -0.1569870412349701,
      "step": 1760
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.764024876318357e-08,
      "logits/chosen": -0.414639949798584,
      "logits/rejected": -0.43213406205177307,
      "logps/chosen": -588.9749755859375,
      "logps/rejected": -631.2474365234375,
      "loss": 0.1871,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.09701113402843475,
      "rewards/margins": 0.057556282728910446,
      "rewards/rejected": -0.1545674204826355,
      "step": 1770
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.902058672559633e-08,
      "logits/chosen": -0.42236852645874023,
      "logits/rejected": -0.3948751389980316,
      "logps/chosen": -486.19598388671875,
      "logps/rejected": -501.712646484375,
      "loss": 0.1907,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.08502433449029922,
      "rewards/margins": 0.051293183118104935,
      "rewards/rejected": -0.13631752133369446,
      "step": 1780
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.125434899876933e-08,
      "logits/chosen": -0.4508518576622009,
      "logits/rejected": -0.2803448438644409,
      "logps/chosen": -528.0969848632812,
      "logps/rejected": -578.2789306640625,
      "loss": 0.1681,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.0905873253941536,
      "rewards/margins": 0.07002563774585724,
      "rewards/rejected": -0.16061295568943024,
      "step": 1790
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.4344228777145873e-08,
      "logits/chosen": -0.4513494372367859,
      "logits/rejected": -0.4356984496116638,
      "logps/chosen": -499.56884765625,
      "logps/rejected": -582.083740234375,
      "loss": 0.1743,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.08331683278083801,
      "rewards/margins": 0.07355960458517075,
      "rewards/rejected": -0.15687642991542816,
      "step": 1800
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.829262236869772e-08,
      "logits/chosen": -0.46016424894332886,
      "logits/rejected": -0.29402580857276917,
      "logps/chosen": -451.9331970214844,
      "logps/rejected": -494.04815673828125,
      "loss": 0.1806,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.0696534812450409,
      "rewards/margins": 0.06958705186843872,
      "rewards/rejected": -0.13924053311347961,
      "step": 1810
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.3101628363929586e-08,
      "logits/chosen": -0.501002848148346,
      "logits/rejected": -0.35800036787986755,
      "logps/chosen": -489.6319274902344,
      "logps/rejected": -548.3452758789062,
      "loss": 0.1739,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.08254210650920868,
      "rewards/margins": 0.06711296737194061,
      "rewards/rejected": -0.1496550738811493,
      "step": 1820
    },
    {
      "epoch": 0.98,
      "learning_rate": 8.773046908123195e-09,
      "logits/chosen": -0.42636099457740784,
      "logits/rejected": -0.3364613652229309,
      "logps/chosen": -483.1874084472656,
      "logps/rejected": -515.3287353515625,
      "loss": 0.172,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.07863263040781021,
      "rewards/margins": 0.057226501405239105,
      "rewards/rejected": -0.1358591467142105,
      "step": 1830
    },
    {
      "epoch": 0.98,
      "learning_rate": 5.308379077080817e-09,
      "logits/chosen": -0.45698413252830505,
      "logits/rejected": -0.3893563151359558,
      "logps/chosen": -492.31610107421875,
      "logps/rejected": -549.0144653320312,
      "loss": 0.1795,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.08794066309928894,
      "rewards/margins": 0.055044692009687424,
      "rewards/rejected": -0.14298535883426666,
      "step": 1840
    },
    {
      "epoch": 0.99,
      "learning_rate": 2.7088263565760996e-09,
      "logits/chosen": -0.4179384112358093,
      "logits/rejected": -0.3843657672405243,
      "logps/chosen": -512.0191650390625,
      "logps/rejected": -557.4730224609375,
      "loss": 0.1648,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.07361841946840286,
      "rewards/margins": 0.06739214807748795,
      "rewards/rejected": -0.141010582447052,
      "step": 1850
    },
    {
      "epoch": 0.99,
      "learning_rate": 9.752902257023633e-10,
      "logits/chosen": -0.4060386121273041,
      "logits/rejected": -0.3960750699043274,
      "logps/chosen": -479.4234313964844,
      "logps/rejected": -551.9636840820312,
      "loss": 0.1975,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.08916465938091278,
      "rewards/margins": 0.05007479339838028,
      "rewards/rejected": -0.13923944532871246,
      "step": 1860
    },
    {
      "epoch": 1.0,
      "learning_rate": 1.083718442532189e-10,
      "logits/chosen": -0.42799386382102966,
      "logits/rejected": -0.3839925229549408,
      "logps/chosen": -454.05499267578125,
      "logps/rejected": -501.215087890625,
      "loss": 0.1847,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.07664839923381805,
      "rewards/margins": 0.05011936277151108,
      "rewards/rejected": -0.12676775455474854,
      "step": 1870
    },
    {
      "epoch": 1.0,
      "step": 1875,
      "total_flos": 0.0,
      "train_loss": 0.1858275575319926,
      "train_runtime": 11460.3939,
      "train_samples_per_second": 2.618,
      "train_steps_per_second": 0.164
    }
  ],
  "logging_steps": 10,
  "max_steps": 1875,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}