{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.9992254066615027,
  "eval_steps": 100,
  "global_step": 2904,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 1.7182130584192438e-09,
      "logits/chosen": -2.067911148071289,
      "logits/rejected": -1.9660799503326416,
      "logps/chosen": -266.23834228515625,
      "logps/rejected": -247.68406677246094,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "learning_rate": 1.718213058419244e-08,
      "logits/chosen": -2.029080390930176,
      "logits/rejected": -1.9344679117202759,
      "logps/chosen": -291.5944519042969,
      "logps/rejected": -213.14134216308594,
      "loss": 0.6938,
      "rewards/accuracies": 0.4652777910232544,
      "rewards/chosen": 0.002623258624225855,
      "rewards/margins": 0.0006023693131282926,
      "rewards/rejected": 0.002020889427512884,
      "step": 10
    },
    {
      "epoch": 0.02,
      "learning_rate": 3.436426116838488e-08,
      "logits/chosen": -2.0757479667663574,
      "logits/rejected": -1.9724565744400024,
      "logps/chosen": -287.428955078125,
      "logps/rejected": -239.43740844726562,
      "loss": 0.6925,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.0022773859091103077,
      "rewards/margins": 0.00433021504431963,
      "rewards/rejected": -0.0020528293680399656,
      "step": 20
    },
    {
      "epoch": 0.03,
      "learning_rate": 5.154639175257731e-08,
      "logits/chosen": -2.0716583728790283,
      "logits/rejected": -2.011603832244873,
      "logps/chosen": -255.67135620117188,
      "logps/rejected": -218.0836181640625,
      "loss": 0.6949,
      "rewards/accuracies": 0.5218750238418579,
      "rewards/chosen": -0.0011378427734598517,
      "rewards/margins": -0.0006913787801750004,
      "rewards/rejected": -0.0004464639350771904,
      "step": 30
    },
    {
      "epoch": 0.04,
      "learning_rate": 6.872852233676976e-08,
      "logits/chosen": -2.079998731613159,
      "logits/rejected": -2.0144975185394287,
      "logps/chosen": -276.287353515625,
      "logps/rejected": -230.61965942382812,
      "loss": 0.6944,
      "rewards/accuracies": 0.484375,
      "rewards/chosen": -0.0009711446473374963,
      "rewards/margins": -3.942637704312801e-06,
      "rewards/rejected": -0.0009672018932178617,
      "step": 40
    },
    {
      "epoch": 0.05,
      "learning_rate": 8.59106529209622e-08,
      "logits/chosen": -1.9645802974700928,
      "logits/rejected": -1.873591661453247,
      "logps/chosen": -269.802490234375,
      "logps/rejected": -218.36865234375,
      "loss": 0.6929,
      "rewards/accuracies": 0.534375011920929,
      "rewards/chosen": 0.003975920379161835,
      "rewards/margins": 0.0016158551443368196,
      "rewards/rejected": 0.002360065234825015,
      "step": 50
    },
    {
      "epoch": 0.06,
      "learning_rate": 1.0309278350515462e-07,
      "logits/chosen": -1.9920260906219482,
      "logits/rejected": -2.014172077178955,
      "logps/chosen": -263.1590270996094,
      "logps/rejected": -230.80264282226562,
      "loss": 0.6923,
      "rewards/accuracies": 0.534375011920929,
      "rewards/chosen": 0.0037864961195737123,
      "rewards/margins": 0.0038312277756631374,
      "rewards/rejected": -4.4732307287631556e-05,
      "step": 60
    },
    {
      "epoch": 0.07,
      "learning_rate": 1.202749140893471e-07,
      "logits/chosen": -2.0915229320526123,
      "logits/rejected": -1.9606736898422241,
      "logps/chosen": -292.42498779296875,
      "logps/rejected": -242.525146484375,
      "loss": 0.6927,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.0015628642868250608,
      "rewards/margins": -0.0018646640237420797,
      "rewards/rejected": 0.0003017998533323407,
      "step": 70
    },
    {
      "epoch": 0.08,
      "learning_rate": 1.3745704467353952e-07,
      "logits/chosen": -2.064213275909424,
      "logits/rejected": -1.9719228744506836,
      "logps/chosen": -279.91070556640625,
      "logps/rejected": -226.50143432617188,
      "loss": 0.6928,
      "rewards/accuracies": 0.49687498807907104,
      "rewards/chosen": -0.002248315839096904,
      "rewards/margins": 0.00012451801740098745,
      "rewards/rejected": -0.002372834598645568,
      "step": 80
    },
    {
      "epoch": 0.09,
      "learning_rate": 1.5463917525773197e-07,
      "logits/chosen": -1.9888744354248047,
      "logits/rejected": -1.9386621713638306,
      "logps/chosen": -261.8848571777344,
      "logps/rejected": -214.2974395751953,
      "loss": 0.6927,
      "rewards/accuracies": 0.503125011920929,
      "rewards/chosen": -0.001680875546298921,
      "rewards/margins": -2.3123900973587297e-05,
      "rewards/rejected": -0.001657751388847828,
      "step": 90
    },
    {
      "epoch": 0.1,
      "learning_rate": 1.718213058419244e-07,
      "logits/chosen": -1.9539144039154053,
      "logits/rejected": -1.8788636922836304,
      "logps/chosen": -271.3200378417969,
      "logps/rejected": -216.4374542236328,
      "loss": 0.6922,
      "rewards/accuracies": 0.534375011920929,
      "rewards/chosen": 0.0035204023588448763,
      "rewards/margins": 0.0032666183542460203,
      "rewards/rejected": 0.0002537833934184164,
      "step": 100
    },
    {
      "epoch": 0.11,
      "learning_rate": 1.8900343642611682e-07,
      "logits/chosen": -2.0144975185394287,
      "logits/rejected": -1.870060682296753,
      "logps/chosen": -256.0116271972656,
      "logps/rejected": -214.73751831054688,
      "loss": 0.6904,
      "rewards/accuracies": 0.5218750238418579,
      "rewards/chosen": 0.0029162573628127575,
      "rewards/margins": 0.004548544529825449,
      "rewards/rejected": -0.0016322873998433352,
      "step": 110
    },
    {
      "epoch": 0.12,
      "learning_rate": 2.0618556701030925e-07,
      "logits/chosen": -2.0126750469207764,
      "logits/rejected": -1.9877809286117554,
      "logps/chosen": -294.35772705078125,
      "logps/rejected": -230.9095916748047,
      "loss": 0.6898,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": 0.006383317522704601,
      "rewards/margins": 0.010343357920646667,
      "rewards/rejected": -0.003960040397942066,
      "step": 120
    },
    {
      "epoch": 0.13,
      "learning_rate": 2.2336769759450173e-07,
      "logits/chosen": -2.039785385131836,
      "logits/rejected": -1.9796234369277954,
      "logps/chosen": -270.469970703125,
      "logps/rejected": -223.00634765625,
      "loss": 0.6921,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": 0.008164801634848118,
      "rewards/margins": 0.00915917381644249,
      "rewards/rejected": -0.0009943728800863028,
      "step": 130
    },
    {
      "epoch": 0.14,
      "learning_rate": 2.405498281786942e-07,
      "logits/chosen": -2.1231284141540527,
      "logits/rejected": -2.010619640350342,
      "logps/chosen": -290.5582580566406,
      "logps/rejected": -214.4351043701172,
      "loss": 0.6908,
      "rewards/accuracies": 0.528124988079071,
      "rewards/chosen": 0.004269259050488472,
      "rewards/margins": 0.007282177917659283,
      "rewards/rejected": -0.0030129191000014544,
      "step": 140
    },
    {
      "epoch": 0.15,
      "learning_rate": 2.5773195876288655e-07,
      "logits/chosen": -2.0174431800842285,
      "logits/rejected": -2.000694751739502,
      "logps/chosen": -267.80718994140625,
      "logps/rejected": -228.5482177734375,
      "loss": 0.6879,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": 0.007982608862221241,
      "rewards/margins": 0.00814131461083889,
      "rewards/rejected": -0.00015870490460656583,
      "step": 150
    },
    {
      "epoch": 0.17,
      "learning_rate": 2.7491408934707903e-07,
      "logits/chosen": -2.1166934967041016,
      "logits/rejected": -2.0119781494140625,
      "logps/chosen": -268.8128356933594,
      "logps/rejected": -215.2762451171875,
      "loss": 0.6871,
      "rewards/accuracies": 0.6156250238418579,
      "rewards/chosen": 0.006122085265815258,
      "rewards/margins": 0.012544795870780945,
      "rewards/rejected": -0.006422711070626974,
      "step": 160
    },
    {
      "epoch": 0.18,
      "learning_rate": 2.9209621993127146e-07,
      "logits/chosen": -2.031827211380005,
      "logits/rejected": -2.0032095909118652,
      "logps/chosen": -270.9142150878906,
      "logps/rejected": -227.9224090576172,
      "loss": 0.6851,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.012479735538363457,
      "rewards/margins": 0.015380832366645336,
      "rewards/rejected": -0.0029010965954512358,
      "step": 170
    },
    {
      "epoch": 0.19,
      "learning_rate": 3.0927835051546394e-07,
      "logits/chosen": -2.0390381813049316,
      "logits/rejected": -1.9806989431381226,
      "logps/chosen": -251.3668975830078,
      "logps/rejected": -210.38339233398438,
      "loss": 0.6836,
      "rewards/accuracies": 0.621874988079071,
      "rewards/chosen": 0.01502176932990551,
      "rewards/margins": 0.022143321111798286,
      "rewards/rejected": -0.007121548987925053,
      "step": 180
    },
    {
      "epoch": 0.2,
      "learning_rate": 3.2646048109965636e-07,
      "logits/chosen": -2.027550220489502,
      "logits/rejected": -1.9014009237289429,
      "logps/chosen": -264.51934814453125,
      "logps/rejected": -213.80886840820312,
      "loss": 0.6818,
      "rewards/accuracies": 0.6343749761581421,
      "rewards/chosen": 0.015689093619585037,
      "rewards/margins": 0.027370622381567955,
      "rewards/rejected": -0.011681526899337769,
      "step": 190
    },
    {
      "epoch": 0.21,
      "learning_rate": 3.436426116838488e-07,
      "logits/chosen": -2.034299850463867,
      "logits/rejected": -1.9709196090698242,
      "logps/chosen": -273.13226318359375,
      "logps/rejected": -211.93576049804688,
      "loss": 0.6807,
      "rewards/accuracies": 0.621874988079071,
      "rewards/chosen": 0.016958530992269516,
      "rewards/margins": 0.023664183914661407,
      "rewards/rejected": -0.0067056515254080296,
      "step": 200
    },
    {
      "epoch": 0.22,
      "learning_rate": 3.608247422680412e-07,
      "logits/chosen": -2.0183119773864746,
      "logits/rejected": -1.9083303213119507,
      "logps/chosen": -275.7691345214844,
      "logps/rejected": -208.54074096679688,
      "loss": 0.6773,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.01859709620475769,
      "rewards/margins": 0.031639259308576584,
      "rewards/rejected": -0.013042164035141468,
      "step": 210
    },
    {
      "epoch": 0.23,
      "learning_rate": 3.7800687285223364e-07,
      "logits/chosen": -2.016589879989624,
      "logits/rejected": -1.9477627277374268,
      "logps/chosen": -259.4232177734375,
      "logps/rejected": -226.46463012695312,
      "loss": 0.6769,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.024477588012814522,
      "rewards/margins": 0.03540473431348801,
      "rewards/rejected": -0.010927150025963783,
      "step": 220
    },
    {
      "epoch": 0.24,
      "learning_rate": 3.9518900343642607e-07,
      "logits/chosen": -2.0396270751953125,
      "logits/rejected": -1.9370222091674805,
      "logps/chosen": -257.0804748535156,
      "logps/rejected": -235.4674835205078,
      "loss": 0.6789,
      "rewards/accuracies": 0.6343749761581421,
      "rewards/chosen": 0.023364732041954994,
      "rewards/margins": 0.03109937347471714,
      "rewards/rejected": -0.0077346437610685825,
      "step": 230
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.123711340206185e-07,
      "logits/chosen": -1.9943723678588867,
      "logits/rejected": -1.8853381872177124,
      "logps/chosen": -273.07086181640625,
      "logps/rejected": -206.6876983642578,
      "loss": 0.6748,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.022974053397774696,
      "rewards/margins": 0.03632950782775879,
      "rewards/rejected": -0.013355454429984093,
      "step": 240
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.2955326460481097e-07,
      "logits/chosen": -2.0440022945404053,
      "logits/rejected": -1.9706653356552124,
      "logps/chosen": -280.75225830078125,
      "logps/rejected": -229.0762939453125,
      "loss": 0.673,
      "rewards/accuracies": 0.6031249761581421,
      "rewards/chosen": 0.02196577563881874,
      "rewards/margins": 0.03916791081428528,
      "rewards/rejected": -0.017202135175466537,
      "step": 250
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.4673539518900345e-07,
      "logits/chosen": -1.9852807521820068,
      "logits/rejected": -1.913886308670044,
      "logps/chosen": -251.1424102783203,
      "logps/rejected": -209.325927734375,
      "loss": 0.6707,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.027156567201018333,
      "rewards/margins": 0.04766073822975159,
      "rewards/rejected": -0.020504174754023552,
      "step": 260
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.639175257731959e-07,
      "logits/chosen": -2.0986721515655518,
      "logits/rejected": -2.0114688873291016,
      "logps/chosen": -259.94476318359375,
      "logps/rejected": -216.35888671875,
      "loss": 0.6658,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.03609105944633484,
      "rewards/margins": 0.06675679981708527,
      "rewards/rejected": -0.030665744096040726,
      "step": 270
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.810996563573884e-07,
      "logits/chosen": -2.023041248321533,
      "logits/rejected": -1.9758949279785156,
      "logps/chosen": -279.7530212402344,
      "logps/rejected": -239.0085906982422,
      "loss": 0.6637,
      "rewards/accuracies": 0.684374988079071,
      "rewards/chosen": 0.0377194806933403,
      "rewards/margins": 0.07151970267295837,
      "rewards/rejected": -0.03380022197961807,
      "step": 280
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.982817869415807e-07,
      "logits/chosen": -1.9995616674423218,
      "logits/rejected": -1.9275434017181396,
      "logps/chosen": -255.81723022460938,
      "logps/rejected": -212.13525390625,
      "loss": 0.6633,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.0429212860763073,
      "rewards/margins": 0.07059122622013092,
      "rewards/rejected": -0.027669942006468773,
      "step": 290
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.982778415614236e-07,
      "logits/chosen": -1.9670026302337646,
      "logits/rejected": -1.9164260625839233,
      "logps/chosen": -247.3227996826172,
      "logps/rejected": -229.3400115966797,
      "loss": 0.6599,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.041430626064538956,
      "rewards/margins": 0.06908479332923889,
      "rewards/rejected": -0.027654165402054787,
      "step": 300
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.963643321852277e-07,
      "logits/chosen": -2.008664131164551,
      "logits/rejected": -2.0269663333892822,
      "logps/chosen": -278.21331787109375,
      "logps/rejected": -230.90005493164062,
      "loss": 0.6557,
      "rewards/accuracies": 0.6781250238418579,
      "rewards/chosen": 0.04368427395820618,
      "rewards/margins": 0.09482843428850174,
      "rewards/rejected": -0.05114416405558586,
      "step": 310
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.944508228090318e-07,
      "logits/chosen": -2.0187923908233643,
      "logits/rejected": -1.9102197885513306,
      "logps/chosen": -257.12445068359375,
      "logps/rejected": -189.22702026367188,
      "loss": 0.645,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.05188002064824104,
      "rewards/margins": 0.10916055738925934,
      "rewards/rejected": -0.057280540466308594,
      "step": 320
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.925373134328357e-07,
      "logits/chosen": -2.0044243335723877,
      "logits/rejected": -1.9528205394744873,
      "logps/chosen": -269.754150390625,
      "logps/rejected": -210.67074584960938,
      "loss": 0.6513,
      "rewards/accuracies": 0.653124988079071,
      "rewards/chosen": 0.05257422477006912,
      "rewards/margins": 0.09687207639217377,
      "rewards/rejected": -0.04429786279797554,
      "step": 330
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.906238040566398e-07,
      "logits/chosen": -2.006479263305664,
      "logits/rejected": -1.8888683319091797,
      "logps/chosen": -239.5758819580078,
      "logps/rejected": -227.4214630126953,
      "loss": 0.6485,
      "rewards/accuracies": 0.6656249761581421,
      "rewards/chosen": 0.039284877479076385,
      "rewards/margins": 0.08342822641134262,
      "rewards/rejected": -0.04414334148168564,
      "step": 340
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.887102946804438e-07,
      "logits/chosen": -2.125704288482666,
      "logits/rejected": -2.004089117050171,
      "logps/chosen": -290.8746337890625,
      "logps/rejected": -230.57009887695312,
      "loss": 0.6404,
      "rewards/accuracies": 0.715624988079071,
      "rewards/chosen": 0.06995740532875061,
      "rewards/margins": 0.1391131579875946,
      "rewards/rejected": -0.0691557452082634,
      "step": 350
    },
    {
      "epoch": 0.37,
      "learning_rate": 4.867967853042479e-07,
      "logits/chosen": -2.009556770324707,
      "logits/rejected": -2.0069127082824707,
      "logps/chosen": -254.616943359375,
      "logps/rejected": -215.4530792236328,
      "loss": 0.6299,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.05212836712598801,
      "rewards/margins": 0.15012125670909882,
      "rewards/rejected": -0.09799288213253021,
      "step": 360
    },
    {
      "epoch": 0.38,
      "learning_rate": 4.84883275928052e-07,
      "logits/chosen": -2.1299102306365967,
      "logits/rejected": -1.983077049255371,
      "logps/chosen": -284.6875305175781,
      "logps/rejected": -251.5108642578125,
      "loss": 0.6313,
      "rewards/accuracies": 0.7093750238418579,
      "rewards/chosen": 0.08003100007772446,
      "rewards/margins": 0.1480170488357544,
      "rewards/rejected": -0.06798602640628815,
      "step": 370
    },
    {
      "epoch": 0.39,
      "learning_rate": 4.82969766551856e-07,
      "logits/chosen": -2.0552008152008057,
      "logits/rejected": -1.9739630222320557,
      "logps/chosen": -276.0761413574219,
      "logps/rejected": -239.1278839111328,
      "loss": 0.6369,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.06775949150323868,
      "rewards/margins": 0.1486402004957199,
      "rewards/rejected": -0.08088071644306183,
      "step": 380
    },
    {
      "epoch": 0.4,
      "learning_rate": 4.810562571756601e-07,
      "logits/chosen": -2.0520501136779785,
      "logits/rejected": -1.945320725440979,
      "logps/chosen": -284.12017822265625,
      "logps/rejected": -231.8722686767578,
      "loss": 0.6302,
      "rewards/accuracies": 0.7281249761581421,
      "rewards/chosen": 0.06102215498685837,
      "rewards/margins": 0.17737627029418945,
      "rewards/rejected": -0.11635412275791168,
      "step": 390
    },
    {
      "epoch": 0.41,
      "learning_rate": 4.791427477994642e-07,
      "logits/chosen": -2.008812189102173,
      "logits/rejected": -1.9848194122314453,
      "logps/chosen": -277.79376220703125,
      "logps/rejected": -224.36083984375,
      "loss": 0.6243,
      "rewards/accuracies": 0.684374988079071,
      "rewards/chosen": 0.0664195567369461,
      "rewards/margins": 0.17244522273540497,
      "rewards/rejected": -0.10602565854787827,
      "step": 400
    },
    {
      "epoch": 0.42,
      "learning_rate": 4.772292384232682e-07,
      "logits/chosen": -2.009459972381592,
      "logits/rejected": -1.9265193939208984,
      "logps/chosen": -263.84259033203125,
      "logps/rejected": -243.3984832763672,
      "loss": 0.621,
      "rewards/accuracies": 0.6781250238418579,
      "rewards/chosen": 0.056257862597703934,
      "rewards/margins": 0.17955340445041656,
      "rewards/rejected": -0.12329553067684174,
      "step": 410
    },
    {
      "epoch": 0.43,
      "learning_rate": 4.753157290470723e-07,
      "logits/chosen": -2.0507168769836426,
      "logits/rejected": -1.961004614830017,
      "logps/chosen": -253.4503173828125,
      "logps/rejected": -205.6368408203125,
      "loss": 0.6309,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": 0.02212332747876644,
      "rewards/margins": 0.1393744796514511,
      "rewards/rejected": -0.11725115776062012,
      "step": 420
    },
    {
      "epoch": 0.44,
      "learning_rate": 4.7340221967087635e-07,
      "logits/chosen": -2.042525291442871,
      "logits/rejected": -1.9645607471466064,
      "logps/chosen": -264.80645751953125,
      "logps/rejected": -226.8705596923828,
      "loss": 0.6296,
      "rewards/accuracies": 0.684374988079071,
      "rewards/chosen": 0.02932189777493477,
      "rewards/margins": 0.15064987540245056,
      "rewards/rejected": -0.12132799625396729,
      "step": 430
    },
    {
      "epoch": 0.45,
      "learning_rate": 4.714887102946804e-07,
      "logits/chosen": -2.0784430503845215,
      "logits/rejected": -1.9688632488250732,
      "logps/chosen": -276.26812744140625,
      "logps/rejected": -225.707275390625,
      "loss": 0.6125,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.05800630897283554,
      "rewards/margins": 0.21699848771095276,
      "rewards/rejected": -0.15899215638637543,
      "step": 440
    },
    {
      "epoch": 0.46,
      "learning_rate": 4.6957520091848447e-07,
      "logits/chosen": -1.9616053104400635,
      "logits/rejected": -1.991091012954712,
      "logps/chosen": -256.5301208496094,
      "logps/rejected": -231.73568725585938,
      "loss": 0.6361,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.03827190026640892,
      "rewards/margins": 0.13751840591430664,
      "rewards/rejected": -0.09924649447202682,
      "step": 450
    },
    {
      "epoch": 0.48,
      "learning_rate": 4.6766169154228853e-07,
      "logits/chosen": -2.022930145263672,
      "logits/rejected": -1.9804435968399048,
      "logps/chosen": -263.57073974609375,
      "logps/rejected": -228.65420532226562,
      "loss": 0.5982,
      "rewards/accuracies": 0.6968749761581421,
      "rewards/chosen": 0.049893591552972794,
      "rewards/margins": 0.2338247299194336,
      "rewards/rejected": -0.1839311420917511,
      "step": 460
    },
    {
      "epoch": 0.49,
      "learning_rate": 4.657481821660926e-07,
      "logits/chosen": -2.0467746257781982,
      "logits/rejected": -1.957139015197754,
      "logps/chosen": -251.3452911376953,
      "logps/rejected": -205.276611328125,
      "loss": 0.6046,
      "rewards/accuracies": 0.6968749761581421,
      "rewards/chosen": 0.07499710470438004,
      "rewards/margins": 0.266188383102417,
      "rewards/rejected": -0.19119124114513397,
      "step": 470
    },
    {
      "epoch": 0.5,
      "learning_rate": 4.6383467278989666e-07,
      "logits/chosen": -2.097900867462158,
      "logits/rejected": -1.9721593856811523,
      "logps/chosen": -262.32122802734375,
      "logps/rejected": -229.8935546875,
      "loss": 0.6135,
      "rewards/accuracies": 0.6781250238418579,
      "rewards/chosen": 0.04003770649433136,
      "rewards/margins": 0.23067660629749298,
      "rewards/rejected": -0.19063889980316162,
      "step": 480
    },
    {
      "epoch": 0.51,
      "learning_rate": 4.6192116341370067e-07,
      "logits/chosen": -2.130352258682251,
      "logits/rejected": -1.9925979375839233,
      "logps/chosen": -264.1942443847656,
      "logps/rejected": -227.0887908935547,
      "loss": 0.5973,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": 0.06981977075338364,
      "rewards/margins": 0.277499794960022,
      "rewards/rejected": -0.20768003165721893,
      "step": 490
    },
    {
      "epoch": 0.52,
      "learning_rate": 4.6000765403750473e-07,
      "logits/chosen": -2.038247585296631,
      "logits/rejected": -1.940549612045288,
      "logps/chosen": -263.71923828125,
      "logps/rejected": -212.58712768554688,
      "loss": 0.5982,
      "rewards/accuracies": 0.721875011920929,
      "rewards/chosen": 0.09329665452241898,
      "rewards/margins": 0.28916245698928833,
      "rewards/rejected": -0.19586579501628876,
      "step": 500
    },
    {
      "epoch": 0.53,
      "learning_rate": 4.580941446613088e-07,
      "logits/chosen": -1.9596498012542725,
      "logits/rejected": -1.9526113271713257,
      "logps/chosen": -270.47509765625,
      "logps/rejected": -229.74697875976562,
      "loss": 0.598,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.053180061280727386,
      "rewards/margins": 0.26379942893981934,
      "rewards/rejected": -0.21061936020851135,
      "step": 510
    },
    {
      "epoch": 0.54,
      "learning_rate": 4.5618063528511285e-07,
      "logits/chosen": -2.062441825866699,
      "logits/rejected": -1.9531586170196533,
      "logps/chosen": -262.12518310546875,
      "logps/rejected": -221.9291229248047,
      "loss": 0.604,
      "rewards/accuracies": 0.7093750238418579,
      "rewards/chosen": 0.052028585225343704,
      "rewards/margins": 0.2801084518432617,
      "rewards/rejected": -0.2280798852443695,
      "step": 520
    },
    {
      "epoch": 0.55,
      "learning_rate": 4.542671259089169e-07,
      "logits/chosen": -1.9766355752944946,
      "logits/rejected": -1.936336874961853,
      "logps/chosen": -235.9636993408203,
      "logps/rejected": -208.9865264892578,
      "loss": 0.6075,
      "rewards/accuracies": 0.6781250238418579,
      "rewards/chosen": 0.024395454674959183,
      "rewards/margins": 0.23684442043304443,
      "rewards/rejected": -0.21244895458221436,
      "step": 530
    },
    {
      "epoch": 0.56,
      "learning_rate": 4.52353616532721e-07,
      "logits/chosen": -2.052635908126831,
      "logits/rejected": -1.9279887676239014,
      "logps/chosen": -272.1145324707031,
      "logps/rejected": -228.16616821289062,
      "loss": 0.6071,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.06684047728776932,
      "rewards/margins": 0.3025699555873871,
      "rewards/rejected": -0.23572945594787598,
      "step": 540
    },
    {
      "epoch": 0.57,
      "learning_rate": 4.5044010715652504e-07,
      "logits/chosen": -2.086524248123169,
      "logits/rejected": -1.9210243225097656,
      "logps/chosen": -265.66644287109375,
      "logps/rejected": -231.428466796875,
      "loss": 0.6064,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.044201988726854324,
      "rewards/margins": 0.2510104179382324,
      "rewards/rejected": -0.2068084180355072,
      "step": 550
    },
    {
      "epoch": 0.58,
      "learning_rate": 4.485265977803291e-07,
      "logits/chosen": -2.052487850189209,
      "logits/rejected": -1.963782548904419,
      "logps/chosen": -274.49444580078125,
      "logps/rejected": -228.4085693359375,
      "loss": 0.5943,
      "rewards/accuracies": 0.715624988079071,
      "rewards/chosen": 0.07493807375431061,
      "rewards/margins": 0.3155871033668518,
      "rewards/rejected": -0.2406490296125412,
      "step": 560
    },
    {
      "epoch": 0.59,
      "learning_rate": 4.4661308840413316e-07,
      "logits/chosen": -2.0338144302368164,
      "logits/rejected": -1.9587665796279907,
      "logps/chosen": -292.2133483886719,
      "logps/rejected": -242.80636596679688,
      "loss": 0.598,
      "rewards/accuracies": 0.721875011920929,
      "rewards/chosen": 0.07242296636104584,
      "rewards/margins": 0.331615149974823,
      "rewards/rejected": -0.25919219851493835,
      "step": 570
    },
    {
      "epoch": 0.6,
      "learning_rate": 4.446995790279372e-07,
      "logits/chosen": -2.022578477859497,
      "logits/rejected": -1.9327418804168701,
      "logps/chosen": -275.12005615234375,
      "logps/rejected": -230.9717559814453,
      "loss": 0.5784,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.08595068752765656,
      "rewards/margins": 0.37252914905548096,
      "rewards/rejected": -0.2865784466266632,
      "step": 580
    },
    {
      "epoch": 0.61,
      "learning_rate": 4.4278606965174123e-07,
      "logits/chosen": -2.0303807258605957,
      "logits/rejected": -1.9269481897354126,
      "logps/chosen": -267.8510437011719,
      "logps/rejected": -225.8118896484375,
      "loss": 0.5948,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.05344559997320175,
      "rewards/margins": 0.3274499475955963,
      "rewards/rejected": -0.2740043103694916,
      "step": 590
    },
    {
      "epoch": 0.62,
      "learning_rate": 4.408725602755453e-07,
      "logits/chosen": -2.0389864444732666,
      "logits/rejected": -1.9150155782699585,
      "logps/chosen": -240.3448028564453,
      "logps/rejected": -215.1417694091797,
      "loss": 0.6007,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.01124047301709652,
      "rewards/margins": 0.26555973291397095,
      "rewards/rejected": -0.2543192505836487,
      "step": 600
    },
    {
      "epoch": 0.63,
      "learning_rate": 4.3895905089934936e-07,
      "logits/chosen": -1.9739296436309814,
      "logits/rejected": -1.8714065551757812,
      "logps/chosen": -251.74917602539062,
      "logps/rejected": -219.4097900390625,
      "loss": 0.5993,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.04064112901687622,
      "rewards/margins": 0.270423948764801,
      "rewards/rejected": -0.22978278994560242,
      "step": 610
    },
    {
      "epoch": 0.64,
      "learning_rate": 4.370455415231534e-07,
      "logits/chosen": -2.0759198665618896,
      "logits/rejected": -1.94767165184021,
      "logps/chosen": -309.49407958984375,
      "logps/rejected": -251.93539428710938,
      "loss": 0.5811,
      "rewards/accuracies": 0.7093750238418579,
      "rewards/chosen": 0.08607195317745209,
      "rewards/margins": 0.3619040548801422,
      "rewards/rejected": -0.2758321166038513,
      "step": 620
    },
    {
      "epoch": 0.65,
      "learning_rate": 4.351320321469575e-07,
      "logits/chosen": -2.035043239593506,
      "logits/rejected": -1.9782638549804688,
      "logps/chosen": -262.66717529296875,
      "logps/rejected": -229.33609008789062,
      "loss": 0.591,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.05298718065023422,
      "rewards/margins": 0.3540084660053253,
      "rewards/rejected": -0.3010213077068329,
      "step": 630
    },
    {
      "epoch": 0.66,
      "learning_rate": 4.3321852277076154e-07,
      "logits/chosen": -2.0530083179473877,
      "logits/rejected": -1.959703803062439,
      "logps/chosen": -257.1555480957031,
      "logps/rejected": -225.45669555664062,
      "loss": 0.5719,
      "rewards/accuracies": 0.684374988079071,
      "rewards/chosen": 0.018595661967992783,
      "rewards/margins": 0.3380756974220276,
      "rewards/rejected": -0.3194800615310669,
      "step": 640
    },
    {
      "epoch": 0.67,
      "learning_rate": 4.313050133945656e-07,
      "logits/chosen": -2.026218891143799,
      "logits/rejected": -1.9713115692138672,
      "logps/chosen": -266.67706298828125,
      "logps/rejected": -217.25247192382812,
      "loss": 0.5864,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.06421274691820145,
      "rewards/margins": 0.4028696119785309,
      "rewards/rejected": -0.33865687251091003,
      "step": 650
    },
    {
      "epoch": 0.68,
      "learning_rate": 4.2939150401836967e-07,
      "logits/chosen": -2.0551767349243164,
      "logits/rejected": -1.9555637836456299,
      "logps/chosen": -277.0030212402344,
      "logps/rejected": -237.25796508789062,
      "loss": 0.5773,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": 0.0361778661608696,
      "rewards/margins": 0.36955034732818604,
      "rewards/rejected": -0.3333725035190582,
      "step": 660
    },
    {
      "epoch": 0.69,
      "learning_rate": 4.2747799464217373e-07,
      "logits/chosen": -2.0053582191467285,
      "logits/rejected": -1.964155912399292,
      "logps/chosen": -274.88311767578125,
      "logps/rejected": -226.7574005126953,
      "loss": 0.5813,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.0418616458773613,
      "rewards/margins": 0.3720625042915344,
      "rewards/rejected": -0.33020082116127014,
      "step": 670
    },
    {
      "epoch": 0.7,
      "learning_rate": 4.255644852659778e-07,
      "logits/chosen": -2.0546717643737793,
      "logits/rejected": -1.953681230545044,
      "logps/chosen": -285.4710388183594,
      "logps/rejected": -235.042724609375,
      "loss": 0.6163,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.025151943787932396,
      "rewards/margins": 0.2663043141365051,
      "rewards/rejected": -0.24115240573883057,
      "step": 680
    },
    {
      "epoch": 0.71,
      "learning_rate": 4.236509758897818e-07,
      "logits/chosen": -2.008673906326294,
      "logits/rejected": -2.008507490158081,
      "logps/chosen": -249.8711700439453,
      "logps/rejected": -212.5075225830078,
      "loss": 0.6118,
      "rewards/accuracies": 0.684374988079071,
      "rewards/chosen": -0.0028725930023938417,
      "rewards/margins": 0.3535704016685486,
      "rewards/rejected": -0.35644301772117615,
      "step": 690
    },
    {
      "epoch": 0.72,
      "learning_rate": 4.2173746651358586e-07,
      "logits/chosen": -2.024738073348999,
      "logits/rejected": -1.9645500183105469,
      "logps/chosen": -276.9503173828125,
      "logps/rejected": -216.16281127929688,
      "loss": 0.5986,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.018686672672629356,
      "rewards/margins": 0.3331596255302429,
      "rewards/rejected": -0.31447291374206543,
      "step": 700
    },
    {
      "epoch": 0.73,
      "learning_rate": 4.198239571373899e-07,
      "logits/chosen": -2.060493230819702,
      "logits/rejected": -1.9129841327667236,
      "logps/chosen": -254.92068481445312,
      "logps/rejected": -211.83877563476562,
      "loss": 0.5869,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.0011024778941646218,
      "rewards/margins": 0.3638603985309601,
      "rewards/rejected": -0.36275792121887207,
      "step": 710
    },
    {
      "epoch": 0.74,
      "learning_rate": 4.17910447761194e-07,
      "logits/chosen": -2.045718193054199,
      "logits/rejected": -2.025928497314453,
      "logps/chosen": -294.1854248046875,
      "logps/rejected": -255.2843017578125,
      "loss": 0.5978,
      "rewards/accuracies": 0.6781250238418579,
      "rewards/chosen": 0.029152315109968185,
      "rewards/margins": 0.2995935380458832,
      "rewards/rejected": -0.2704412341117859,
      "step": 720
    },
    {
      "epoch": 0.75,
      "learning_rate": 4.1599693838499805e-07,
      "logits/chosen": -2.038595676422119,
      "logits/rejected": -1.9576959609985352,
      "logps/chosen": -268.7452697753906,
      "logps/rejected": -240.6742401123047,
      "loss": 0.5823,
      "rewards/accuracies": 0.715624988079071,
      "rewards/chosen": 0.03609790652990341,
      "rewards/margins": 0.3287338614463806,
      "rewards/rejected": -0.292635977268219,
      "step": 730
    },
    {
      "epoch": 0.76,
      "learning_rate": 4.140834290088021e-07,
      "logits/chosen": -2.000624179840088,
      "logits/rejected": -1.9431140422821045,
      "logps/chosen": -274.5948181152344,
      "logps/rejected": -218.09573364257812,
      "loss": 0.5771,
      "rewards/accuracies": 0.6656249761581421,
      "rewards/chosen": -0.0008543640142306685,
      "rewards/margins": 0.4075881540775299,
      "rewards/rejected": -0.40844249725341797,
      "step": 740
    },
    {
      "epoch": 0.77,
      "learning_rate": 4.121699196326062e-07,
      "logits/chosen": -1.9728513956069946,
      "logits/rejected": -1.9374927282333374,
      "logps/chosen": -255.46408081054688,
      "logps/rejected": -212.78280639648438,
      "loss": 0.5899,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.01102383527904749,
      "rewards/margins": 0.3705114722251892,
      "rewards/rejected": -0.3594876229763031,
      "step": 750
    },
    {
      "epoch": 0.78,
      "learning_rate": 4.1025641025641024e-07,
      "logits/chosen": -1.9775402545928955,
      "logits/rejected": -1.9780277013778687,
      "logps/chosen": -252.36978149414062,
      "logps/rejected": -251.9305419921875,
      "loss": 0.5854,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.045078955590724945,
      "rewards/margins": 0.3284622132778168,
      "rewards/rejected": -0.3735411763191223,
      "step": 760
    },
    {
      "epoch": 0.8,
      "learning_rate": 4.083429008802143e-07,
      "logits/chosen": -2.0461549758911133,
      "logits/rejected": -1.897442102432251,
      "logps/chosen": -269.2337341308594,
      "logps/rejected": -231.2041778564453,
      "loss": 0.5673,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.01757672056555748,
      "rewards/margins": 0.3843061327934265,
      "rewards/rejected": -0.3667294383049011,
      "step": 770
    },
    {
      "epoch": 0.81,
      "learning_rate": 4.0642939150401836e-07,
      "logits/chosen": -2.1057353019714355,
      "logits/rejected": -1.9940143823623657,
      "logps/chosen": -267.21270751953125,
      "logps/rejected": -229.93746948242188,
      "loss": 0.573,
      "rewards/accuracies": 0.721875011920929,
      "rewards/chosen": -0.0010803266195580363,
      "rewards/margins": 0.4319356083869934,
      "rewards/rejected": -0.433015912771225,
      "step": 780
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.0451588212782237e-07,
      "logits/chosen": -2.0213711261749268,
      "logits/rejected": -1.9726299047470093,
      "logps/chosen": -266.59857177734375,
      "logps/rejected": -213.43722534179688,
      "loss": 0.5461,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": 0.03156222775578499,
      "rewards/margins": 0.4636878967285156,
      "rewards/rejected": -0.4321257174015045,
      "step": 790
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.0260237275162643e-07,
      "logits/chosen": -1.9949146509170532,
      "logits/rejected": -1.9421812295913696,
      "logps/chosen": -277.64373779296875,
      "logps/rejected": -235.33944702148438,
      "loss": 0.5902,
      "rewards/accuracies": 0.6468750238418579,
      "rewards/chosen": 0.017059288918972015,
      "rewards/margins": 0.35789138078689575,
      "rewards/rejected": -0.34083208441734314,
      "step": 800
    },
    {
      "epoch": 0.84,
      "learning_rate": 4.006888633754305e-07,
      "logits/chosen": -2.038794994354248,
      "logits/rejected": -1.943619728088379,
      "logps/chosen": -264.1832580566406,
      "logps/rejected": -225.75350952148438,
      "loss": 0.5496,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.014380586333572865,
      "rewards/margins": 0.4638059735298157,
      "rewards/rejected": -0.4494253695011139,
      "step": 810
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.9877535399923456e-07,
      "logits/chosen": -2.0996642112731934,
      "logits/rejected": -1.9559507369995117,
      "logps/chosen": -281.420654296875,
      "logps/rejected": -241.8292236328125,
      "loss": 0.5706,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": 0.036143362522125244,
      "rewards/margins": 0.47268205881118774,
      "rewards/rejected": -0.4365386366844177,
      "step": 820
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.968618446230386e-07,
      "logits/chosen": -2.040336847305298,
      "logits/rejected": -1.9150043725967407,
      "logps/chosen": -299.0230712890625,
      "logps/rejected": -229.7438201904297,
      "loss": 0.5555,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 0.0862685814499855,
      "rewards/margins": 0.593079686164856,
      "rewards/rejected": -0.5068112015724182,
      "step": 830
    },
    {
      "epoch": 0.87,
      "learning_rate": 3.949483352468427e-07,
      "logits/chosen": -2.0034563541412354,
      "logits/rejected": -1.9005334377288818,
      "logps/chosen": -263.16143798828125,
      "logps/rejected": -223.8854522705078,
      "loss": 0.6149,
      "rewards/accuracies": 0.659375011920929,
      "rewards/chosen": -0.07270374894142151,
      "rewards/margins": 0.34188348054885864,
      "rewards/rejected": -0.41458725929260254,
      "step": 840
    },
    {
      "epoch": 0.88,
      "learning_rate": 3.9303482587064674e-07,
      "logits/chosen": -2.024224042892456,
      "logits/rejected": -1.9794381856918335,
      "logps/chosen": -282.8343200683594,
      "logps/rejected": -233.40713500976562,
      "loss": 0.5746,
      "rewards/accuracies": 0.690625011920929,
      "rewards/chosen": -0.0055493758991360664,
      "rewards/margins": 0.4002881944179535,
      "rewards/rejected": -0.4058375954627991,
      "step": 850
    },
    {
      "epoch": 0.89,
      "learning_rate": 3.911213164944508e-07,
      "logits/chosen": -2.1257824897766113,
      "logits/rejected": -2.0204081535339355,
      "logps/chosen": -276.70672607421875,
      "logps/rejected": -241.56045532226562,
      "loss": 0.5721,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.0032120614778250456,
      "rewards/margins": 0.45397186279296875,
      "rewards/rejected": -0.45718392729759216,
      "step": 860
    },
    {
      "epoch": 0.9,
      "learning_rate": 3.8920780711825487e-07,
      "logits/chosen": -1.9964282512664795,
      "logits/rejected": -1.908174753189087,
      "logps/chosen": -286.13702392578125,
      "logps/rejected": -234.1906280517578,
      "loss": 0.5772,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.0039267390966415405,
      "rewards/margins": 0.4336286187171936,
      "rewards/rejected": -0.43755531311035156,
      "step": 870
    },
    {
      "epoch": 0.91,
      "learning_rate": 3.8729429774205893e-07,
      "logits/chosen": -2.024934768676758,
      "logits/rejected": -1.9702039957046509,
      "logps/chosen": -283.3232421875,
      "logps/rejected": -234.4091033935547,
      "loss": 0.5749,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.012345449067652225,
      "rewards/margins": 0.42594093084335327,
      "rewards/rejected": -0.43828636407852173,
      "step": 880
    },
    {
      "epoch": 0.92,
      "learning_rate": 3.8538078836586294e-07,
      "logits/chosen": -2.00795578956604,
      "logits/rejected": -2.0201306343078613,
      "logps/chosen": -253.5358123779297,
      "logps/rejected": -228.1576385498047,
      "loss": 0.5818,
      "rewards/accuracies": 0.715624988079071,
      "rewards/chosen": 0.026046928018331528,
      "rewards/margins": 0.48412972688674927,
      "rewards/rejected": -0.45808282494544983,
      "step": 890
    },
    {
      "epoch": 0.93,
      "learning_rate": 3.83467278989667e-07,
      "logits/chosen": -1.9805667400360107,
      "logits/rejected": -1.8971214294433594,
      "logps/chosen": -259.73486328125,
      "logps/rejected": -205.6741943359375,
      "loss": 0.5589,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.019825313240289688,
      "rewards/margins": 0.4857255816459656,
      "rewards/rejected": -0.5055509805679321,
      "step": 900
    },
    {
      "epoch": 0.94,
      "learning_rate": 3.8155376961347106e-07,
      "logits/chosen": -2.0091850757598877,
      "logits/rejected": -1.965311050415039,
      "logps/chosen": -243.44900512695312,
      "logps/rejected": -203.65777587890625,
      "loss": 0.58,
      "rewards/accuracies": 0.7093750238418579,
      "rewards/chosen": -0.024844542145729065,
      "rewards/margins": 0.46328315138816833,
      "rewards/rejected": -0.4881277084350586,
      "step": 910
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.796402602372751e-07,
      "logits/chosen": -2.1025612354278564,
      "logits/rejected": -2.0319173336029053,
      "logps/chosen": -278.753173828125,
      "logps/rejected": -242.23977661132812,
      "loss": 0.5718,
      "rewards/accuracies": 0.690625011920929,
      "rewards/chosen": -0.0475626215338707,
      "rewards/margins": 0.3921951651573181,
      "rewards/rejected": -0.4397578239440918,
      "step": 920
    },
    {
      "epoch": 0.96,
      "learning_rate": 3.777267508610792e-07,
      "logits/chosen": -1.9871509075164795,
      "logits/rejected": -1.9420807361602783,
      "logps/chosen": -247.9656982421875,
      "logps/rejected": -206.86196899414062,
      "loss": 0.5746,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.032572999596595764,
      "rewards/margins": 0.4663190245628357,
      "rewards/rejected": -0.49889200925827026,
      "step": 930
    },
    {
      "epoch": 0.97,
      "learning_rate": 3.7581324148488325e-07,
      "logits/chosen": -2.034703016281128,
      "logits/rejected": -1.9743045568466187,
      "logps/chosen": -283.9637451171875,
      "logps/rejected": -233.59152221679688,
      "loss": 0.5768,
      "rewards/accuracies": 0.684374988079071,
      "rewards/chosen": -0.024589989334344864,
      "rewards/margins": 0.41736704111099243,
      "rewards/rejected": -0.4419569969177246,
      "step": 940
    },
    {
      "epoch": 0.98,
      "learning_rate": 3.738997321086873e-07,
      "logits/chosen": -2.0080621242523193,
      "logits/rejected": -1.931701421737671,
      "logps/chosen": -269.7777404785156,
      "logps/rejected": -223.91110229492188,
      "loss": 0.5847,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.037136368453502655,
      "rewards/margins": 0.41323599219322205,
      "rewards/rejected": -0.4503723084926605,
      "step": 950
    },
    {
      "epoch": 0.99,
      "learning_rate": 3.7198622273249137e-07,
      "logits/chosen": -1.994866132736206,
      "logits/rejected": -1.9065024852752686,
      "logps/chosen": -264.7047119140625,
      "logps/rejected": -227.7999725341797,
      "loss": 0.5546,
      "rewards/accuracies": 0.715624988079071,
      "rewards/chosen": -0.01111307181417942,
      "rewards/margins": 0.4391903281211853,
      "rewards/rejected": -0.45030340552330017,
      "step": 960
    },
    {
      "epoch": 1.0,
      "eval_logits/chosen": -1.9394429922103882,
      "eval_logits/rejected": -1.8507767915725708,
      "eval_logps/chosen": -282.7758483886719,
      "eval_logps/rejected": -234.58616638183594,
      "eval_loss": 0.567847490310669,
      "eval_rewards/accuracies": 0.7053571343421936,
      "eval_rewards/chosen": -0.009052435867488384,
      "eval_rewards/margins": 0.4890976846218109,
      "eval_rewards/rejected": -0.4981500804424286,
      "eval_runtime": 874.7806,
      "eval_samples_per_second": 2.286,
      "eval_steps_per_second": 0.072,
      "step": 968
    },
    {
      "epoch": 1.0,
      "learning_rate": 3.7007271335629544e-07,
      "logits/chosen": -2.007869243621826,
      "logits/rejected": -1.9336410760879517,
      "logps/chosen": -262.53546142578125,
      "logps/rejected": -226.2677764892578,
      "loss": 0.5609,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.021611416712403297,
      "rewards/margins": 0.47191959619522095,
      "rewards/rejected": -0.4935310482978821,
      "step": 970
    },
    {
      "epoch": 1.01,
      "learning_rate": 3.681592039800995e-07,
      "logits/chosen": -2.0216825008392334,
      "logits/rejected": -1.932154655456543,
      "logps/chosen": -253.0286102294922,
      "logps/rejected": -214.9832000732422,
      "loss": 0.5595,
      "rewards/accuracies": 0.7093750238418579,
      "rewards/chosen": -0.021348293870687485,
      "rewards/margins": 0.5129477977752686,
      "rewards/rejected": -0.5342960953712463,
      "step": 980
    },
    {
      "epoch": 1.02,
      "learning_rate": 3.662456946039035e-07,
      "logits/chosen": -1.9851185083389282,
      "logits/rejected": -1.8990024328231812,
      "logps/chosen": -234.0498046875,
      "logps/rejected": -218.21554565429688,
      "loss": 0.6073,
      "rewards/accuracies": 0.684374988079071,
      "rewards/chosen": -0.08372095972299576,
      "rewards/margins": 0.39530184864997864,
      "rewards/rejected": -0.4790228009223938,
      "step": 990
    },
    {
      "epoch": 1.03,
      "learning_rate": 3.6433218522770757e-07,
      "logits/chosen": -2.0040273666381836,
      "logits/rejected": -1.9534374475479126,
      "logps/chosen": -261.78216552734375,
      "logps/rejected": -232.15872192382812,
      "loss": 0.586,
      "rewards/accuracies": 0.6968749761581421,
      "rewards/chosen": -0.06383053213357925,
      "rewards/margins": 0.39705249667167664,
      "rewards/rejected": -0.4608830511569977,
      "step": 1000
    },
    {
      "epoch": 1.04,
      "learning_rate": 3.6241867585151163e-07,
      "logits/chosen": -2.057312250137329,
      "logits/rejected": -1.9657714366912842,
      "logps/chosen": -271.1720886230469,
      "logps/rejected": -239.08377075195312,
      "loss": 0.5623,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.03590734675526619,
      "rewards/margins": 0.46565335988998413,
      "rewards/rejected": -0.5015607476234436,
      "step": 1010
    },
    {
      "epoch": 1.05,
      "learning_rate": 3.605051664753157e-07,
      "logits/chosen": -2.0540215969085693,
      "logits/rejected": -1.9568777084350586,
      "logps/chosen": -274.85723876953125,
      "logps/rejected": -215.91989135742188,
      "loss": 0.5587,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.06657890975475311,
      "rewards/margins": 0.5352650880813599,
      "rewards/rejected": -0.6018439531326294,
      "step": 1020
    },
    {
      "epoch": 1.06,
      "learning_rate": 3.5859165709911975e-07,
      "logits/chosen": -2.0053391456604004,
      "logits/rejected": -1.9338204860687256,
      "logps/chosen": -280.01971435546875,
      "logps/rejected": -240.06552124023438,
      "loss": 0.5448,
      "rewards/accuracies": 0.721875011920929,
      "rewards/chosen": 0.02629667893052101,
      "rewards/margins": 0.537010133266449,
      "rewards/rejected": -0.510713517665863,
      "step": 1030
    },
    {
      "epoch": 1.07,
      "learning_rate": 3.566781477229238e-07,
      "logits/chosen": -2.038978099822998,
      "logits/rejected": -1.9422070980072021,
      "logps/chosen": -271.25006103515625,
      "logps/rejected": -247.28543090820312,
      "loss": 0.5631,
      "rewards/accuracies": 0.7093750238418579,
      "rewards/chosen": 0.002108191605657339,
      "rewards/margins": 0.47303467988967896,
      "rewards/rejected": -0.4709264636039734,
      "step": 1040
    },
    {
      "epoch": 1.08,
      "learning_rate": 3.547646383467279e-07,
      "logits/chosen": -2.0076701641082764,
      "logits/rejected": -1.9359521865844727,
      "logps/chosen": -265.30242919921875,
      "logps/rejected": -219.84677124023438,
      "loss": 0.5406,
      "rewards/accuracies": 0.753125011920929,
      "rewards/chosen": -0.022836418822407722,
      "rewards/margins": 0.5607357621192932,
      "rewards/rejected": -0.5835721492767334,
      "step": 1050
    },
    {
      "epoch": 1.09,
      "learning_rate": 3.5285112897053194e-07,
      "logits/chosen": -2.00288462638855,
      "logits/rejected": -1.867516279220581,
      "logps/chosen": -261.2705993652344,
      "logps/rejected": -231.8849334716797,
      "loss": 0.5549,
      "rewards/accuracies": 0.715624988079071,
      "rewards/chosen": -0.03538358584046364,
      "rewards/margins": 0.5409261584281921,
      "rewards/rejected": -0.5763096809387207,
      "step": 1060
    },
    {
      "epoch": 1.11,
      "learning_rate": 3.50937619594336e-07,
      "logits/chosen": -2.028400421142578,
      "logits/rejected": -1.8853849172592163,
      "logps/chosen": -275.1874084472656,
      "logps/rejected": -231.4445037841797,
      "loss": 0.5522,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.019299419596791267,
      "rewards/margins": 0.51097571849823,
      "rewards/rejected": -0.5302751660346985,
      "step": 1070
    },
    {
      "epoch": 1.12,
      "learning_rate": 3.4902411021814007e-07,
      "logits/chosen": -2.072242498397827,
      "logits/rejected": -1.9163639545440674,
      "logps/chosen": -270.66961669921875,
      "logps/rejected": -220.4494171142578,
      "loss": 0.5427,
      "rewards/accuracies": 0.721875011920929,
      "rewards/chosen": -0.027892639860510826,
      "rewards/margins": 0.49505919218063354,
      "rewards/rejected": -0.5229519009590149,
      "step": 1080
    },
    {
      "epoch": 1.13,
      "learning_rate": 3.4711060084194413e-07,
      "logits/chosen": -1.9672114849090576,
      "logits/rejected": -1.8720566034317017,
      "logps/chosen": -266.3045959472656,
      "logps/rejected": -223.64028930664062,
      "loss": 0.5794,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.08450685441493988,
      "rewards/margins": 0.42633262276649475,
      "rewards/rejected": -0.510839581489563,
      "step": 1090
    },
    {
      "epoch": 1.14,
      "learning_rate": 3.4519709146574814e-07,
      "logits/chosen": -1.9523357152938843,
      "logits/rejected": -1.9406931400299072,
      "logps/chosen": -243.3000946044922,
      "logps/rejected": -213.4463348388672,
      "loss": 0.5655,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.11162979900836945,
      "rewards/margins": 0.41768351197242737,
      "rewards/rejected": -0.5293132662773132,
      "step": 1100
    },
    {
      "epoch": 1.15,
      "learning_rate": 3.432835820895522e-07,
      "logits/chosen": -2.0237271785736084,
      "logits/rejected": -1.9257009029388428,
      "logps/chosen": -268.1784362792969,
      "logps/rejected": -244.5179443359375,
      "loss": 0.5485,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.0015948250656947494,
      "rewards/margins": 0.519138514995575,
      "rewards/rejected": -0.5175436735153198,
      "step": 1110
    },
    {
      "epoch": 1.16,
      "learning_rate": 3.4137007271335626e-07,
      "logits/chosen": -1.979138970375061,
      "logits/rejected": -1.8770424127578735,
      "logps/chosen": -248.8733673095703,
      "logps/rejected": -217.2586669921875,
      "loss": 0.5551,
      "rewards/accuracies": 0.690625011920929,
      "rewards/chosen": -0.07530103623867035,
      "rewards/margins": 0.49078720808029175,
      "rewards/rejected": -0.5660881996154785,
      "step": 1120
    },
    {
      "epoch": 1.17,
      "learning_rate": 3.394565633371603e-07,
      "logits/chosen": -1.9930393695831299,
      "logits/rejected": -1.8950424194335938,
      "logps/chosen": -284.9536437988281,
      "logps/rejected": -229.8956298828125,
      "loss": 0.5479,
      "rewards/accuracies": 0.715624988079071,
      "rewards/chosen": -0.04036648944020271,
      "rewards/margins": 0.5784170627593994,
      "rewards/rejected": -0.6187835931777954,
      "step": 1130
    },
    {
      "epoch": 1.18,
      "learning_rate": 3.375430539609644e-07,
      "logits/chosen": -1.9363939762115479,
      "logits/rejected": -1.8576284646987915,
      "logps/chosen": -259.5916748046875,
      "logps/rejected": -230.9806365966797,
      "loss": 0.524,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.06810829788446426,
      "rewards/margins": 0.5644280314445496,
      "rewards/rejected": -0.6325362920761108,
      "step": 1140
    },
    {
      "epoch": 1.19,
      "learning_rate": 3.3562954458476845e-07,
      "logits/chosen": -2.0367255210876465,
      "logits/rejected": -1.9168630838394165,
      "logps/chosen": -275.58404541015625,
      "logps/rejected": -228.10073852539062,
      "loss": 0.5655,
      "rewards/accuracies": 0.6968749761581421,
      "rewards/chosen": -0.10140882432460785,
      "rewards/margins": 0.46295589208602905,
      "rewards/rejected": -0.5643647909164429,
      "step": 1150
    },
    {
      "epoch": 1.2,
      "learning_rate": 3.337160352085725e-07,
      "logits/chosen": -1.9923851490020752,
      "logits/rejected": -1.8931716680526733,
      "logps/chosen": -288.2662048339844,
      "logps/rejected": -243.01809692382812,
      "loss": 0.5404,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.05664486810564995,
      "rewards/margins": 0.5657026171684265,
      "rewards/rejected": -0.6223475337028503,
      "step": 1160
    },
    {
      "epoch": 1.21,
      "learning_rate": 3.3180252583237657e-07,
      "logits/chosen": -2.015646457672119,
      "logits/rejected": -1.8909307718276978,
      "logps/chosen": -284.00213623046875,
      "logps/rejected": -226.9560089111328,
      "loss": 0.5686,
      "rewards/accuracies": 0.7093750238418579,
      "rewards/chosen": -0.03462731093168259,
      "rewards/margins": 0.5479375123977661,
      "rewards/rejected": -0.5825648903846741,
      "step": 1170
    },
    {
      "epoch": 1.22,
      "learning_rate": 3.2988901645618063e-07,
      "logits/chosen": -1.993431806564331,
      "logits/rejected": -1.8949865102767944,
      "logps/chosen": -274.4952392578125,
      "logps/rejected": -233.14193725585938,
      "loss": 0.5515,
      "rewards/accuracies": 0.690625011920929,
      "rewards/chosen": -0.11015860736370087,
      "rewards/margins": 0.4747910499572754,
      "rewards/rejected": -0.5849496722221375,
      "step": 1180
    },
    {
      "epoch": 1.23,
      "learning_rate": 3.279755070799847e-07,
      "logits/chosen": -1.9973537921905518,
      "logits/rejected": -1.937077283859253,
      "logps/chosen": -267.6756896972656,
      "logps/rejected": -239.6106414794922,
      "loss": 0.5382,
      "rewards/accuracies": 0.7718750238418579,
      "rewards/chosen": -0.05113865062594414,
      "rewards/margins": 0.5464734435081482,
      "rewards/rejected": -0.5976120829582214,
      "step": 1190
    },
    {
      "epoch": 1.24,
      "learning_rate": 3.260619977037887e-07,
      "logits/chosen": -1.9573997259140015,
      "logits/rejected": -1.861267328262329,
      "logps/chosen": -258.67901611328125,
      "logps/rejected": -224.18716430664062,
      "loss": 0.5347,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07424889504909515,
      "rewards/margins": 0.5720986723899841,
      "rewards/rejected": -0.6463476419448853,
      "step": 1200
    },
    {
      "epoch": 1.25,
      "learning_rate": 3.2414848832759277e-07,
      "logits/chosen": -2.055069923400879,
      "logits/rejected": -1.9554643630981445,
      "logps/chosen": -266.70721435546875,
      "logps/rejected": -230.57394409179688,
      "loss": 0.536,
      "rewards/accuracies": 0.7593749761581421,
      "rewards/chosen": -0.05199545621871948,
      "rewards/margins": 0.5812665224075317,
      "rewards/rejected": -0.6332619190216064,
      "step": 1210
    },
    {
      "epoch": 1.26,
      "learning_rate": 3.2223497895139683e-07,
      "logits/chosen": -2.010925769805908,
      "logits/rejected": -1.9447513818740845,
      "logps/chosen": -263.6624755859375,
      "logps/rejected": -239.1702423095703,
      "loss": 0.5616,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.0350295789539814,
      "rewards/margins": 0.4964603781700134,
      "rewards/rejected": -0.5314900279045105,
      "step": 1220
    },
    {
      "epoch": 1.27,
      "learning_rate": 3.203214695752009e-07,
      "logits/chosen": -2.0407071113586426,
      "logits/rejected": -1.9032310247421265,
      "logps/chosen": -290.69842529296875,
      "logps/rejected": -240.09481811523438,
      "loss": 0.575,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.08812502771615982,
      "rewards/margins": 0.4931270182132721,
      "rewards/rejected": -0.5812520384788513,
      "step": 1230
    },
    {
      "epoch": 1.28,
      "learning_rate": 3.1840796019900495e-07,
      "logits/chosen": -2.0096232891082764,
      "logits/rejected": -1.9704147577285767,
      "logps/chosen": -275.1475830078125,
      "logps/rejected": -246.9510040283203,
      "loss": 0.5631,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.024968769401311874,
      "rewards/margins": 0.5439773797988892,
      "rewards/rejected": -0.5689461827278137,
      "step": 1240
    },
    {
      "epoch": 1.29,
      "learning_rate": 3.16494450822809e-07,
      "logits/chosen": -2.008104085922241,
      "logits/rejected": -1.9298263788223267,
      "logps/chosen": -273.0186767578125,
      "logps/rejected": -211.26467895507812,
      "loss": 0.5398,
      "rewards/accuracies": 0.721875011920929,
      "rewards/chosen": -0.08886951208114624,
      "rewards/margins": 0.5550605654716492,
      "rewards/rejected": -0.6439300775527954,
      "step": 1250
    },
    {
      "epoch": 1.3,
      "learning_rate": 3.145809414466131e-07,
      "logits/chosen": -2.024360418319702,
      "logits/rejected": -1.9324896335601807,
      "logps/chosen": -269.1455078125,
      "logps/rejected": -233.44287109375,
      "loss": 0.5557,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.016493279486894608,
      "rewards/margins": 0.541469931602478,
      "rewards/rejected": -0.5579632520675659,
      "step": 1260
    },
    {
      "epoch": 1.31,
      "learning_rate": 3.1266743207041714e-07,
      "logits/chosen": -1.9940264225006104,
      "logits/rejected": -1.9849786758422852,
      "logps/chosen": -279.35693359375,
      "logps/rejected": -226.4078369140625,
      "loss": 0.5579,
      "rewards/accuracies": 0.715624988079071,
      "rewards/chosen": -0.09030825644731522,
      "rewards/margins": 0.5069442391395569,
      "rewards/rejected": -0.5972524881362915,
      "step": 1270
    },
    {
      "epoch": 1.32,
      "learning_rate": 3.107539226942212e-07,
      "logits/chosen": -1.980614423751831,
      "logits/rejected": -1.9127609729766846,
      "logps/chosen": -258.0218200683594,
      "logps/rejected": -236.0473175048828,
      "loss": 0.5912,
      "rewards/accuracies": 0.690625011920929,
      "rewards/chosen": -0.06303338706493378,
      "rewards/margins": 0.47944989800453186,
      "rewards/rejected": -0.5424833297729492,
      "step": 1280
    },
    {
      "epoch": 1.33,
      "learning_rate": 3.0884041331802526e-07,
      "logits/chosen": -2.0076379776000977,
      "logits/rejected": -1.9233248233795166,
      "logps/chosen": -267.875244140625,
      "logps/rejected": -229.68124389648438,
      "loss": 0.5583,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.08163460344076157,
      "rewards/margins": 0.5768612027168274,
      "rewards/rejected": -0.6584957242012024,
      "step": 1290
    },
    {
      "epoch": 1.34,
      "learning_rate": 3.0692690394182927e-07,
      "logits/chosen": -2.0399606227874756,
      "logits/rejected": -2.0132060050964355,
      "logps/chosen": -290.6252746582031,
      "logps/rejected": -235.99917602539062,
      "loss": 0.5551,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.018822865560650826,
      "rewards/margins": 0.5866366624832153,
      "rewards/rejected": -0.6054595112800598,
      "step": 1300
    },
    {
      "epoch": 1.35,
      "learning_rate": 3.0501339456563334e-07,
      "logits/chosen": -2.0360805988311768,
      "logits/rejected": -1.9700591564178467,
      "logps/chosen": -269.5166015625,
      "logps/rejected": -219.4673309326172,
      "loss": 0.5436,
      "rewards/accuracies": 0.721875011920929,
      "rewards/chosen": -0.040003370493650436,
      "rewards/margins": 0.5832095742225647,
      "rewards/rejected": -0.6232129335403442,
      "step": 1310
    },
    {
      "epoch": 1.36,
      "learning_rate": 3.030998851894374e-07,
      "logits/chosen": -1.9755254983901978,
      "logits/rejected": -1.872722864151001,
      "logps/chosen": -236.5614471435547,
      "logps/rejected": -203.92333984375,
      "loss": 0.5274,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.09586571156978607,
      "rewards/margins": 0.5836740732192993,
      "rewards/rejected": -0.6795397996902466,
      "step": 1320
    },
    {
      "epoch": 1.37,
      "learning_rate": 3.0118637581324146e-07,
      "logits/chosen": -1.9765536785125732,
      "logits/rejected": -1.9652169942855835,
      "logps/chosen": -251.5444793701172,
      "logps/rejected": -222.391357421875,
      "loss": 0.5565,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.10412011295557022,
      "rewards/margins": 0.5656259655952454,
      "rewards/rejected": -0.6697460412979126,
      "step": 1330
    },
    {
      "epoch": 1.38,
      "learning_rate": 2.992728664370455e-07,
      "logits/chosen": -1.9934202432632446,
      "logits/rejected": -1.918143630027771,
      "logps/chosen": -251.29257202148438,
      "logps/rejected": -215.9584197998047,
      "loss": 0.5577,
      "rewards/accuracies": 0.715624988079071,
      "rewards/chosen": -0.0995405912399292,
      "rewards/margins": 0.5579150915145874,
      "rewards/rejected": -0.657455563545227,
      "step": 1340
    },
    {
      "epoch": 1.39,
      "learning_rate": 2.973593570608496e-07,
      "logits/chosen": -1.9801356792449951,
      "logits/rejected": -1.8740001916885376,
      "logps/chosen": -269.63409423828125,
      "logps/rejected": -226.4136505126953,
      "loss": 0.5604,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.07889227569103241,
      "rewards/margins": 0.6068841218948364,
      "rewards/rejected": -0.68577641248703,
      "step": 1350
    },
    {
      "epoch": 1.4,
      "learning_rate": 2.9544584768465365e-07,
      "logits/chosen": -1.9733374118804932,
      "logits/rejected": -1.8860597610473633,
      "logps/chosen": -270.7989196777344,
      "logps/rejected": -233.51712036132812,
      "loss": 0.5682,
      "rewards/accuracies": 0.684374988079071,
      "rewards/chosen": -0.12125426530838013,
      "rewards/margins": 0.4999443590641022,
      "rewards/rejected": -0.6211986541748047,
      "step": 1360
    },
    {
      "epoch": 1.41,
      "learning_rate": 2.935323383084577e-07,
      "logits/chosen": -2.055373191833496,
      "logits/rejected": -1.9579395055770874,
      "logps/chosen": -284.0872497558594,
      "logps/rejected": -218.99630737304688,
      "loss": 0.5564,
      "rewards/accuracies": 0.721875011920929,
      "rewards/chosen": -0.09817256033420563,
      "rewards/margins": 0.6084792613983154,
      "rewards/rejected": -0.7066518068313599,
      "step": 1370
    },
    {
      "epoch": 1.43,
      "learning_rate": 2.9161882893226177e-07,
      "logits/chosen": -1.9930992126464844,
      "logits/rejected": -1.8951447010040283,
      "logps/chosen": -263.5531005859375,
      "logps/rejected": -220.1865692138672,
      "loss": 0.5546,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.08504018932580948,
      "rewards/margins": 0.5876429080963135,
      "rewards/rejected": -0.67268306016922,
      "step": 1380
    },
    {
      "epoch": 1.44,
      "learning_rate": 2.8970531955606583e-07,
      "logits/chosen": -2.061654806137085,
      "logits/rejected": -1.9857797622680664,
      "logps/chosen": -281.9578552246094,
      "logps/rejected": -235.69680786132812,
      "loss": 0.5682,
      "rewards/accuracies": 0.7281249761581421,
      "rewards/chosen": -0.08588355034589767,
      "rewards/margins": 0.5460136532783508,
      "rewards/rejected": -0.6318972706794739,
      "step": 1390
    },
    {
      "epoch": 1.45,
      "learning_rate": 2.8779181017986984e-07,
      "logits/chosen": -2.0682482719421387,
      "logits/rejected": -1.935006856918335,
      "logps/chosen": -277.1342468261719,
      "logps/rejected": -245.4555206298828,
      "loss": 0.5532,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.0687127336859703,
      "rewards/margins": 0.5643824934959412,
      "rewards/rejected": -0.6330951452255249,
      "step": 1400
    },
    {
      "epoch": 1.46,
      "learning_rate": 2.858783008036739e-07,
      "logits/chosen": -1.976893663406372,
      "logits/rejected": -1.8818718194961548,
      "logps/chosen": -265.47113037109375,
      "logps/rejected": -224.09805297851562,
      "loss": 0.545,
      "rewards/accuracies": 0.721875011920929,
      "rewards/chosen": -0.10257352888584137,
      "rewards/margins": 0.539416491985321,
      "rewards/rejected": -0.641990065574646,
      "step": 1410
    },
    {
      "epoch": 1.47,
      "learning_rate": 2.8396479142747797e-07,
      "logits/chosen": -2.019167184829712,
      "logits/rejected": -1.9711027145385742,
      "logps/chosen": -269.1687927246094,
      "logps/rejected": -226.229736328125,
      "loss": 0.5455,
      "rewards/accuracies": 0.746874988079071,
      "rewards/chosen": -0.07154490053653717,
      "rewards/margins": 0.6214567422866821,
      "rewards/rejected": -0.6930015683174133,
      "step": 1420
    },
    {
      "epoch": 1.48,
      "learning_rate": 2.8205128205128203e-07,
      "logits/chosen": -1.9249579906463623,
      "logits/rejected": -1.9243910312652588,
      "logps/chosen": -255.308837890625,
      "logps/rejected": -243.5002899169922,
      "loss": 0.5503,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.08084328472614288,
      "rewards/margins": 0.5392208099365234,
      "rewards/rejected": -0.6200639605522156,
      "step": 1430
    },
    {
      "epoch": 1.49,
      "learning_rate": 2.801377726750861e-07,
      "logits/chosen": -2.0300347805023193,
      "logits/rejected": -1.9318290948867798,
      "logps/chosen": -263.9398498535156,
      "logps/rejected": -231.84042358398438,
      "loss": 0.5516,
      "rewards/accuracies": 0.7093750238418579,
      "rewards/chosen": -0.048500318080186844,
      "rewards/margins": 0.5793226957321167,
      "rewards/rejected": -0.6278230547904968,
      "step": 1440
    },
    {
      "epoch": 1.5,
      "learning_rate": 2.7822426329889015e-07,
      "logits/chosen": -1.9659664630889893,
      "logits/rejected": -1.8714187145233154,
      "logps/chosen": -249.6382293701172,
      "logps/rejected": -217.23147583007812,
      "loss": 0.578,
      "rewards/accuracies": 0.7093750238418579,
      "rewards/chosen": -0.14306652545928955,
      "rewards/margins": 0.5336703062057495,
      "rewards/rejected": -0.6767368316650391,
      "step": 1450
    },
    {
      "epoch": 1.51,
      "learning_rate": 2.763107539226942e-07,
      "logits/chosen": -2.0092709064483643,
      "logits/rejected": -1.8980295658111572,
      "logps/chosen": -263.8819885253906,
      "logps/rejected": -228.8859100341797,
      "loss": 0.563,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.11218663305044174,
      "rewards/margins": 0.49865245819091797,
      "rewards/rejected": -0.6108390688896179,
      "step": 1460
    },
    {
      "epoch": 1.52,
      "learning_rate": 2.743972445464983e-07,
      "logits/chosen": -1.9666690826416016,
      "logits/rejected": -1.9279935359954834,
      "logps/chosen": -260.71295166015625,
      "logps/rejected": -216.5955047607422,
      "loss": 0.5668,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.12362690269947052,
      "rewards/margins": 0.5188626050949097,
      "rewards/rejected": -0.642489492893219,
      "step": 1470
    },
    {
      "epoch": 1.53,
      "learning_rate": 2.7248373517030234e-07,
      "logits/chosen": -2.0014965534210205,
      "logits/rejected": -1.8935635089874268,
      "logps/chosen": -254.9728546142578,
      "logps/rejected": -228.10171508789062,
      "loss": 0.5456,
      "rewards/accuracies": 0.6781250238418579,
      "rewards/chosen": -0.1154770627617836,
      "rewards/margins": 0.5314809083938599,
      "rewards/rejected": -0.6469579935073853,
      "step": 1480
    },
    {
      "epoch": 1.54,
      "learning_rate": 2.705702257941064e-07,
      "logits/chosen": -2.0562753677368164,
      "logits/rejected": -1.9150091409683228,
      "logps/chosen": -304.14825439453125,
      "logps/rejected": -234.85153198242188,
      "loss": 0.5296,
      "rewards/accuracies": 0.778124988079071,
      "rewards/chosen": -0.06085296720266342,
      "rewards/margins": 0.620803952217102,
      "rewards/rejected": -0.6816568374633789,
      "step": 1490
    },
    {
      "epoch": 1.55,
      "learning_rate": 2.686567164179104e-07,
      "logits/chosen": -2.017127513885498,
      "logits/rejected": -1.9755398035049438,
      "logps/chosen": -258.0429992675781,
      "logps/rejected": -211.6239776611328,
      "loss": 0.5573,
      "rewards/accuracies": 0.715624988079071,
      "rewards/chosen": -0.11576205492019653,
      "rewards/margins": 0.5504833459854126,
      "rewards/rejected": -0.6662454009056091,
      "step": 1500
    },
    {
      "epoch": 1.56,
      "learning_rate": 2.6674320704171447e-07,
      "logits/chosen": -1.97178053855896,
      "logits/rejected": -1.9140660762786865,
      "logps/chosen": -265.73797607421875,
      "logps/rejected": -215.9640655517578,
      "loss": 0.532,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07512752711772919,
      "rewards/margins": 0.6296411752700806,
      "rewards/rejected": -0.7047686576843262,
      "step": 1510
    },
    {
      "epoch": 1.57,
      "learning_rate": 2.6482969766551853e-07,
      "logits/chosen": -1.9850788116455078,
      "logits/rejected": -1.9302679300308228,
      "logps/chosen": -242.3767547607422,
      "logps/rejected": -230.31936645507812,
      "loss": 0.5533,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.0870606005191803,
      "rewards/margins": 0.502352774143219,
      "rewards/rejected": -0.5894134044647217,
      "step": 1520
    },
    {
      "epoch": 1.58,
      "learning_rate": 2.629161882893226e-07,
      "logits/chosen": -1.9834896326065063,
      "logits/rejected": -1.944684624671936,
      "logps/chosen": -276.28167724609375,
      "logps/rejected": -224.51611328125,
      "loss": 0.5256,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.08709590137004852,
      "rewards/margins": 0.6131010055541992,
      "rewards/rejected": -0.7001968622207642,
      "step": 1530
    },
    {
      "epoch": 1.59,
      "learning_rate": 2.6100267891312666e-07,
      "logits/chosen": -2.0222060680389404,
      "logits/rejected": -1.9368455410003662,
      "logps/chosen": -283.56488037109375,
      "logps/rejected": -226.9105224609375,
      "loss": 0.5702,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0962596982717514,
      "rewards/margins": 0.5741952061653137,
      "rewards/rejected": -0.6704548597335815,
      "step": 1540
    },
    {
      "epoch": 1.6,
      "learning_rate": 2.590891695369307e-07,
      "logits/chosen": -1.9781615734100342,
      "logits/rejected": -1.8281803131103516,
      "logps/chosen": -259.8866271972656,
      "logps/rejected": -207.38735961914062,
      "loss": 0.5389,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.04752311855554581,
      "rewards/margins": 0.6475936770439148,
      "rewards/rejected": -0.6951168179512024,
      "step": 1550
    },
    {
      "epoch": 1.61,
      "learning_rate": 2.571756601607348e-07,
      "logits/chosen": -2.0197582244873047,
      "logits/rejected": -1.9641326665878296,
      "logps/chosen": -285.61285400390625,
      "logps/rejected": -232.82080078125,
      "loss": 0.5437,
      "rewards/accuracies": 0.7281249761581421,
      "rewards/chosen": -0.07258148491382599,
      "rewards/margins": 0.6079379320144653,
      "rewards/rejected": -0.6805194616317749,
      "step": 1560
    },
    {
      "epoch": 1.62,
      "learning_rate": 2.5526215078453884e-07,
      "logits/chosen": -2.0040359497070312,
      "logits/rejected": -2.0012545585632324,
      "logps/chosen": -277.3008728027344,
      "logps/rejected": -250.6426544189453,
      "loss": 0.5448,
      "rewards/accuracies": 0.7281249761581421,
      "rewards/chosen": -0.08066655695438385,
      "rewards/margins": 0.5909748077392578,
      "rewards/rejected": -0.6716413497924805,
      "step": 1570
    },
    {
      "epoch": 1.63,
      "learning_rate": 2.533486414083429e-07,
      "logits/chosen": -2.0492279529571533,
      "logits/rejected": -2.0106427669525146,
      "logps/chosen": -290.88690185546875,
      "logps/rejected": -222.43844604492188,
      "loss": 0.5248,
      "rewards/accuracies": 0.7718750238418579,
      "rewards/chosen": -0.05663519352674484,
      "rewards/margins": 0.7258026003837585,
      "rewards/rejected": -0.782437801361084,
      "step": 1580
    },
    {
      "epoch": 1.64,
      "learning_rate": 2.5143513203214697e-07,
      "logits/chosen": -2.0188114643096924,
      "logits/rejected": -1.9185588359832764,
      "logps/chosen": -271.60699462890625,
      "logps/rejected": -232.2000732421875,
      "loss": 0.5615,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07878180593252182,
      "rewards/margins": 0.5395839810371399,
      "rewards/rejected": -0.6183658242225647,
      "step": 1590
    },
    {
      "epoch": 1.65,
      "learning_rate": 2.49521622655951e-07,
      "logits/chosen": -1.9827849864959717,
      "logits/rejected": -1.8625599145889282,
      "logps/chosen": -269.30218505859375,
      "logps/rejected": -236.34426879882812,
      "loss": 0.5602,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.09624549746513367,
      "rewards/margins": 0.5341801047325134,
      "rewards/rejected": -0.6304256319999695,
      "step": 1600
    },
    {
      "epoch": 1.66,
      "learning_rate": 2.4760811327975504e-07,
      "logits/chosen": -2.0221943855285645,
      "logits/rejected": -1.9246753454208374,
      "logps/chosen": -268.66131591796875,
      "logps/rejected": -226.4938507080078,
      "loss": 0.5467,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.11650047451257706,
      "rewards/margins": 0.5621265172958374,
      "rewards/rejected": -0.6786269545555115,
      "step": 1610
    },
    {
      "epoch": 1.67,
      "learning_rate": 2.456946039035591e-07,
      "logits/chosen": -1.9986492395401,
      "logits/rejected": -2.009213924407959,
      "logps/chosen": -278.3379211425781,
      "logps/rejected": -229.5263214111328,
      "loss": 0.5695,
      "rewards/accuracies": 0.6656249761581421,
      "rewards/chosen": -0.15835532546043396,
      "rewards/margins": 0.4587886929512024,
      "rewards/rejected": -0.617143988609314,
      "step": 1620
    },
    {
      "epoch": 1.68,
      "learning_rate": 2.4378109452736316e-07,
      "logits/chosen": -2.032945156097412,
      "logits/rejected": -1.9581798315048218,
      "logps/chosen": -275.6993103027344,
      "logps/rejected": -223.4126739501953,
      "loss": 0.5353,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.030741268768906593,
      "rewards/margins": 0.6569101810455322,
      "rewards/rejected": -0.6876514554023743,
      "step": 1630
    },
    {
      "epoch": 1.69,
      "learning_rate": 2.418675851511672e-07,
      "logits/chosen": -1.9015934467315674,
      "logits/rejected": -1.951825737953186,
      "logps/chosen": -260.4790954589844,
      "logps/rejected": -224.67916870117188,
      "loss": 0.5695,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.10039174556732178,
      "rewards/margins": 0.5715700387954712,
      "rewards/rejected": -0.6719617247581482,
      "step": 1640
    },
    {
      "epoch": 1.7,
      "learning_rate": 2.399540757749713e-07,
      "logits/chosen": -2.040576934814453,
      "logits/rejected": -1.9723045825958252,
      "logps/chosen": -276.28387451171875,
      "logps/rejected": -231.3341064453125,
      "loss": 0.5466,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.10415879637002945,
      "rewards/margins": 0.5966567993164062,
      "rewards/rejected": -0.7008155584335327,
      "step": 1650
    },
    {
      "epoch": 1.71,
      "learning_rate": 2.3804056639877535e-07,
      "logits/chosen": -1.9606773853302002,
      "logits/rejected": -1.8021656274795532,
      "logps/chosen": -257.7226257324219,
      "logps/rejected": -222.80703735351562,
      "loss": 0.564,
      "rewards/accuracies": 0.746874988079071,
      "rewards/chosen": -0.02898462489247322,
      "rewards/margins": 0.6081128120422363,
      "rewards/rejected": -0.6370974779129028,
      "step": 1660
    },
    {
      "epoch": 1.72,
      "learning_rate": 2.361270570225794e-07,
      "logits/chosen": -1.965126633644104,
      "logits/rejected": -1.925889015197754,
      "logps/chosen": -280.3318176269531,
      "logps/rejected": -220.78652954101562,
      "loss": 0.521,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.013979366049170494,
      "rewards/margins": 0.6451238393783569,
      "rewards/rejected": -0.6591032147407532,
      "step": 1670
    },
    {
      "epoch": 1.74,
      "learning_rate": 2.3421354764638345e-07,
      "logits/chosen": -2.068727731704712,
      "logits/rejected": -1.9082515239715576,
      "logps/chosen": -275.98199462890625,
      "logps/rejected": -241.04684448242188,
      "loss": 0.564,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.09447016566991806,
      "rewards/margins": 0.5412524342536926,
      "rewards/rejected": -0.6357226371765137,
      "step": 1680
    },
    {
      "epoch": 1.75,
      "learning_rate": 2.323000382701875e-07,
      "logits/chosen": -2.0002846717834473,
      "logits/rejected": -1.9545902013778687,
      "logps/chosen": -280.78656005859375,
      "logps/rejected": -230.1374969482422,
      "loss": 0.5251,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06173492223024368,
      "rewards/margins": 0.6311557292938232,
      "rewards/rejected": -0.6928905844688416,
      "step": 1690
    },
    {
      "epoch": 1.76,
      "learning_rate": 2.3038652889399157e-07,
      "logits/chosen": -1.9079262018203735,
      "logits/rejected": -1.8384666442871094,
      "logps/chosen": -243.6171112060547,
      "logps/rejected": -227.16397094726562,
      "loss": 0.531,
      "rewards/accuracies": 0.746874988079071,
      "rewards/chosen": -0.12050791084766388,
      "rewards/margins": 0.5977984666824341,
      "rewards/rejected": -0.7183063626289368,
      "step": 1700
    },
    {
      "epoch": 1.77,
      "learning_rate": 2.2847301951779563e-07,
      "logits/chosen": -1.978109359741211,
      "logits/rejected": -1.9294769763946533,
      "logps/chosen": -262.1457824707031,
      "logps/rejected": -238.79464721679688,
      "loss": 0.5353,
      "rewards/accuracies": 0.715624988079071,
      "rewards/chosen": -0.12331397831439972,
      "rewards/margins": 0.6272130608558655,
      "rewards/rejected": -0.750527024269104,
      "step": 1710
    },
    {
      "epoch": 1.78,
      "learning_rate": 2.265595101415997e-07,
      "logits/chosen": -1.9727439880371094,
      "logits/rejected": -1.8363845348358154,
      "logps/chosen": -278.3986511230469,
      "logps/rejected": -249.4431610107422,
      "loss": 0.5496,
      "rewards/accuracies": 0.6968749761581421,
      "rewards/chosen": -0.06688477098941803,
      "rewards/margins": 0.6391483545303345,
      "rewards/rejected": -0.7060332298278809,
      "step": 1720
    },
    {
      "epoch": 1.79,
      "learning_rate": 2.2464600076540373e-07,
      "logits/chosen": -2.069528102874756,
      "logits/rejected": -2.0197949409484863,
      "logps/chosen": -289.6525573730469,
      "logps/rejected": -228.1117401123047,
      "loss": 0.5426,
      "rewards/accuracies": 0.746874988079071,
      "rewards/chosen": -0.09579763561487198,
      "rewards/margins": 0.6648635864257812,
      "rewards/rejected": -0.7606611847877502,
      "step": 1730
    },
    {
      "epoch": 1.8,
      "learning_rate": 2.227324913892078e-07,
      "logits/chosen": -1.9910539388656616,
      "logits/rejected": -1.8664817810058594,
      "logps/chosen": -275.5558166503906,
      "logps/rejected": -226.4915771484375,
      "loss": 0.5386,
      "rewards/accuracies": 0.715624988079071,
      "rewards/chosen": -0.14371807873249054,
      "rewards/margins": 0.6043248176574707,
      "rewards/rejected": -0.7480429410934448,
      "step": 1740
    },
    {
      "epoch": 1.81,
      "learning_rate": 2.2081898201301186e-07,
      "logits/chosen": -2.0246949195861816,
      "logits/rejected": -1.9563093185424805,
      "logps/chosen": -275.5317077636719,
      "logps/rejected": -229.1376190185547,
      "loss": 0.5386,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.08551143109798431,
      "rewards/margins": 0.5989985466003418,
      "rewards/rejected": -0.6845099329948425,
      "step": 1750
    },
    {
      "epoch": 1.82,
      "learning_rate": 2.1890547263681592e-07,
      "logits/chosen": -1.9884989261627197,
      "logits/rejected": -1.9024312496185303,
      "logps/chosen": -263.49151611328125,
      "logps/rejected": -228.72201538085938,
      "loss": 0.5394,
      "rewards/accuracies": 0.721875011920929,
      "rewards/chosen": -0.1202956885099411,
      "rewards/margins": 0.6059435606002808,
      "rewards/rejected": -0.7262393236160278,
      "step": 1760
    },
    {
      "epoch": 1.83,
      "learning_rate": 2.1699196326061998e-07,
      "logits/chosen": -2.0003156661987305,
      "logits/rejected": -1.931420922279358,
      "logps/chosen": -247.48104858398438,
      "logps/rejected": -230.5535430908203,
      "loss": 0.5538,
      "rewards/accuracies": 0.7093750238418579,
      "rewards/chosen": -0.18307314813137054,
      "rewards/margins": 0.5675541162490845,
      "rewards/rejected": -0.7506272196769714,
      "step": 1770
    },
    {
      "epoch": 1.84,
      "learning_rate": 2.1507845388442402e-07,
      "logits/chosen": -2.006080150604248,
      "logits/rejected": -1.950289011001587,
      "logps/chosen": -259.63653564453125,
      "logps/rejected": -214.5558624267578,
      "loss": 0.5366,
      "rewards/accuracies": 0.7093750238418579,
      "rewards/chosen": -0.09678243100643158,
      "rewards/margins": 0.556891918182373,
      "rewards/rejected": -0.6536744236946106,
      "step": 1780
    },
    {
      "epoch": 1.85,
      "learning_rate": 2.1316494450822808e-07,
      "logits/chosen": -2.001936912536621,
      "logits/rejected": -1.9808794260025024,
      "logps/chosen": -274.71990966796875,
      "logps/rejected": -227.1686553955078,
      "loss": 0.5538,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.14255772531032562,
      "rewards/margins": 0.6057079434394836,
      "rewards/rejected": -0.7482656836509705,
      "step": 1790
    },
    {
      "epoch": 1.86,
      "learning_rate": 2.1125143513203214e-07,
      "logits/chosen": -1.9766113758087158,
      "logits/rejected": -1.8938961029052734,
      "logps/chosen": -275.33319091796875,
      "logps/rejected": -217.985595703125,
      "loss": 0.5274,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.05210094526410103,
      "rewards/margins": 0.6903704404830933,
      "rewards/rejected": -0.7424713969230652,
      "step": 1800
    },
    {
      "epoch": 1.87,
      "learning_rate": 2.093379257558362e-07,
      "logits/chosen": -2.008404493331909,
      "logits/rejected": -1.8764822483062744,
      "logps/chosen": -264.9756774902344,
      "logps/rejected": -230.31790161132812,
      "loss": 0.5565,
      "rewards/accuracies": 0.715624988079071,
      "rewards/chosen": -0.1377423107624054,
      "rewards/margins": 0.545425295829773,
      "rewards/rejected": -0.6831676363945007,
      "step": 1810
    },
    {
      "epoch": 1.88,
      "learning_rate": 2.0742441637964026e-07,
      "logits/chosen": -1.948041319847107,
      "logits/rejected": -1.9308229684829712,
      "logps/chosen": -286.62335205078125,
      "logps/rejected": -231.4746856689453,
      "loss": 0.5245,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.09207528829574585,
      "rewards/margins": 0.6919302940368652,
      "rewards/rejected": -0.7840056419372559,
      "step": 1820
    },
    {
      "epoch": 1.89,
      "learning_rate": 2.055109070034443e-07,
      "logits/chosen": -2.0039772987365723,
      "logits/rejected": -1.9030472040176392,
      "logps/chosen": -279.74847412109375,
      "logps/rejected": -234.2050323486328,
      "loss": 0.5282,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.15545189380645752,
      "rewards/margins": 0.535307765007019,
      "rewards/rejected": -0.6907596588134766,
      "step": 1830
    },
    {
      "epoch": 1.9,
      "learning_rate": 2.0359739762724836e-07,
      "logits/chosen": -2.042912006378174,
      "logits/rejected": -1.9346129894256592,
      "logps/chosen": -291.7068786621094,
      "logps/rejected": -240.38339233398438,
      "loss": 0.5332,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.10581548511981964,
      "rewards/margins": 0.67552649974823,
      "rewards/rejected": -0.7813419699668884,
      "step": 1840
    },
    {
      "epoch": 1.91,
      "learning_rate": 2.0168388825105242e-07,
      "logits/chosen": -2.014024257659912,
      "logits/rejected": -1.8882499933242798,
      "logps/chosen": -274.66912841796875,
      "logps/rejected": -245.4554443359375,
      "loss": 0.5265,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.06417706608772278,
      "rewards/margins": 0.7017583847045898,
      "rewards/rejected": -0.7659355401992798,
      "step": 1850
    },
    {
      "epoch": 1.92,
      "learning_rate": 1.997703788748565e-07,
      "logits/chosen": -2.037475109100342,
      "logits/rejected": -1.9030876159667969,
      "logps/chosen": -286.89794921875,
      "logps/rejected": -218.78182983398438,
      "loss": 0.5452,
      "rewards/accuracies": 0.7593749761581421,
      "rewards/chosen": -0.0933770015835762,
      "rewards/margins": 0.6915080547332764,
      "rewards/rejected": -0.7848849296569824,
      "step": 1860
    },
    {
      "epoch": 1.93,
      "learning_rate": 1.9785686949866055e-07,
      "logits/chosen": -1.995123267173767,
      "logits/rejected": -1.9561309814453125,
      "logps/chosen": -275.4853515625,
      "logps/rejected": -236.72006225585938,
      "loss": 0.5678,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.15089653432369232,
      "rewards/margins": 0.46189427375793457,
      "rewards/rejected": -0.6127907633781433,
      "step": 1870
    },
    {
      "epoch": 1.94,
      "learning_rate": 1.9594336012246458e-07,
      "logits/chosen": -2.0241754055023193,
      "logits/rejected": -1.9385268688201904,
      "logps/chosen": -263.27264404296875,
      "logps/rejected": -232.6566162109375,
      "loss": 0.5237,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.030437633395195007,
      "rewards/margins": 0.69951331615448,
      "rewards/rejected": -0.7299508452415466,
      "step": 1880
    },
    {
      "epoch": 1.95,
      "learning_rate": 1.9402985074626865e-07,
      "logits/chosen": -2.011044979095459,
      "logits/rejected": -1.931051254272461,
      "logps/chosen": -275.4610290527344,
      "logps/rejected": -232.85659790039062,
      "loss": 0.5614,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1333492547273636,
      "rewards/margins": 0.5849149227142334,
      "rewards/rejected": -0.7182641625404358,
      "step": 1890
    },
    {
      "epoch": 1.96,
      "learning_rate": 1.921163413700727e-07,
      "logits/chosen": -1.9748862981796265,
      "logits/rejected": -1.8773696422576904,
      "logps/chosen": -263.586669921875,
      "logps/rejected": -236.25424194335938,
      "loss": 0.5823,
      "rewards/accuracies": 0.6968749761581421,
      "rewards/chosen": -0.19906353950500488,
      "rewards/margins": 0.5175008773803711,
      "rewards/rejected": -0.7165643572807312,
      "step": 1900
    },
    {
      "epoch": 1.97,
      "learning_rate": 1.9020283199387677e-07,
      "logits/chosen": -2.0126466751098633,
      "logits/rejected": -1.8699384927749634,
      "logps/chosen": -262.9295959472656,
      "logps/rejected": -229.73031616210938,
      "loss": 0.5594,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.05735217407345772,
      "rewards/margins": 0.637950599193573,
      "rewards/rejected": -0.6953027248382568,
      "step": 1910
    },
    {
      "epoch": 1.98,
      "learning_rate": 1.8828932261768083e-07,
      "logits/chosen": -2.0402626991271973,
      "logits/rejected": -1.917449712753296,
      "logps/chosen": -268.4327392578125,
      "logps/rejected": -221.59817504882812,
      "loss": 0.5291,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.070868119597435,
      "rewards/margins": 0.6999655365943909,
      "rewards/rejected": -0.7708336114883423,
      "step": 1920
    },
    {
      "epoch": 1.99,
      "learning_rate": 1.8637581324148487e-07,
      "logits/chosen": -2.0629758834838867,
      "logits/rejected": -1.9960095882415771,
      "logps/chosen": -284.08978271484375,
      "logps/rejected": -242.78659057617188,
      "loss": 0.5491,
      "rewards/accuracies": 0.7281249761581421,
      "rewards/chosen": -0.10976877063512802,
      "rewards/margins": 0.6353492140769958,
      "rewards/rejected": -0.7451180219650269,
      "step": 1930
    },
    {
      "epoch": 2.0,
      "eval_logits/chosen": -1.9162061214447021,
      "eval_logits/rejected": -1.827931523323059,
      "eval_logps/chosen": -283.52166748046875,
      "eval_logps/rejected": -236.8552703857422,
      "eval_loss": 0.5437692403793335,
      "eval_rewards/accuracies": 0.7192460298538208,
      "eval_rewards/chosen": -0.08363496512174606,
      "eval_rewards/margins": 0.6414214968681335,
      "eval_rewards/rejected": -0.7250563502311707,
      "eval_runtime": 875.182,
      "eval_samples_per_second": 2.285,
      "eval_steps_per_second": 0.072,
      "step": 1936
    },
    {
      "epoch": 2.0,
      "learning_rate": 1.8446230386528893e-07,
      "logits/chosen": -2.0089569091796875,
      "logits/rejected": -1.9302289485931396,
      "logps/chosen": -247.7608184814453,
      "logps/rejected": -208.72274780273438,
      "loss": 0.5511,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.14254480600357056,
      "rewards/margins": 0.5513399839401245,
      "rewards/rejected": -0.6938849091529846,
      "step": 1940
    },
    {
      "epoch": 2.01,
      "learning_rate": 1.82548794489093e-07,
      "logits/chosen": -2.0011610984802246,
      "logits/rejected": -1.95159113407135,
      "logps/chosen": -274.76385498046875,
      "logps/rejected": -252.4306182861328,
      "loss": 0.5413,
      "rewards/accuracies": 0.715624988079071,
      "rewards/chosen": -0.13799461722373962,
      "rewards/margins": 0.6053641438484192,
      "rewards/rejected": -0.7433587908744812,
      "step": 1950
    },
    {
      "epoch": 2.02,
      "learning_rate": 1.8063528511289706e-07,
      "logits/chosen": -1.971734642982483,
      "logits/rejected": -1.9144260883331299,
      "logps/chosen": -266.9194641113281,
      "logps/rejected": -260.69573974609375,
      "loss": 0.5458,
      "rewards/accuracies": 0.715624988079071,
      "rewards/chosen": -0.11405863612890244,
      "rewards/margins": 0.6057092547416687,
      "rewards/rejected": -0.7197679281234741,
      "step": 1960
    },
    {
      "epoch": 2.03,
      "learning_rate": 1.7872177573670112e-07,
      "logits/chosen": -1.9765701293945312,
      "logits/rejected": -1.987164855003357,
      "logps/chosen": -261.4850158691406,
      "logps/rejected": -214.2167510986328,
      "loss": 0.5407,
      "rewards/accuracies": 0.746874988079071,
      "rewards/chosen": -0.06964612007141113,
      "rewards/margins": 0.6810376644134521,
      "rewards/rejected": -0.7506837844848633,
      "step": 1970
    },
    {
      "epoch": 2.04,
      "learning_rate": 1.7680826636050515e-07,
      "logits/chosen": -1.9535974264144897,
      "logits/rejected": -1.8755991458892822,
      "logps/chosen": -275.89776611328125,
      "logps/rejected": -231.1304931640625,
      "loss": 0.5352,
      "rewards/accuracies": 0.7406250238418579,
      "rewards/chosen": -0.09391085803508759,
      "rewards/margins": 0.6898946762084961,
      "rewards/rejected": -0.7838054895401001,
      "step": 1980
    },
    {
      "epoch": 2.06,
      "learning_rate": 1.7489475698430921e-07,
      "logits/chosen": -1.9550580978393555,
      "logits/rejected": -1.9118692874908447,
      "logps/chosen": -261.0256652832031,
      "logps/rejected": -222.1853485107422,
      "loss": 0.5543,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.10755525529384613,
      "rewards/margins": 0.5950790643692017,
      "rewards/rejected": -0.7026342749595642,
      "step": 1990
    },
    {
      "epoch": 2.07,
      "learning_rate": 1.7298124760811328e-07,
      "logits/chosen": -2.0680158138275146,
      "logits/rejected": -1.8314135074615479,
      "logps/chosen": -273.5735778808594,
      "logps/rejected": -221.8857421875,
      "loss": 0.5556,
      "rewards/accuracies": 0.7593749761581421,
      "rewards/chosen": -0.08592827618122101,
      "rewards/margins": 0.7069720029830933,
      "rewards/rejected": -0.7929002046585083,
      "step": 2000
    },
    {
      "epoch": 2.08,
      "learning_rate": 1.7106773823191734e-07,
      "logits/chosen": -2.041013240814209,
      "logits/rejected": -1.9429569244384766,
      "logps/chosen": -261.6728820800781,
      "logps/rejected": -217.9615020751953,
      "loss": 0.5335,
      "rewards/accuracies": 0.7593749761581421,
      "rewards/chosen": -0.09488911926746368,
      "rewards/margins": 0.7252017259597778,
      "rewards/rejected": -0.8200907707214355,
      "step": 2010
    },
    {
      "epoch": 2.09,
      "learning_rate": 1.691542288557214e-07,
      "logits/chosen": -1.9867607355117798,
      "logits/rejected": -1.8171255588531494,
      "logps/chosen": -277.5356750488281,
      "logps/rejected": -232.9903106689453,
      "loss": 0.5532,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12692232429981232,
      "rewards/margins": 0.6040672063827515,
      "rewards/rejected": -0.7309895753860474,
      "step": 2020
    },
    {
      "epoch": 2.1,
      "learning_rate": 1.6724071947952544e-07,
      "logits/chosen": -2.0031819343566895,
      "logits/rejected": -1.9382264614105225,
      "logps/chosen": -273.3340759277344,
      "logps/rejected": -218.45809936523438,
      "loss": 0.5437,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.13407178223133087,
      "rewards/margins": 0.6485711932182312,
      "rewards/rejected": -0.7826430201530457,
      "step": 2030
    },
    {
      "epoch": 2.11,
      "learning_rate": 1.653272101033295e-07,
      "logits/chosen": -1.94761061668396,
      "logits/rejected": -1.8981701135635376,
      "logps/chosen": -279.50396728515625,
      "logps/rejected": -234.0538330078125,
      "loss": 0.5158,
      "rewards/accuracies": 0.753125011920929,
      "rewards/chosen": -0.09492893517017365,
      "rewards/margins": 0.7014483213424683,
      "rewards/rejected": -0.7963773608207703,
      "step": 2040
    },
    {
      "epoch": 2.12,
      "learning_rate": 1.6341370072713356e-07,
      "logits/chosen": -2.0171449184417725,
      "logits/rejected": -1.953629493713379,
      "logps/chosen": -290.34478759765625,
      "logps/rejected": -250.620849609375,
      "loss": 0.5349,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.11246202141046524,
      "rewards/margins": 0.6265016794204712,
      "rewards/rejected": -0.738963782787323,
      "step": 2050
    },
    {
      "epoch": 2.13,
      "learning_rate": 1.6150019135093762e-07,
      "logits/chosen": -2.0697007179260254,
      "logits/rejected": -1.931609869003296,
      "logps/chosen": -298.30694580078125,
      "logps/rejected": -233.61599731445312,
      "loss": 0.539,
      "rewards/accuracies": 0.7281249761581421,
      "rewards/chosen": -0.08447974920272827,
      "rewards/margins": 0.6712833046913147,
      "rewards/rejected": -0.755763053894043,
      "step": 2060
    },
    {
      "epoch": 2.14,
      "learning_rate": 1.5958668197474169e-07,
      "logits/chosen": -2.0353074073791504,
      "logits/rejected": -1.9606077671051025,
      "logps/chosen": -266.8646545410156,
      "logps/rejected": -252.83901977539062,
      "loss": 0.5429,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.04690312594175339,
      "rewards/margins": 0.5956650376319885,
      "rewards/rejected": -0.6425682306289673,
      "step": 2070
    },
    {
      "epoch": 2.15,
      "learning_rate": 1.5767317259854572e-07,
      "logits/chosen": -1.9791412353515625,
      "logits/rejected": -1.9253734350204468,
      "logps/chosen": -278.138916015625,
      "logps/rejected": -210.71810913085938,
      "loss": 0.5448,
      "rewards/accuracies": 0.7406250238418579,
      "rewards/chosen": -0.008852415718138218,
      "rewards/margins": 0.6631059646606445,
      "rewards/rejected": -0.671958327293396,
      "step": 2080
    },
    {
      "epoch": 2.16,
      "learning_rate": 1.5575966322234978e-07,
      "logits/chosen": -1.9551115036010742,
      "logits/rejected": -1.9077117443084717,
      "logps/chosen": -258.12127685546875,
      "logps/rejected": -231.0892333984375,
      "loss": 0.5313,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.08477520197629929,
      "rewards/margins": 0.6596347093582153,
      "rewards/rejected": -0.74440997838974,
      "step": 2090
    },
    {
      "epoch": 2.17,
      "learning_rate": 1.5384615384615385e-07,
      "logits/chosen": -2.013572931289673,
      "logits/rejected": -1.9160667657852173,
      "logps/chosen": -266.0271911621094,
      "logps/rejected": -214.8003387451172,
      "loss": 0.5301,
      "rewards/accuracies": 0.778124988079071,
      "rewards/chosen": -0.12546849250793457,
      "rewards/margins": 0.7021899223327637,
      "rewards/rejected": -0.827658474445343,
      "step": 2100
    },
    {
      "epoch": 2.18,
      "learning_rate": 1.519326444699579e-07,
      "logits/chosen": -1.9861072301864624,
      "logits/rejected": -1.8711364269256592,
      "logps/chosen": -256.51129150390625,
      "logps/rejected": -210.8562774658203,
      "loss": 0.5306,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.11212404072284698,
      "rewards/margins": 0.6418614387512207,
      "rewards/rejected": -0.7539855241775513,
      "step": 2110
    },
    {
      "epoch": 2.19,
      "learning_rate": 1.5001913509376197e-07,
      "logits/chosen": -2.0319206714630127,
      "logits/rejected": -1.9812395572662354,
      "logps/chosen": -287.5851135253906,
      "logps/rejected": -243.9886932373047,
      "loss": 0.5534,
      "rewards/accuracies": 0.721875011920929,
      "rewards/chosen": -0.15451140701770782,
      "rewards/margins": 0.6129166483879089,
      "rewards/rejected": -0.7674281001091003,
      "step": 2120
    },
    {
      "epoch": 2.2,
      "learning_rate": 1.4810562571756603e-07,
      "logits/chosen": -2.04410719871521,
      "logits/rejected": -1.8800079822540283,
      "logps/chosen": -247.6931610107422,
      "logps/rejected": -227.32296752929688,
      "loss": 0.5552,
      "rewards/accuracies": 0.7281249761581421,
      "rewards/chosen": -0.08454032987356186,
      "rewards/margins": 0.7250841856002808,
      "rewards/rejected": -0.8096244931221008,
      "step": 2130
    },
    {
      "epoch": 2.21,
      "learning_rate": 1.4619211634137007e-07,
      "logits/chosen": -1.9265568256378174,
      "logits/rejected": -1.8220539093017578,
      "logps/chosen": -256.62548828125,
      "logps/rejected": -212.54470825195312,
      "loss": 0.5332,
      "rewards/accuracies": 0.753125011920929,
      "rewards/chosen": -0.101638063788414,
      "rewards/margins": 0.6411160230636597,
      "rewards/rejected": -0.7427541613578796,
      "step": 2140
    },
    {
      "epoch": 2.22,
      "learning_rate": 1.4427860696517413e-07,
      "logits/chosen": -2.0046191215515137,
      "logits/rejected": -1.8799793720245361,
      "logps/chosen": -281.8408508300781,
      "logps/rejected": -235.22567749023438,
      "loss": 0.5653,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.11603154242038727,
      "rewards/margins": 0.5813745260238647,
      "rewards/rejected": -0.6974060535430908,
      "step": 2150
    },
    {
      "epoch": 2.23,
      "learning_rate": 1.423650975889782e-07,
      "logits/chosen": -2.0367672443389893,
      "logits/rejected": -1.9846054315567017,
      "logps/chosen": -287.5448303222656,
      "logps/rejected": -236.52615356445312,
      "loss": 0.5336,
      "rewards/accuracies": 0.746874988079071,
      "rewards/chosen": -0.05621805042028427,
      "rewards/margins": 0.7016788721084595,
      "rewards/rejected": -0.757896900177002,
      "step": 2160
    },
    {
      "epoch": 2.24,
      "learning_rate": 1.4045158821278225e-07,
      "logits/chosen": -1.9967548847198486,
      "logits/rejected": -1.8803222179412842,
      "logps/chosen": -260.0086975097656,
      "logps/rejected": -229.26632690429688,
      "loss": 0.5327,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.12442709505558014,
      "rewards/margins": 0.7182163000106812,
      "rewards/rejected": -0.8426433801651001,
      "step": 2170
    },
    {
      "epoch": 2.25,
      "learning_rate": 1.3853807883658632e-07,
      "logits/chosen": -2.06010103225708,
      "logits/rejected": -1.979443907737732,
      "logps/chosen": -284.96319580078125,
      "logps/rejected": -228.4374237060547,
      "loss": 0.5603,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.12428168207406998,
      "rewards/margins": 0.5993887186050415,
      "rewards/rejected": -0.7236703634262085,
      "step": 2180
    },
    {
      "epoch": 2.26,
      "learning_rate": 1.3662456946039035e-07,
      "logits/chosen": -1.9753601551055908,
      "logits/rejected": -1.9079968929290771,
      "logps/chosen": -248.7126922607422,
      "logps/rejected": -220.49960327148438,
      "loss": 0.5399,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.11765311658382416,
      "rewards/margins": 0.5911535024642944,
      "rewards/rejected": -0.7088066339492798,
      "step": 2190
    },
    {
      "epoch": 2.27,
      "learning_rate": 1.3471106008419441e-07,
      "logits/chosen": -1.9624128341674805,
      "logits/rejected": -1.902761697769165,
      "logps/chosen": -259.12322998046875,
      "logps/rejected": -221.6943359375,
      "loss": 0.5428,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.16727255284786224,
      "rewards/margins": 0.5986199378967285,
      "rewards/rejected": -0.7658926248550415,
      "step": 2200
    },
    {
      "epoch": 2.28,
      "learning_rate": 1.3279755070799848e-07,
      "logits/chosen": -2.0055718421936035,
      "logits/rejected": -1.8739973306655884,
      "logps/chosen": -271.47802734375,
      "logps/rejected": -227.26358032226562,
      "loss": 0.5605,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.1572277843952179,
      "rewards/margins": 0.5724858045578003,
      "rewards/rejected": -0.7297135591506958,
      "step": 2210
    },
    {
      "epoch": 2.29,
      "learning_rate": 1.3088404133180254e-07,
      "logits/chosen": -2.0300581455230713,
      "logits/rejected": -1.9286314249038696,
      "logps/chosen": -301.9215393066406,
      "logps/rejected": -230.28164672851562,
      "loss": 0.4922,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.0035975924693048,
      "rewards/margins": 0.7717592716217041,
      "rewards/rejected": -0.7753568291664124,
      "step": 2220
    },
    {
      "epoch": 2.3,
      "learning_rate": 1.289705319556066e-07,
      "logits/chosen": -1.9785239696502686,
      "logits/rejected": -1.8718605041503906,
      "logps/chosen": -260.2724609375,
      "logps/rejected": -227.3611297607422,
      "loss": 0.5343,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.11053315550088882,
      "rewards/margins": 0.6837419271469116,
      "rewards/rejected": -0.7942751049995422,
      "step": 2230
    },
    {
      "epoch": 2.31,
      "learning_rate": 1.2705702257941064e-07,
      "logits/chosen": -2.042748212814331,
      "logits/rejected": -1.9830291271209717,
      "logps/chosen": -286.2877502441406,
      "logps/rejected": -233.0985870361328,
      "loss": 0.5189,
      "rewards/accuracies": 0.753125011920929,
      "rewards/chosen": -0.12127647548913956,
      "rewards/margins": 0.6687451004981995,
      "rewards/rejected": -0.790021538734436,
      "step": 2240
    },
    {
      "epoch": 2.32,
      "learning_rate": 1.251435132032147e-07,
      "logits/chosen": -2.056126117706299,
      "logits/rejected": -1.927869439125061,
      "logps/chosen": -270.32427978515625,
      "logps/rejected": -233.14218139648438,
      "loss": 0.5377,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.08137933909893036,
      "rewards/margins": 0.6417652368545532,
      "rewards/rejected": -0.7231445908546448,
      "step": 2250
    },
    {
      "epoch": 2.33,
      "learning_rate": 1.2323000382701873e-07,
      "logits/chosen": -2.028714656829834,
      "logits/rejected": -1.9313879013061523,
      "logps/chosen": -268.63214111328125,
      "logps/rejected": -234.97305297851562,
      "loss": 0.5715,
      "rewards/accuracies": 0.715624988079071,
      "rewards/chosen": -0.17626860737800598,
      "rewards/margins": 0.5661368370056152,
      "rewards/rejected": -0.7424054145812988,
      "step": 2260
    },
    {
      "epoch": 2.34,
      "learning_rate": 1.213164944508228e-07,
      "logits/chosen": -2.0209312438964844,
      "logits/rejected": -1.9482758045196533,
      "logps/chosen": -280.43505859375,
      "logps/rejected": -251.8485107421875,
      "loss": 0.5177,
      "rewards/accuracies": 0.746874988079071,
      "rewards/chosen": -0.16871343553066254,
      "rewards/margins": 0.7220279574394226,
      "rewards/rejected": -0.8907413482666016,
      "step": 2270
    },
    {
      "epoch": 2.35,
      "learning_rate": 1.1940298507462686e-07,
      "logits/chosen": -1.9852972030639648,
      "logits/rejected": -1.9430131912231445,
      "logps/chosen": -278.09869384765625,
      "logps/rejected": -217.71533203125,
      "loss": 0.5268,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.1097816452383995,
      "rewards/margins": 0.7086385488510132,
      "rewards/rejected": -0.8184202313423157,
      "step": 2280
    },
    {
      "epoch": 2.37,
      "learning_rate": 1.1748947569843092e-07,
      "logits/chosen": -2.027526378631592,
      "logits/rejected": -1.938901662826538,
      "logps/chosen": -276.50341796875,
      "logps/rejected": -230.8983154296875,
      "loss": 0.5165,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.07388017326593399,
      "rewards/margins": 0.7378325462341309,
      "rewards/rejected": -0.8117126226425171,
      "step": 2290
    },
    {
      "epoch": 2.38,
      "learning_rate": 1.1557596632223497e-07,
      "logits/chosen": -2.0729262828826904,
      "logits/rejected": -1.900803565979004,
      "logps/chosen": -271.7540283203125,
      "logps/rejected": -227.78384399414062,
      "loss": 0.5282,
      "rewards/accuracies": 0.721875011920929,
      "rewards/chosen": -0.07709081470966339,
      "rewards/margins": 0.7070693969726562,
      "rewards/rejected": -0.7841601967811584,
      "step": 2300
    },
    {
      "epoch": 2.39,
      "learning_rate": 1.1366245694603903e-07,
      "logits/chosen": -1.99917471408844,
      "logits/rejected": -1.9048084020614624,
      "logps/chosen": -258.48968505859375,
      "logps/rejected": -220.8121337890625,
      "loss": 0.5524,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.14988255500793457,
      "rewards/margins": 0.5792022943496704,
      "rewards/rejected": -0.729084849357605,
      "step": 2310
    },
    {
      "epoch": 2.4,
      "learning_rate": 1.1174894756984308e-07,
      "logits/chosen": -1.9502952098846436,
      "logits/rejected": -1.9144408702850342,
      "logps/chosen": -243.72299194335938,
      "logps/rejected": -227.6197967529297,
      "loss": 0.528,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.16638337075710297,
      "rewards/margins": 0.5764808654785156,
      "rewards/rejected": -0.7428642511367798,
      "step": 2320
    },
    {
      "epoch": 2.41,
      "learning_rate": 1.0983543819364714e-07,
      "logits/chosen": -2.0321359634399414,
      "logits/rejected": -1.9428303241729736,
      "logps/chosen": -265.5797119140625,
      "logps/rejected": -238.43368530273438,
      "loss": 0.4978,
      "rewards/accuracies": 0.7593749761581421,
      "rewards/chosen": -0.11228694766759872,
      "rewards/margins": 0.6872363090515137,
      "rewards/rejected": -0.7995232343673706,
      "step": 2330
    },
    {
      "epoch": 2.42,
      "learning_rate": 1.079219288174512e-07,
      "logits/chosen": -1.9360284805297852,
      "logits/rejected": -1.8506568670272827,
      "logps/chosen": -245.03762817382812,
      "logps/rejected": -236.2658233642578,
      "loss": 0.5417,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.12498140335083008,
      "rewards/margins": 0.6189488172531128,
      "rewards/rejected": -0.7439301609992981,
      "step": 2340
    },
    {
      "epoch": 2.43,
      "learning_rate": 1.0600841944125525e-07,
      "logits/chosen": -1.9778602123260498,
      "logits/rejected": -1.8423837423324585,
      "logps/chosen": -260.97308349609375,
      "logps/rejected": -221.7749786376953,
      "loss": 0.5554,
      "rewards/accuracies": 0.721875011920929,
      "rewards/chosen": -0.19020505249500275,
      "rewards/margins": 0.5690335631370544,
      "rewards/rejected": -0.759238600730896,
      "step": 2350
    },
    {
      "epoch": 2.44,
      "learning_rate": 1.0409491006505931e-07,
      "logits/chosen": -2.021061897277832,
      "logits/rejected": -1.9514410495758057,
      "logps/chosen": -264.15435791015625,
      "logps/rejected": -229.5314483642578,
      "loss": 0.5351,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.0817098617553711,
      "rewards/margins": 0.7072576880455017,
      "rewards/rejected": -0.7889676094055176,
      "step": 2360
    },
    {
      "epoch": 2.45,
      "learning_rate": 1.0218140068886336e-07,
      "logits/chosen": -1.971008062362671,
      "logits/rejected": -1.8351669311523438,
      "logps/chosen": -272.0447082519531,
      "logps/rejected": -224.9204559326172,
      "loss": 0.5529,
      "rewards/accuracies": 0.7093750238418579,
      "rewards/chosen": -0.10915365070104599,
      "rewards/margins": 0.6357630491256714,
      "rewards/rejected": -0.7449167370796204,
      "step": 2370
    },
    {
      "epoch": 2.46,
      "learning_rate": 1.0026789131266743e-07,
      "logits/chosen": -1.9370200634002686,
      "logits/rejected": -1.8351131677627563,
      "logps/chosen": -282.9969177246094,
      "logps/rejected": -241.0260772705078,
      "loss": 0.5493,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.0709105059504509,
      "rewards/margins": 0.6791272163391113,
      "rewards/rejected": -0.7500376105308533,
      "step": 2380
    },
    {
      "epoch": 2.47,
      "learning_rate": 9.835438193647149e-08,
      "logits/chosen": -1.9906991720199585,
      "logits/rejected": -1.9335302114486694,
      "logps/chosen": -265.366455078125,
      "logps/rejected": -219.4970245361328,
      "loss": 0.5563,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1341801881790161,
      "rewards/margins": 0.6088107824325562,
      "rewards/rejected": -0.7429909706115723,
      "step": 2390
    },
    {
      "epoch": 2.48,
      "learning_rate": 9.644087256027554e-08,
      "logits/chosen": -1.961403250694275,
      "logits/rejected": -1.9385201930999756,
      "logps/chosen": -272.18280029296875,
      "logps/rejected": -234.25277709960938,
      "loss": 0.5334,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.10721113532781601,
      "rewards/margins": 0.6185725927352905,
      "rewards/rejected": -0.7257837057113647,
      "step": 2400
    },
    {
      "epoch": 2.49,
      "learning_rate": 9.45273631840796e-08,
      "logits/chosen": -2.0202138423919678,
      "logits/rejected": -1.9500621557235718,
      "logps/chosen": -258.01361083984375,
      "logps/rejected": -228.53732299804688,
      "loss": 0.5474,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.13280650973320007,
      "rewards/margins": 0.6008110046386719,
      "rewards/rejected": -0.7336176037788391,
      "step": 2410
    },
    {
      "epoch": 2.5,
      "learning_rate": 9.261385380788366e-08,
      "logits/chosen": -1.9771407842636108,
      "logits/rejected": -1.956167459487915,
      "logps/chosen": -261.42169189453125,
      "logps/rejected": -235.85366821289062,
      "loss": 0.5367,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.12852981686592102,
      "rewards/margins": 0.5638247132301331,
      "rewards/rejected": -0.6923545598983765,
      "step": 2420
    },
    {
      "epoch": 2.51,
      "learning_rate": 9.070034443168771e-08,
      "logits/chosen": -2.0048177242279053,
      "logits/rejected": -1.9468837976455688,
      "logps/chosen": -258.286376953125,
      "logps/rejected": -239.4003448486328,
      "loss": 0.557,
      "rewards/accuracies": 0.7406250238418579,
      "rewards/chosen": -0.10692036151885986,
      "rewards/margins": 0.6991198658943176,
      "rewards/rejected": -0.8060401678085327,
      "step": 2430
    },
    {
      "epoch": 2.52,
      "learning_rate": 8.878683505549177e-08,
      "logits/chosen": -2.012662410736084,
      "logits/rejected": -1.9248571395874023,
      "logps/chosen": -260.6874694824219,
      "logps/rejected": -243.2574005126953,
      "loss": 0.5111,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.09256302565336227,
      "rewards/margins": 0.6950146555900574,
      "rewards/rejected": -0.7875776886940002,
      "step": 2440
    },
    {
      "epoch": 2.53,
      "learning_rate": 8.687332567929582e-08,
      "logits/chosen": -1.9357149600982666,
      "logits/rejected": -1.927011489868164,
      "logps/chosen": -264.5352478027344,
      "logps/rejected": -239.3921356201172,
      "loss": 0.5381,
      "rewards/accuracies": 0.753125011920929,
      "rewards/chosen": -0.1247233971953392,
      "rewards/margins": 0.6627745032310486,
      "rewards/rejected": -0.7874979972839355,
      "step": 2450
    },
    {
      "epoch": 2.54,
      "learning_rate": 8.495981630309988e-08,
      "logits/chosen": -1.9526309967041016,
      "logits/rejected": -1.9026234149932861,
      "logps/chosen": -281.9859924316406,
      "logps/rejected": -228.44052124023438,
      "loss": 0.5448,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13007353246212006,
      "rewards/margins": 0.6389485001564026,
      "rewards/rejected": -0.7690221071243286,
      "step": 2460
    },
    {
      "epoch": 2.55,
      "learning_rate": 8.304630692690395e-08,
      "logits/chosen": -2.0521068572998047,
      "logits/rejected": -1.9333775043487549,
      "logps/chosen": -267.8842468261719,
      "logps/rejected": -223.2080535888672,
      "loss": 0.5419,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.09863010793924332,
      "rewards/margins": 0.6975767016410828,
      "rewards/rejected": -0.7962067723274231,
      "step": 2470
    },
    {
      "epoch": 2.56,
      "learning_rate": 8.1132797550708e-08,
      "logits/chosen": -2.065293073654175,
      "logits/rejected": -1.9522087574005127,
      "logps/chosen": -302.3513488769531,
      "logps/rejected": -239.3452606201172,
      "loss": 0.5312,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.041663073003292084,
      "rewards/margins": 0.7064257264137268,
      "rewards/rejected": -0.7480887770652771,
      "step": 2480
    },
    {
      "epoch": 2.57,
      "learning_rate": 7.921928817451206e-08,
      "logits/chosen": -2.0390076637268066,
      "logits/rejected": -1.9214242696762085,
      "logps/chosen": -267.5995788574219,
      "logps/rejected": -236.94271850585938,
      "loss": 0.5194,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10287908464670181,
      "rewards/margins": 0.7216282486915588,
      "rewards/rejected": -0.8245073556900024,
      "step": 2490
    },
    {
      "epoch": 2.58,
      "learning_rate": 7.73057787983161e-08,
      "logits/chosen": -1.987099051475525,
      "logits/rejected": -1.9656906127929688,
      "logps/chosen": -286.71746826171875,
      "logps/rejected": -242.8203887939453,
      "loss": 0.5309,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.08397471904754639,
      "rewards/margins": 0.6902012825012207,
      "rewards/rejected": -0.7741760015487671,
      "step": 2500
    },
    {
      "epoch": 2.59,
      "learning_rate": 7.539226942212017e-08,
      "logits/chosen": -1.9945751428604126,
      "logits/rejected": -1.8857038021087646,
      "logps/chosen": -256.72479248046875,
      "logps/rejected": -226.22207641601562,
      "loss": 0.5227,
      "rewards/accuracies": 0.753125011920929,
      "rewards/chosen": -0.1443193554878235,
      "rewards/margins": 0.7128327488899231,
      "rewards/rejected": -0.8571521639823914,
      "step": 2510
    },
    {
      "epoch": 2.6,
      "learning_rate": 7.347876004592423e-08,
      "logits/chosen": -2.042407512664795,
      "logits/rejected": -1.8979930877685547,
      "logps/chosen": -257.240478515625,
      "logps/rejected": -211.95138549804688,
      "loss": 0.5415,
      "rewards/accuracies": 0.690625011920929,
      "rewards/chosen": -0.10604407638311386,
      "rewards/margins": 0.6722184419631958,
      "rewards/rejected": -0.7782624363899231,
      "step": 2520
    },
    {
      "epoch": 2.61,
      "learning_rate": 7.156525066972828e-08,
      "logits/chosen": -1.9255244731903076,
      "logits/rejected": -1.905170202255249,
      "logps/chosen": -253.9196319580078,
      "logps/rejected": -221.67984008789062,
      "loss": 0.5594,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1309424787759781,
      "rewards/margins": 0.5638185739517212,
      "rewards/rejected": -0.6947609782218933,
      "step": 2530
    },
    {
      "epoch": 2.62,
      "learning_rate": 6.965174129353234e-08,
      "logits/chosen": -1.9832531213760376,
      "logits/rejected": -1.9322866201400757,
      "logps/chosen": -277.56982421875,
      "logps/rejected": -221.8081512451172,
      "loss": 0.5328,
      "rewards/accuracies": 0.7281249761581421,
      "rewards/chosen": -0.05555805563926697,
      "rewards/margins": 0.7537458539009094,
      "rewards/rejected": -0.8093039393424988,
      "step": 2540
    },
    {
      "epoch": 2.63,
      "learning_rate": 6.773823191733639e-08,
      "logits/chosen": -1.9376776218414307,
      "logits/rejected": -1.8852014541625977,
      "logps/chosen": -259.39947509765625,
      "logps/rejected": -222.1279296875,
      "loss": 0.5364,
      "rewards/accuracies": 0.7093750238418579,
      "rewards/chosen": -0.07750894874334335,
      "rewards/margins": 0.7031437754631042,
      "rewards/rejected": -0.780652642250061,
      "step": 2550
    },
    {
      "epoch": 2.64,
      "learning_rate": 6.582472254114045e-08,
      "logits/chosen": -1.945090889930725,
      "logits/rejected": -1.9398300647735596,
      "logps/chosen": -260.34881591796875,
      "logps/rejected": -241.45230102539062,
      "loss": 0.5712,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.19983193278312683,
      "rewards/margins": 0.5203108787536621,
      "rewards/rejected": -0.7201427221298218,
      "step": 2560
    },
    {
      "epoch": 2.65,
      "learning_rate": 6.391121316494451e-08,
      "logits/chosen": -1.9702688455581665,
      "logits/rejected": -1.9331938028335571,
      "logps/chosen": -273.8888854980469,
      "logps/rejected": -226.31692504882812,
      "loss": 0.5264,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.11131913959980011,
      "rewards/margins": 0.6655842065811157,
      "rewards/rejected": -0.7769032716751099,
      "step": 2570
    },
    {
      "epoch": 2.66,
      "learning_rate": 6.199770378874856e-08,
      "logits/chosen": -1.9572299718856812,
      "logits/rejected": -1.9173994064331055,
      "logps/chosen": -266.03692626953125,
      "logps/rejected": -233.6370849609375,
      "loss": 0.5311,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.05556722730398178,
      "rewards/margins": 0.6352362036705017,
      "rewards/rejected": -0.6908034086227417,
      "step": 2580
    },
    {
      "epoch": 2.67,
      "learning_rate": 6.008419441255262e-08,
      "logits/chosen": -1.973695993423462,
      "logits/rejected": -1.9042612314224243,
      "logps/chosen": -282.314208984375,
      "logps/rejected": -224.1729736328125,
      "loss": 0.5393,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.08098714798688889,
      "rewards/margins": 0.6813811659812927,
      "rewards/rejected": -0.762368381023407,
      "step": 2590
    },
    {
      "epoch": 2.69,
      "learning_rate": 5.817068503635668e-08,
      "logits/chosen": -2.041367769241333,
      "logits/rejected": -1.9625844955444336,
      "logps/chosen": -278.90740966796875,
      "logps/rejected": -237.153076171875,
      "loss": 0.5267,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.08425331115722656,
      "rewards/margins": 0.6965413093566895,
      "rewards/rejected": -0.780794620513916,
      "step": 2600
    },
    {
      "epoch": 2.7,
      "learning_rate": 5.6257175660160735e-08,
      "logits/chosen": -1.9530586004257202,
      "logits/rejected": -1.9685173034667969,
      "logps/chosen": -242.7851104736328,
      "logps/rejected": -225.36294555664062,
      "loss": 0.521,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.10606219619512558,
      "rewards/margins": 0.6712923645973206,
      "rewards/rejected": -0.7773545384407043,
      "step": 2610
    },
    {
      "epoch": 2.71,
      "learning_rate": 5.4343666283964784e-08,
      "logits/chosen": -1.9664418697357178,
      "logits/rejected": -1.8895866870880127,
      "logps/chosen": -274.50543212890625,
      "logps/rejected": -228.39794921875,
      "loss": 0.5143,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12829892337322235,
      "rewards/margins": 0.7127763032913208,
      "rewards/rejected": -0.8410751223564148,
      "step": 2620
    },
    {
      "epoch": 2.72,
      "learning_rate": 5.243015690776884e-08,
      "logits/chosen": -1.9619324207305908,
      "logits/rejected": -1.8757219314575195,
      "logps/chosen": -283.93951416015625,
      "logps/rejected": -227.21240234375,
      "loss": 0.525,
      "rewards/accuracies": 0.7281249761581421,
      "rewards/chosen": -0.09419764578342438,
      "rewards/margins": 0.6821730136871338,
      "rewards/rejected": -0.776370644569397,
      "step": 2630
    },
    {
      "epoch": 2.73,
      "learning_rate": 5.05166475315729e-08,
      "logits/chosen": -1.965478539466858,
      "logits/rejected": -1.9349143505096436,
      "logps/chosen": -277.0966796875,
      "logps/rejected": -230.806640625,
      "loss": 0.5439,
      "rewards/accuracies": 0.7093750238418579,
      "rewards/chosen": -0.1608276665210724,
      "rewards/margins": 0.6512233018875122,
      "rewards/rejected": -0.812050998210907,
      "step": 2640
    },
    {
      "epoch": 2.74,
      "learning_rate": 4.860313815537696e-08,
      "logits/chosen": -1.9939587116241455,
      "logits/rejected": -1.9180082082748413,
      "logps/chosen": -264.33380126953125,
      "logps/rejected": -241.66598510742188,
      "loss": 0.5395,
      "rewards/accuracies": 0.784375011920929,
      "rewards/chosen": -0.09295650571584702,
      "rewards/margins": 0.7515963315963745,
      "rewards/rejected": -0.8445528745651245,
      "step": 2650
    },
    {
      "epoch": 2.75,
      "learning_rate": 4.668962877918101e-08,
      "logits/chosen": -1.9551360607147217,
      "logits/rejected": -1.897792100906372,
      "logps/chosen": -246.9778594970703,
      "logps/rejected": -241.43692016601562,
      "loss": 0.5288,
      "rewards/accuracies": 0.690625011920929,
      "rewards/chosen": -0.14015261828899384,
      "rewards/margins": 0.6561464071273804,
      "rewards/rejected": -0.7962990999221802,
      "step": 2660
    },
    {
      "epoch": 2.76,
      "learning_rate": 4.477611940298507e-08,
      "logits/chosen": -2.035179615020752,
      "logits/rejected": -1.9448999166488647,
      "logps/chosen": -280.6395568847656,
      "logps/rejected": -226.279541015625,
      "loss": 0.5099,
      "rewards/accuracies": 0.778124988079071,
      "rewards/chosen": -0.013490036129951477,
      "rewards/margins": 0.8554097414016724,
      "rewards/rejected": -0.8688998222351074,
      "step": 2670
    },
    {
      "epoch": 2.77,
      "learning_rate": 4.2862610026789124e-08,
      "logits/chosen": -2.105760097503662,
      "logits/rejected": -1.9209930896759033,
      "logps/chosen": -263.80010986328125,
      "logps/rejected": -233.33126831054688,
      "loss": 0.5576,
      "rewards/accuracies": 0.715624988079071,
      "rewards/chosen": -0.13011452555656433,
      "rewards/margins": 0.6017405390739441,
      "rewards/rejected": -0.731855034828186,
      "step": 2680
    },
    {
      "epoch": 2.78,
      "learning_rate": 4.0949100650593186e-08,
      "logits/chosen": -1.9066162109375,
      "logits/rejected": -1.8646472692489624,
      "logps/chosen": -257.67529296875,
      "logps/rejected": -236.4352569580078,
      "loss": 0.5459,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17842397093772888,
      "rewards/margins": 0.5756587982177734,
      "rewards/rejected": -0.7540827393531799,
      "step": 2690
    },
    {
      "epoch": 2.79,
      "learning_rate": 3.903559127439724e-08,
      "logits/chosen": -2.0057380199432373,
      "logits/rejected": -1.9169094562530518,
      "logps/chosen": -273.3626403808594,
      "logps/rejected": -229.9825439453125,
      "loss": 0.5079,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.052403002977371216,
      "rewards/margins": 0.7288587689399719,
      "rewards/rejected": -0.7812617421150208,
      "step": 2700
    },
    {
      "epoch": 2.8,
      "learning_rate": 3.71220818982013e-08,
      "logits/chosen": -2.037144899368286,
      "logits/rejected": -1.9049354791641235,
      "logps/chosen": -272.304443359375,
      "logps/rejected": -238.1118621826172,
      "loss": 0.5517,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.2206374704837799,
      "rewards/margins": 0.542033314704895,
      "rewards/rejected": -0.7626706957817078,
      "step": 2710
    },
    {
      "epoch": 2.81,
      "learning_rate": 3.520857252200535e-08,
      "logits/chosen": -1.9005661010742188,
      "logits/rejected": -1.87981379032135,
      "logps/chosen": -263.9569091796875,
      "logps/rejected": -239.2077178955078,
      "loss": 0.539,
      "rewards/accuracies": 0.715624988079071,
      "rewards/chosen": -0.11894295364618301,
      "rewards/margins": 0.5795784592628479,
      "rewards/rejected": -0.6985214352607727,
      "step": 2720
    },
    {
      "epoch": 2.82,
      "learning_rate": 3.3295063145809414e-08,
      "logits/chosen": -1.9351590871810913,
      "logits/rejected": -1.9006996154785156,
      "logps/chosen": -270.80377197265625,
      "logps/rejected": -246.0286102294922,
      "loss": 0.5474,
      "rewards/accuracies": 0.778124988079071,
      "rewards/chosen": -0.13235792517662048,
      "rewards/margins": 0.6689977645874023,
      "rewards/rejected": -0.8013556599617004,
      "step": 2730
    },
    {
      "epoch": 2.83,
      "learning_rate": 3.138155376961347e-08,
      "logits/chosen": -2.017043113708496,
      "logits/rejected": -1.929476022720337,
      "logps/chosen": -285.06182861328125,
      "logps/rejected": -244.6565399169922,
      "loss": 0.5623,
      "rewards/accuracies": 0.746874988079071,
      "rewards/chosen": -0.10070965439081192,
      "rewards/margins": 0.6912192702293396,
      "rewards/rejected": -0.7919288873672485,
      "step": 2740
    },
    {
      "epoch": 2.84,
      "learning_rate": 2.9468044393417525e-08,
      "logits/chosen": -1.9757921695709229,
      "logits/rejected": -1.8634798526763916,
      "logps/chosen": -261.4624938964844,
      "logps/rejected": -230.32046508789062,
      "loss": 0.5245,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.07247430086135864,
      "rewards/margins": 0.7779797315597534,
      "rewards/rejected": -0.8504541516304016,
      "step": 2750
    },
    {
      "epoch": 2.85,
      "learning_rate": 2.755453501722158e-08,
      "logits/chosen": -1.9504101276397705,
      "logits/rejected": -1.9742431640625,
      "logps/chosen": -264.0960998535156,
      "logps/rejected": -229.517822265625,
      "loss": 0.5135,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.09173519909381866,
      "rewards/margins": 0.7189399003982544,
      "rewards/rejected": -0.8106750249862671,
      "step": 2760
    },
    {
      "epoch": 2.86,
      "learning_rate": 2.564102564102564e-08,
      "logits/chosen": -2.043729305267334,
      "logits/rejected": -1.9805526733398438,
      "logps/chosen": -265.21710205078125,
      "logps/rejected": -245.47158813476562,
      "loss": 0.5323,
      "rewards/accuracies": 0.721875011920929,
      "rewards/chosen": -0.1146395206451416,
      "rewards/margins": 0.673359751701355,
      "rewards/rejected": -0.7879992723464966,
      "step": 2770
    },
    {
      "epoch": 2.87,
      "learning_rate": 2.3727516264829695e-08,
      "logits/chosen": -2.017853021621704,
      "logits/rejected": -1.9051122665405273,
      "logps/chosen": -283.5233459472656,
      "logps/rejected": -237.43875122070312,
      "loss": 0.5349,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.0835433155298233,
      "rewards/margins": 0.6877850294113159,
      "rewards/rejected": -0.7713284492492676,
      "step": 2780
    },
    {
      "epoch": 2.88,
      "learning_rate": 2.1814006888633754e-08,
      "logits/chosen": -1.9762601852416992,
      "logits/rejected": -1.9173065423965454,
      "logps/chosen": -271.5351257324219,
      "logps/rejected": -227.7848663330078,
      "loss": 0.5481,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.10751036554574966,
      "rewards/margins": 0.6145915389060974,
      "rewards/rejected": -0.7221018671989441,
      "step": 2790
    },
    {
      "epoch": 2.89,
      "learning_rate": 1.990049751243781e-08,
      "logits/chosen": -2.0152127742767334,
      "logits/rejected": -1.894903540611267,
      "logps/chosen": -277.72955322265625,
      "logps/rejected": -226.2139434814453,
      "loss": 0.5381,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.09227684885263443,
      "rewards/margins": 0.6984345316886902,
      "rewards/rejected": -0.7907114028930664,
      "step": 2800
    },
    {
      "epoch": 2.9,
      "learning_rate": 1.7986988136241865e-08,
      "logits/chosen": -1.8761422634124756,
      "logits/rejected": -1.8549368381500244,
      "logps/chosen": -274.33721923828125,
      "logps/rejected": -242.8546600341797,
      "loss": 0.5018,
      "rewards/accuracies": 0.7593749761581421,
      "rewards/chosen": -0.08238744735717773,
      "rewards/margins": 0.722454309463501,
      "rewards/rejected": -0.8048418164253235,
      "step": 2810
    },
    {
      "epoch": 2.91,
      "learning_rate": 1.6073478760045924e-08,
      "logits/chosen": -1.9757686853408813,
      "logits/rejected": -1.8756663799285889,
      "logps/chosen": -282.96954345703125,
      "logps/rejected": -235.4677734375,
      "loss": 0.5485,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.16583535075187683,
      "rewards/margins": 0.7119665145874023,
      "rewards/rejected": -0.877801775932312,
      "step": 2820
    },
    {
      "epoch": 2.92,
      "learning_rate": 1.4159969383849981e-08,
      "logits/chosen": -1.966698408126831,
      "logits/rejected": -1.959288239479065,
      "logps/chosen": -271.11260986328125,
      "logps/rejected": -223.1236572265625,
      "loss": 0.5281,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.08993484824895859,
      "rewards/margins": 0.6874836087226868,
      "rewards/rejected": -0.7774184346199036,
      "step": 2830
    },
    {
      "epoch": 2.93,
      "learning_rate": 1.2246460007654037e-08,
      "logits/chosen": -1.9788850545883179,
      "logits/rejected": -1.9086517095565796,
      "logps/chosen": -266.74066162109375,
      "logps/rejected": -227.94088745117188,
      "loss": 0.5735,
      "rewards/accuracies": 0.659375011920929,
      "rewards/chosen": -0.18117864429950714,
      "rewards/margins": 0.4861881136894226,
      "rewards/rejected": -0.6673667430877686,
      "step": 2840
    },
    {
      "epoch": 2.94,
      "learning_rate": 1.0332950631458094e-08,
      "logits/chosen": -1.937133550643921,
      "logits/rejected": -1.9378074407577515,
      "logps/chosen": -265.6098937988281,
      "logps/rejected": -227.0642547607422,
      "loss": 0.5439,
      "rewards/accuracies": 0.7281249761581421,
      "rewards/chosen": -0.0965404137969017,
      "rewards/margins": 0.6471257209777832,
      "rewards/rejected": -0.7436660528182983,
      "step": 2850
    },
    {
      "epoch": 2.95,
      "learning_rate": 8.419441255262151e-09,
      "logits/chosen": -1.9160661697387695,
      "logits/rejected": -1.9068663120269775,
      "logps/chosen": -261.45025634765625,
      "logps/rejected": -236.23446655273438,
      "loss": 0.5177,
      "rewards/accuracies": 0.753125011920929,
      "rewards/chosen": -0.1480577141046524,
      "rewards/margins": 0.7233497500419617,
      "rewards/rejected": -0.8714075088500977,
      "step": 2860
    },
    {
      "epoch": 2.96,
      "learning_rate": 6.505931879066207e-09,
      "logits/chosen": -1.9042994976043701,
      "logits/rejected": -1.9673118591308594,
      "logps/chosen": -255.11904907226562,
      "logps/rejected": -217.8560333251953,
      "loss": 0.5419,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2056998312473297,
      "rewards/margins": 0.5817626714706421,
      "rewards/rejected": -0.7874624729156494,
      "step": 2870
    },
    {
      "epoch": 2.97,
      "learning_rate": 4.592422502870264e-09,
      "logits/chosen": -2.018409013748169,
      "logits/rejected": -1.9491459131240845,
      "logps/chosen": -264.1302185058594,
      "logps/rejected": -239.7041015625,
      "loss": 0.5506,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.13123063743114471,
      "rewards/margins": 0.5559999346733093,
      "rewards/rejected": -0.6872305870056152,
      "step": 2880
    },
    {
      "epoch": 2.98,
      "learning_rate": 2.6789131266743202e-09,
      "logits/chosen": -1.965376853942871,
      "logits/rejected": -1.8747295141220093,
      "logps/chosen": -270.21197509765625,
      "logps/rejected": -235.61495971679688,
      "loss": 0.5223,
      "rewards/accuracies": 0.7593749761581421,
      "rewards/chosen": -0.11224790662527084,
      "rewards/margins": 0.7009861469268799,
      "rewards/rejected": -0.8132340312004089,
      "step": 2890
    },
    {
      "epoch": 3.0,
      "learning_rate": 7.654037504783773e-10,
      "logits/chosen": -1.9493694305419922,
      "logits/rejected": -1.9246546030044556,
      "logps/chosen": -252.29348754882812,
      "logps/rejected": -249.4759521484375,
      "loss": 0.5463,
      "rewards/accuracies": 0.7406250238418579,
      "rewards/chosen": -0.1315115988254547,
      "rewards/margins": 0.6227953433990479,
      "rewards/rejected": -0.754306972026825,
      "step": 2900
    },
    {
      "epoch": 3.0,
      "eval_logits/chosen": -1.9102067947387695,
      "eval_logits/rejected": -1.8216071128845215,
      "eval_logps/chosen": -283.6626281738281,
      "eval_logps/rejected": -237.35032653808594,
      "eval_loss": 0.5380748510360718,
      "eval_rewards/accuracies": 0.7182539701461792,
      "eval_rewards/chosen": -0.09773208945989609,
      "eval_rewards/margins": 0.6768320798873901,
      "eval_rewards/rejected": -0.7745641469955444,
      "eval_runtime": 872.6678,
      "eval_samples_per_second": 2.292,
      "eval_steps_per_second": 0.072,
      "step": 2904
    },
    {
      "epoch": 3.0,
      "step": 2904,
      "total_flos": 0.0,
      "train_loss": 0.5707186016333662,
      "train_runtime": 142661.7128,
      "train_samples_per_second": 1.303,
      "train_steps_per_second": 0.02
    }
  ],
  "logging_steps": 10,
  "max_steps": 2904,
  "num_train_epochs": 3,
  "save_steps": 500,
  "total_flos": 0.0,
  "trial_name": null,
  "trial_params": null
}