{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9997382884061764,
  "eval_steps": 500,
  "global_step": 955,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 5.208333333333333e-08,
      "logits/chosen": 0.654504120349884,
      "logits/rejected": 0.5926104187965393,
      "logps/chosen": -305.37469482421875,
      "logps/rejected": -235.55426025390625,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "learning_rate": 5.208333333333334e-07,
      "logits/chosen": 0.7002869248390198,
      "logits/rejected": 0.7474254965782166,
      "logps/chosen": -312.4855041503906,
      "logps/rejected": -270.3090515136719,
      "loss": 0.6928,
      "rewards/accuracies": 0.4722222089767456,
      "rewards/chosen": 0.0012019119458273053,
      "rewards/margins": 0.0008854579646140337,
      "rewards/rejected": 0.00031645383569411933,
      "step": 10
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.0416666666666667e-06,
      "logits/chosen": 0.6109224557876587,
      "logits/rejected": 0.7636935114860535,
      "logps/chosen": -229.7316436767578,
      "logps/rejected": -217.67245483398438,
      "loss": 0.6928,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0013416050933301449,
      "rewards/margins": 0.0012069231597706676,
      "rewards/rejected": 0.00013468178804032505,
      "step": 20
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.5625e-06,
      "logits/chosen": 0.6081576347351074,
      "logits/rejected": 0.65868079662323,
      "logps/chosen": -286.92681884765625,
      "logps/rejected": -243.32778930664062,
      "loss": 0.6929,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00032840127823874354,
      "rewards/margins": 0.0011824481189250946,
      "rewards/rejected": -0.0008540467242710292,
      "step": 30
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.0833333333333334e-06,
      "logits/chosen": 0.616665244102478,
      "logits/rejected": 0.735335648059845,
      "logps/chosen": -291.98284912109375,
      "logps/rejected": -265.10198974609375,
      "loss": 0.6926,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.0013727883342653513,
      "rewards/margins": 0.0006782080745324492,
      "rewards/rejected": -0.0020509962923824787,
      "step": 40
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.604166666666667e-06,
      "logits/chosen": 0.6475738286972046,
      "logits/rejected": 0.660605788230896,
      "logps/chosen": -276.0094299316406,
      "logps/rejected": -254.7918701171875,
      "loss": 0.6923,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -1.2021581824228633e-05,
      "rewards/margins": 0.0021027675829827785,
      "rewards/rejected": -0.002114789094775915,
      "step": 50
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.125e-06,
      "logits/chosen": 0.6682679057121277,
      "logits/rejected": 0.7323259115219116,
      "logps/chosen": -308.230224609375,
      "logps/rejected": -253.301025390625,
      "loss": 0.6917,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.0026237894780933857,
      "rewards/margins": 0.003060306655243039,
      "rewards/rejected": -0.005684095900505781,
      "step": 60
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.6458333333333333e-06,
      "logits/chosen": 0.6251315474510193,
      "logits/rejected": 0.6695118546485901,
      "logps/chosen": -296.9325866699219,
      "logps/rejected": -265.0625915527344,
      "loss": 0.6898,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.003425169037654996,
      "rewards/margins": 0.0062147388234734535,
      "rewards/rejected": -0.009639907628297806,
      "step": 70
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.166666666666667e-06,
      "logits/chosen": 0.6303352117538452,
      "logits/rejected": 0.6176969408988953,
      "logps/chosen": -296.875244140625,
      "logps/rejected": -259.2256774902344,
      "loss": 0.6875,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.011222435161471367,
      "rewards/margins": 0.009960432536900043,
      "rewards/rejected": -0.021182868629693985,
      "step": 80
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.6875000000000004e-06,
      "logits/chosen": 0.6014409065246582,
      "logits/rejected": 0.6135669946670532,
      "logps/chosen": -271.5030212402344,
      "logps/rejected": -246.84310913085938,
      "loss": 0.6859,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.019970973953604698,
      "rewards/margins": 0.015956712886691093,
      "rewards/rejected": -0.03592768684029579,
      "step": 90
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.9997324926814375e-06,
      "logits/chosen": 0.5811746716499329,
      "logits/rejected": 0.582471489906311,
      "logps/chosen": -257.50079345703125,
      "logps/rejected": -245.5070343017578,
      "loss": 0.6848,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.025636669248342514,
      "rewards/margins": 0.01930178329348564,
      "rewards/rejected": -0.044938452541828156,
      "step": 100
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.996723692767927e-06,
      "logits/chosen": 0.6834483742713928,
      "logits/rejected": 0.7622531056404114,
      "logps/chosen": -267.14837646484375,
      "logps/rejected": -244.15005493164062,
      "loss": 0.6816,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.042797841131687164,
      "rewards/margins": 0.023977002128958702,
      "rewards/rejected": -0.06677485257387161,
      "step": 110
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.9903757462135984e-06,
      "logits/chosen": 0.5502853393554688,
      "logits/rejected": 0.6681184768676758,
      "logps/chosen": -267.02532958984375,
      "logps/rejected": -238.77914428710938,
      "loss": 0.6792,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.05984887480735779,
      "rewards/margins": 0.038902174681425095,
      "rewards/rejected": -0.09875104576349258,
      "step": 120
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.980697142834315e-06,
      "logits/chosen": 0.5389119982719421,
      "logits/rejected": 0.6768661737442017,
      "logps/chosen": -308.8938903808594,
      "logps/rejected": -267.13861083984375,
      "loss": 0.6768,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.08925649523735046,
      "rewards/margins": 0.032300472259521484,
      "rewards/rejected": -0.12155697494745255,
      "step": 130
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.967700826904229e-06,
      "logits/chosen": 0.5784076452255249,
      "logits/rejected": 0.6090500950813293,
      "logps/chosen": -268.8511657714844,
      "logps/rejected": -252.6071319580078,
      "loss": 0.6733,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.12314645200967789,
      "rewards/margins": 0.0378914549946785,
      "rewards/rejected": -0.1610378921031952,
      "step": 140
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.951404179843963e-06,
      "logits/chosen": 0.6251226663589478,
      "logits/rejected": 0.6390718817710876,
      "logps/chosen": -277.3585510253906,
      "logps/rejected": -255.3101043701172,
      "loss": 0.6781,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.14656075835227966,
      "rewards/margins": 0.036974769085645676,
      "rewards/rejected": -0.18353553116321564,
      "step": 150
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.931828996974498e-06,
      "logits/chosen": 0.6863027811050415,
      "logits/rejected": 0.5657976865768433,
      "logps/chosen": -279.9803161621094,
      "logps/rejected": -273.37109375,
      "loss": 0.6634,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.22472751140594482,
      "rewards/margins": 0.05435310676693916,
      "rewards/rejected": -0.2790805995464325,
      "step": 160
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.909001458367867e-06,
      "logits/chosen": 0.6299469470977783,
      "logits/rejected": 0.656846284866333,
      "logps/chosen": -332.79541015625,
      "logps/rejected": -292.52850341796875,
      "loss": 0.6598,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.3016909658908844,
      "rewards/margins": 0.08058367669582367,
      "rewards/rejected": -0.38227468729019165,
      "step": 170
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.882952093833628e-06,
      "logits/chosen": 0.5225847363471985,
      "logits/rejected": 0.6088179349899292,
      "logps/chosen": -289.1798095703125,
      "logps/rejected": -311.8951110839844,
      "loss": 0.652,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.43765193223953247,
      "rewards/margins": 0.09067411720752716,
      "rewards/rejected": -0.5283260345458984,
      "step": 180
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.853715742087947e-06,
      "logits/chosen": 0.5354307293891907,
      "logits/rejected": 0.652572512626648,
      "logps/chosen": -359.98126220703125,
      "logps/rejected": -309.4668273925781,
      "loss": 0.6449,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.5696258544921875,
      "rewards/margins": 0.17413343489170074,
      "rewards/rejected": -0.7437592148780823,
      "step": 190
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.821331504159906e-06,
      "logits/chosen": 0.5104254484176636,
      "logits/rejected": 0.49786561727523804,
      "logps/chosen": -336.1935729980469,
      "logps/rejected": -350.4376220703125,
      "loss": 0.6426,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.7382306456565857,
      "rewards/margins": 0.15078046917915344,
      "rewards/rejected": -0.8890112042427063,
      "step": 200
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.7858426910973435e-06,
      "logits/chosen": 0.4257640838623047,
      "logits/rejected": 0.47642454504966736,
      "logps/chosen": -387.05242919921875,
      "logps/rejected": -345.6732177734375,
      "loss": 0.6381,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.8003188967704773,
      "rewards/margins": 0.19792547821998596,
      "rewards/rejected": -0.9982444047927856,
      "step": 210
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.747296766042161e-06,
      "logits/chosen": 0.4104679226875305,
      "logits/rejected": 0.4869573712348938,
      "logps/chosen": -348.92156982421875,
      "logps/rejected": -350.780029296875,
      "loss": 0.637,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.8022894859313965,
      "rewards/margins": 0.19004520773887634,
      "rewards/rejected": -0.9923346638679504,
      "step": 220
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.705745280752586e-06,
      "logits/chosen": 0.3983922600746155,
      "logits/rejected": 0.42386817932128906,
      "logps/chosen": -388.9286804199219,
      "logps/rejected": -352.74444580078125,
      "loss": 0.6239,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.8833669424057007,
      "rewards/margins": 0.20530752837657928,
      "rewards/rejected": -1.088674545288086,
      "step": 230
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.661243806657256e-06,
      "logits/chosen": 0.38030871748924255,
      "logits/rejected": 0.3976936936378479,
      "logps/chosen": -378.4253234863281,
      "logps/rejected": -352.67388916015625,
      "loss": 0.6149,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.9681804776191711,
      "rewards/margins": 0.27472421526908875,
      "rewards/rejected": -1.2429049015045166,
      "step": 240
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.613851860533367e-06,
      "logits/chosen": 0.29627329111099243,
      "logits/rejected": 0.30808278918266296,
      "logps/chosen": -370.7823791503906,
      "logps/rejected": -365.70843505859375,
      "loss": 0.6213,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.9412585496902466,
      "rewards/margins": 0.25268539786338806,
      "rewards/rejected": -1.193943977355957,
      "step": 250
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.563632824908252e-06,
      "logits/chosen": 0.22838203608989716,
      "logits/rejected": 0.2758544087409973,
      "logps/chosen": -370.5462341308594,
      "logps/rejected": -384.4304504394531,
      "loss": 0.6205,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.1226102113723755,
      "rewards/margins": 0.22477027773857117,
      "rewards/rejected": -1.347380518913269,
      "step": 260
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.510653863290871e-06,
      "logits/chosen": 0.16573426127433777,
      "logits/rejected": 0.24319115281105042,
      "logps/chosen": -432.40496826171875,
      "logps/rejected": -434.70465087890625,
      "loss": 0.6065,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.3656351566314697,
      "rewards/margins": 0.3202405571937561,
      "rewards/rejected": -1.6858758926391602,
      "step": 270
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.454985830346574e-06,
      "logits/chosen": 0.15168949961662292,
      "logits/rejected": 0.2517026662826538,
      "logps/chosen": -496.7152404785156,
      "logps/rejected": -481.1207580566406,
      "loss": 0.6244,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.8755683898925781,
      "rewards/margins": 0.3297354280948639,
      "rewards/rejected": -2.20530366897583,
      "step": 280
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.396703177135262e-06,
      "logits/chosen": 0.1856023520231247,
      "logits/rejected": 0.21857495605945587,
      "logps/chosen": -468.2583923339844,
      "logps/rejected": -422.20477294921875,
      "loss": 0.6089,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -1.88265061378479,
      "rewards/margins": 0.23467198014259338,
      "rewards/rejected": -2.1173224449157715,
      "step": 290
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.335883851539693e-06,
      "logits/chosen": 0.10853853076696396,
      "logits/rejected": 0.2359129637479782,
      "logps/chosen": -437.02569580078125,
      "logps/rejected": -435.40606689453125,
      "loss": 0.5821,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.7075822353363037,
      "rewards/margins": 0.393388569355011,
      "rewards/rejected": -2.10097074508667,
      "step": 300
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.2726091940171055e-06,
      "logits/chosen": 0.12634363770484924,
      "logits/rejected": 0.224656343460083,
      "logps/chosen": -436.81683349609375,
      "logps/rejected": -466.3431091308594,
      "loss": 0.5697,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.6430351734161377,
      "rewards/margins": 0.4527590870857239,
      "rewards/rejected": -2.095794200897217,
      "step": 310
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.206963828813555e-06,
      "logits/chosen": 0.1670975536108017,
      "logits/rejected": 0.20784397423267365,
      "logps/chosen": -454.04400634765625,
      "logps/rejected": -471.0421447753906,
      "loss": 0.5959,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.7891740798950195,
      "rewards/margins": 0.4047388434410095,
      "rewards/rejected": -2.193913221359253,
      "step": 320
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.139035550786495e-06,
      "logits/chosen": 0.13126584887504578,
      "logits/rejected": 0.1421881467103958,
      "logps/chosen": -436.185546875,
      "logps/rejected": -425.6346740722656,
      "loss": 0.621,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.7343782186508179,
      "rewards/margins": 0.29571670293807983,
      "rewards/rejected": -2.030094861984253,
      "step": 330
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.068915207986931e-06,
      "logits/chosen": 0.18622621893882751,
      "logits/rejected": 0.2519204020500183,
      "logps/chosen": -453.08856201171875,
      "logps/rejected": -439.3407287597656,
      "loss": 0.63,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -1.8599331378936768,
      "rewards/margins": 0.25747692584991455,
      "rewards/rejected": -2.117410182952881,
      "step": 340
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.996696580158211e-06,
      "logits/chosen": 0.15227162837982178,
      "logits/rejected": 0.23813900351524353,
      "logps/chosen": -502.65252685546875,
      "logps/rejected": -521.7779541015625,
      "loss": 0.5979,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.993268609046936,
      "rewards/margins": 0.44951528310775757,
      "rewards/rejected": -2.442783832550049,
      "step": 350
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.922476253313921e-06,
      "logits/chosen": 0.07589183002710342,
      "logits/rejected": 0.1402803361415863,
      "logps/chosen": -492.88824462890625,
      "logps/rejected": -508.8631286621094,
      "loss": 0.5971,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -2.206943988800049,
      "rewards/margins": 0.2886379361152649,
      "rewards/rejected": -2.495581865310669,
      "step": 360
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.846353490562664e-06,
      "logits/chosen": 0.20907124876976013,
      "logits/rejected": 0.17944853007793427,
      "logps/chosen": -460.7980041503906,
      "logps/rejected": -506.36151123046875,
      "loss": 0.5709,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.9685118198394775,
      "rewards/margins": 0.47328275442123413,
      "rewards/rejected": -2.4417946338653564,
      "step": 370
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.768430099352445e-06,
      "logits/chosen": 0.16458635032176971,
      "logits/rejected": 0.22938723862171173,
      "logps/chosen": -461.98712158203125,
      "logps/rejected": -475.66046142578125,
      "loss": 0.5929,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.9674451351165771,
      "rewards/margins": 0.38506731390953064,
      "rewards/rejected": -2.3525123596191406,
      "step": 380
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6888102953122307e-06,
      "logits/chosen": 0.06095918267965317,
      "logits/rejected": 0.1491270363330841,
      "logps/chosen": -450.43798828125,
      "logps/rejected": -455.1705017089844,
      "loss": 0.6117,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.9474515914916992,
      "rewards/margins": 0.4293443560600281,
      "rewards/rejected": -2.376796007156372,
      "step": 390
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.607600562872785e-06,
      "logits/chosen": 0.09890508651733398,
      "logits/rejected": 0.1348509043455124,
      "logps/chosen": -522.7869873046875,
      "logps/rejected": -504.12469482421875,
      "loss": 0.5751,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -2.1772594451904297,
      "rewards/margins": 0.4235813617706299,
      "rewards/rejected": -2.6008408069610596,
      "step": 400
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5249095128531863e-06,
      "logits/chosen": 0.15617763996124268,
      "logits/rejected": 0.2346019446849823,
      "logps/chosen": -525.8595581054688,
      "logps/rejected": -515.14599609375,
      "loss": 0.6036,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.276054859161377,
      "rewards/margins": 0.37463003396987915,
      "rewards/rejected": -2.6506848335266113,
      "step": 410
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4408477372034743e-06,
      "logits/chosen": 0.057582952082157135,
      "logits/rejected": 0.13709139823913574,
      "logps/chosen": -447.7400817871094,
      "logps/rejected": -488.5953063964844,
      "loss": 0.5871,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -2.0702292919158936,
      "rewards/margins": 0.3465236723423004,
      "rewards/rejected": -2.4167532920837402,
      "step": 420
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.355527661097728e-06,
      "logits/chosen": 0.10504597425460815,
      "logits/rejected": 0.18081213533878326,
      "logps/chosen": -476.1048278808594,
      "logps/rejected": -492.5294494628906,
      "loss": 0.6166,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -2.1964662075042725,
      "rewards/margins": 0.29873451590538025,
      "rewards/rejected": -2.4952008724212646,
      "step": 430
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.269063392575352e-06,
      "logits/chosen": 0.11929740011692047,
      "logits/rejected": 0.15031102299690247,
      "logps/chosen": -505.28363037109375,
      "logps/rejected": -516.3770751953125,
      "loss": 0.6019,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -2.265862226486206,
      "rewards/margins": 0.365284264087677,
      "rewards/rejected": -2.6311464309692383,
      "step": 440
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.181570569931697e-06,
      "logits/chosen": 0.10959301143884659,
      "logits/rejected": 0.15382876992225647,
      "logps/chosen": -448.1092224121094,
      "logps/rejected": -466.357421875,
      "loss": 0.6033,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -2.1950180530548096,
      "rewards/margins": 0.3157437741756439,
      "rewards/rejected": -2.5107619762420654,
      "step": 450
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.09316620706208e-06,
      "logits/chosen": 0.0773458257317543,
      "logits/rejected": 0.18486995995044708,
      "logps/chosen": -508.8724670410156,
      "logps/rejected": -497.22882080078125,
      "loss": 0.585,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -2.2086341381073,
      "rewards/margins": 0.32577189803123474,
      "rewards/rejected": -2.5344061851501465,
      "step": 460
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.0039685369660785e-06,
      "logits/chosen": 0.09328463673591614,
      "logits/rejected": 0.20468135178089142,
      "logps/chosen": -522.9307861328125,
      "logps/rejected": -512.7855834960938,
      "loss": 0.5807,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -2.516700506210327,
      "rewards/margins": 0.32140296697616577,
      "rewards/rejected": -2.8381035327911377,
      "step": 470
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.91409685362137e-06,
      "logits/chosen": 0.1576906144618988,
      "logits/rejected": 0.10724158585071564,
      "logps/chosen": -456.68359375,
      "logps/rejected": -451.1465759277344,
      "loss": 0.6305,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -2.26971697807312,
      "rewards/margins": 0.18658225238323212,
      "rewards/rejected": -2.456299304962158,
      "step": 480
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8236713524386085e-06,
      "logits/chosen": 0.1447945386171341,
      "logits/rejected": 0.1760699599981308,
      "logps/chosen": -467.67889404296875,
      "logps/rejected": -484.5015563964844,
      "loss": 0.5847,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.1700217723846436,
      "rewards/margins": 0.4066441059112549,
      "rewards/rejected": -2.5766658782958984,
      "step": 490
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7328129695107205e-06,
      "logits/chosen": 0.09771154820919037,
      "logits/rejected": 0.21410468220710754,
      "logps/chosen": -505.1236877441406,
      "logps/rejected": -521.1968994140625,
      "loss": 0.5852,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -2.445769786834717,
      "rewards/margins": 0.5605564117431641,
      "rewards/rejected": -3.00632643699646,
      "step": 500
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.641643219871597e-06,
      "logits/chosen": 0.0740189403295517,
      "logits/rejected": 0.10937446355819702,
      "logps/chosen": -575.5813598632812,
      "logps/rejected": -578.539306640625,
      "loss": 0.5878,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -3.042142152786255,
      "rewards/margins": 0.41739901900291443,
      "rewards/rejected": -3.4595413208007812,
      "step": 510
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5502840349805074e-06,
      "logits/chosen": 0.13737405836582184,
      "logits/rejected": 0.1366177350282669,
      "logps/chosen": -577.433837890625,
      "logps/rejected": -586.6515502929688,
      "loss": 0.6018,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.9127590656280518,
      "rewards/margins": 0.4386274814605713,
      "rewards/rejected": -3.351386308670044,
      "step": 520
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.4588575996495797e-06,
      "logits/chosen": 0.009027734398841858,
      "logits/rejected": 0.15047816932201385,
      "logps/chosen": -578.1231689453125,
      "logps/rejected": -593.139404296875,
      "loss": 0.584,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -2.788166046142578,
      "rewards/margins": 0.4355267882347107,
      "rewards/rejected": -3.2236924171447754,
      "step": 530
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.367486188632446e-06,
      "logits/chosen": 0.12746676802635193,
      "logits/rejected": 0.15015694499015808,
      "logps/chosen": -549.587890625,
      "logps/rejected": -567.0057373046875,
      "loss": 0.5919,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -2.6341264247894287,
      "rewards/margins": 0.4422709047794342,
      "rewards/rejected": -3.07639741897583,
      "step": 540
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.276292003092593e-06,
      "logits/chosen": 0.06688917428255081,
      "logits/rejected": 0.1280103623867035,
      "logps/chosen": -534.5496215820312,
      "logps/rejected": -526.1148071289062,
      "loss": 0.6045,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -2.5599446296691895,
      "rewards/margins": 0.3815520107746124,
      "rewards/rejected": -2.9414966106414795,
      "step": 550
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1853970071701415e-06,
      "logits/chosen": 0.05697401612997055,
      "logits/rejected": 0.15568815171718597,
      "logps/chosen": -523.0081787109375,
      "logps/rejected": -505.5555114746094,
      "loss": 0.5914,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -2.705841541290283,
      "rewards/margins": 0.403696209192276,
      "rewards/rejected": -3.1095376014709473,
      "step": 560
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0949227648656194e-06,
      "logits/chosen": 0.08012814819812775,
      "logits/rejected": 0.1830509603023529,
      "logps/chosen": -562.3424682617188,
      "logps/rejected": -572.2830200195312,
      "loss": 0.6085,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.9859776496887207,
      "rewards/margins": 0.37897008657455444,
      "rewards/rejected": -3.364948272705078,
      "step": 570
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.00499027745888e-06,
      "logits/chosen": 0.13961026072502136,
      "logits/rejected": 0.18276450037956238,
      "logps/chosen": -562.8177490234375,
      "logps/rejected": -579.6517333984375,
      "loss": 0.6114,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.9555232524871826,
      "rewards/margins": 0.380531370639801,
      "rewards/rejected": -3.336055040359497,
      "step": 580
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.915719821680624e-06,
      "logits/chosen": 0.10515383630990982,
      "logits/rejected": 0.12738659977912903,
      "logps/chosen": -508.9317932128906,
      "logps/rejected": -502.7018127441406,
      "loss": 0.5984,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -2.51983904838562,
      "rewards/margins": 0.23314905166625977,
      "rewards/rejected": -2.752988338470459,
      "step": 590
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8272307888529276e-06,
      "logits/chosen": 0.1120181530714035,
      "logits/rejected": 0.18100281059741974,
      "logps/chosen": -511.30645751953125,
      "logps/rejected": -542.5186767578125,
      "loss": 0.5837,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.2869346141815186,
      "rewards/margins": 0.4697488248348236,
      "rewards/rejected": -2.756683111190796,
      "step": 600
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.739641525213929e-06,
      "logits/chosen": 0.13541504740715027,
      "logits/rejected": 0.1992477923631668,
      "logps/chosen": -526.412841796875,
      "logps/rejected": -498.51318359375,
      "loss": 0.5803,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -2.441995143890381,
      "rewards/margins": 0.4095155596733093,
      "rewards/rejected": -2.851510524749756,
      "step": 610
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6530691736402317e-06,
      "logits/chosen": 0.11898718029260635,
      "logits/rejected": 0.11196433007717133,
      "logps/chosen": -530.7737426757812,
      "logps/rejected": -540.7352294921875,
      "loss": 0.5947,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -2.539415121078491,
      "rewards/margins": 0.4597281813621521,
      "rewards/rejected": -2.999143362045288,
      "step": 620
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5676295169786864e-06,
      "logits/chosen": 0.12535694241523743,
      "logits/rejected": 0.2773950397968292,
      "logps/chosen": -558.296630859375,
      "logps/rejected": -556.3736572265625,
      "loss": 0.5937,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -2.703909158706665,
      "rewards/margins": 0.4821166396141052,
      "rewards/rejected": -3.186025857925415,
      "step": 630
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4834368231970922e-06,
      "logits/chosen": 0.13473114371299744,
      "logits/rejected": 0.18143931031227112,
      "logps/chosen": -497.7254333496094,
      "logps/rejected": -533.7047119140625,
      "loss": 0.5819,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.562814235687256,
      "rewards/margins": 0.5369300842285156,
      "rewards/rejected": -3.0997443199157715,
      "step": 640
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4006036925609245e-06,
      "logits/chosen": 0.051880162209272385,
      "logits/rejected": 0.20864169299602509,
      "logps/chosen": -551.8402099609375,
      "logps/rejected": -567.7971801757812,
      "loss": 0.5719,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -2.570906162261963,
      "rewards/margins": 0.5435839295387268,
      "rewards/rejected": -3.114490270614624,
      "step": 650
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3192409070404582e-06,
      "logits/chosen": 0.12426358461380005,
      "logits/rejected": 0.08786197006702423,
      "logps/chosen": -543.6322021484375,
      "logps/rejected": -535.1307373046875,
      "loss": 0.596,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -2.673713445663452,
      "rewards/margins": 0.34852334856987,
      "rewards/rejected": -3.0222370624542236,
      "step": 660
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2394572821496953e-06,
      "logits/chosen": 0.07642360031604767,
      "logits/rejected": 0.1105254516005516,
      "logps/chosen": -521.8109130859375,
      "logps/rejected": -533.8194580078125,
      "loss": 0.5713,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -2.5358338356018066,
      "rewards/margins": 0.5434619188308716,
      "rewards/rejected": -3.0792956352233887,
      "step": 670
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1613595214152713e-06,
      "logits/chosen": 0.07207540422677994,
      "logits/rejected": 0.1519361436367035,
      "logps/chosen": -538.2649536132812,
      "logps/rejected": -527.5178833007812,
      "loss": 0.6009,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.5930490493774414,
      "rewards/margins": 0.24494171142578125,
      "rewards/rejected": -2.8379909992218018,
      "step": 680
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.0850520736699362e-06,
      "logits/chosen": 0.09551219642162323,
      "logits/rejected": 0.20561787486076355,
      "logps/chosen": -522.4210815429688,
      "logps/rejected": -531.2017822265625,
      "loss": 0.5918,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.5857510566711426,
      "rewards/margins": 0.47004079818725586,
      "rewards/rejected": -3.0557916164398193,
      "step": 690
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0106369933615043e-06,
      "logits/chosen": 0.0834534615278244,
      "logits/rejected": 0.27686116099357605,
      "logps/chosen": -509.4612731933594,
      "logps/rejected": -507.83258056640625,
      "loss": 0.5759,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -2.489625930786133,
      "rewards/margins": 0.5027724504470825,
      "rewards/rejected": -2.992398262023926,
      "step": 700
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.382138040640714e-07,
      "logits/chosen": 0.08181905746459961,
      "logits/rejected": 0.10011023283004761,
      "logps/chosen": -566.5830078125,
      "logps/rejected": -552.9130249023438,
      "loss": 0.5783,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -2.718501091003418,
      "rewards/margins": 0.40024662017822266,
      "rewards/rejected": -3.1187477111816406,
      "step": 710
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.678793653740633e-07,
      "logits/chosen": 0.0735187977552414,
      "logits/rejected": 0.13505037128925323,
      "logps/chosen": -513.2781372070312,
      "logps/rejected": -523.3275146484375,
      "loss": 0.5936,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -2.5609078407287598,
      "rewards/margins": 0.3891633450984955,
      "rewards/rejected": -2.9500718116760254,
      "step": 720
    },
    {
      "epoch": 0.76,
      "learning_rate": 7.997277433690984e-07,
      "logits/chosen": 0.06112724542617798,
      "logits/rejected": 0.0807592123746872,
      "logps/chosen": -558.6268310546875,
      "logps/rejected": -531.7501220703125,
      "loss": 0.5879,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -2.575448989868164,
      "rewards/margins": 0.3756105601787567,
      "rewards/rejected": -2.951059341430664,
      "step": 730
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.338500848029603e-07,
      "logits/chosen": 0.005916979629546404,
      "logits/rejected": 0.17464521527290344,
      "logps/chosen": -569.9360961914062,
      "logps/rejected": -577.1893310546875,
      "loss": 0.5867,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -2.632016181945801,
      "rewards/margins": 0.4428483545780182,
      "rewards/rejected": -3.074864387512207,
      "step": 740
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.70334495204884e-07,
      "logits/chosen": 0.1314985156059265,
      "logits/rejected": 0.16822925209999084,
      "logps/chosen": -500.46636962890625,
      "logps/rejected": -552.1305541992188,
      "loss": 0.5901,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -2.497035503387451,
      "rewards/margins": 0.5148124694824219,
      "rewards/rejected": -3.011847972869873,
      "step": 750
    },
    {
      "epoch": 0.8,
      "learning_rate": 6.092659210462232e-07,
      "logits/chosen": 0.052326373755931854,
      "logits/rejected": 0.16787569224834442,
      "logps/chosen": -557.7818603515625,
      "logps/rejected": -567.0888061523438,
      "loss": 0.5689,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -2.623483657836914,
      "rewards/margins": 0.46536707878112793,
      "rewards/rejected": -3.088850736618042,
      "step": 760
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.507260361320738e-07,
      "logits/chosen": 0.08203858882188797,
      "logits/rejected": 0.14716270565986633,
      "logps/chosen": -574.7208862304688,
      "logps/rejected": -586.5050659179688,
      "loss": 0.5999,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -2.7186100482940674,
      "rewards/margins": 0.42362919449806213,
      "rewards/rejected": -3.1422390937805176,
      "step": 770
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.947931323697983e-07,
      "logits/chosen": 0.10785499960184097,
      "logits/rejected": 0.18369507789611816,
      "logps/chosen": -542.8053588867188,
      "logps/rejected": -549.775146484375,
      "loss": 0.5772,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -2.621422529220581,
      "rewards/margins": 0.4870278239250183,
      "rewards/rejected": -3.108450412750244,
      "step": 780
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.4154201506053985e-07,
      "logits/chosen": 0.11180669069290161,
      "logits/rejected": 0.14754420518875122,
      "logps/chosen": -532.26123046875,
      "logps/rejected": -550.2486572265625,
      "loss": 0.6016,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.573822021484375,
      "rewards/margins": 0.3509921431541443,
      "rewards/rejected": -2.924813985824585,
      "step": 790
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.910439028537638e-07,
      "logits/chosen": 0.10717664659023285,
      "logits/rejected": 0.24534320831298828,
      "logps/chosen": -510.2649841308594,
      "logps/rejected": -519.7151489257812,
      "loss": 0.562,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -2.3766260147094727,
      "rewards/margins": 0.4747850000858307,
      "rewards/rejected": -2.8514111042022705,
      "step": 800
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.4336633249862084e-07,
      "logits/chosen": 0.07645310461521149,
      "logits/rejected": 0.17721445858478546,
      "logps/chosen": -552.8776245117188,
      "logps/rejected": -514.6980590820312,
      "loss": 0.5707,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.620849847793579,
      "rewards/margins": 0.33325469493865967,
      "rewards/rejected": -2.95410418510437,
      "step": 810
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.98573068519539e-07,
      "logits/chosen": 0.04426421970129013,
      "logits/rejected": 0.21165044605731964,
      "logps/chosen": -541.8455810546875,
      "logps/rejected": -530.4864501953125,
      "loss": 0.5871,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.5347847938537598,
      "rewards/margins": 0.558411180973053,
      "rewards/rejected": -3.093196153640747,
      "step": 820
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.5672401793681854e-07,
      "logits/chosen": 0.11617372184991837,
      "logits/rejected": 0.16498759388923645,
      "logps/chosen": -526.9368286132812,
      "logps/rejected": -577.3372802734375,
      "loss": 0.568,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -2.5850746631622314,
      "rewards/margins": 0.4706156253814697,
      "rewards/rejected": -3.055690050125122,
      "step": 830
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.178751501463036e-07,
      "logits/chosen": 0.06383179128170013,
      "logits/rejected": 0.056464362889528275,
      "logps/chosen": -508.4523010253906,
      "logps/rejected": -521.2445068359375,
      "loss": 0.6203,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.663949728012085,
      "rewards/margins": 0.27986857295036316,
      "rewards/rejected": -2.9438185691833496,
      "step": 840
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.820784220652766e-07,
      "logits/chosen": 0.09964722394943237,
      "logits/rejected": 0.19454631209373474,
      "logps/chosen": -553.30517578125,
      "logps/rejected": -551.3551635742188,
      "loss": 0.5899,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -2.6559367179870605,
      "rewards/margins": 0.4735372066497803,
      "rewards/rejected": -3.12947416305542,
      "step": 850
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.4938170864468636e-07,
      "logits/chosen": 0.05580083280801773,
      "logits/rejected": 0.1297743022441864,
      "logps/chosen": -535.3272705078125,
      "logps/rejected": -541.6774291992188,
      "loss": 0.6136,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -2.696950912475586,
      "rewards/margins": 0.3545316159725189,
      "rewards/rejected": -3.0514824390411377,
      "step": 860
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1982873884064466e-07,
      "logits/chosen": 0.04851340129971504,
      "logits/rejected": 0.13393761217594147,
      "logps/chosen": -480.47125244140625,
      "logps/rejected": -523.5387573242188,
      "loss": 0.592,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.486987590789795,
      "rewards/margins": 0.4138285219669342,
      "rewards/rejected": -2.900815963745117,
      "step": 870
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.345903713082305e-08,
      "logits/chosen": 0.04031291604042053,
      "logits/rejected": 0.15236200392246246,
      "logps/chosen": -546.6204833984375,
      "logps/rejected": -551.7069702148438,
      "loss": 0.6008,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -2.5852303504943848,
      "rewards/margins": 0.3579770624637604,
      "rewards/rejected": -2.943207263946533,
      "step": 880
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.030787065396866e-08,
      "logits/chosen": 0.13291895389556885,
      "logits/rejected": 0.15924690663814545,
      "logps/chosen": -508.44775390625,
      "logps/rejected": -528.5171508789062,
      "loss": 0.6118,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -2.3992390632629395,
      "rewards/margins": 0.3830375373363495,
      "rewards/rejected": -2.7822766304016113,
      "step": 890
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.0406202043228604e-08,
      "logits/chosen": 0.09626724570989609,
      "logits/rejected": 0.17059974372386932,
      "logps/chosen": -527.4398193359375,
      "logps/rejected": -531.8056640625,
      "loss": 0.6108,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.479921817779541,
      "rewards/margins": 0.4598274827003479,
      "rewards/rejected": -2.9397494792938232,
      "step": 900
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.378064801637687e-08,
      "logits/chosen": 0.02929285541176796,
      "logits/rejected": 0.18183095753192902,
      "logps/chosen": -556.4517822265625,
      "logps/rejected": -552.9277954101562,
      "loss": 0.5773,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -2.6010327339172363,
      "rewards/margins": 0.4194592535495758,
      "rewards/rejected": -3.0204920768737793,
      "step": 910
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.0453443778310766e-08,
      "logits/chosen": 0.1071668416261673,
      "logits/rejected": 0.20206472277641296,
      "logps/chosen": -557.05419921875,
      "logps/rejected": -545.5590209960938,
      "loss": 0.5713,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -2.5374794006347656,
      "rewards/margins": 0.5199281573295593,
      "rewards/rejected": -3.057407855987549,
      "step": 920
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.0442413283435759e-08,
      "logits/chosen": 0.07009036839008331,
      "logits/rejected": 0.16552026569843292,
      "logps/chosen": -491.12237548828125,
      "logps/rejected": -496.7190856933594,
      "loss": 0.569,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -2.3404624462127686,
      "rewards/margins": 0.5322117209434509,
      "rewards/rejected": -2.8726742267608643,
      "step": 930
    },
    {
      "epoch": 0.98,
      "learning_rate": 3.760945397705828e-09,
      "logits/chosen": 0.007874694652855396,
      "logits/rejected": 0.0997551903128624,
      "logps/chosen": -552.2734985351562,
      "logps/rejected": -549.2908935546875,
      "loss": 0.5796,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -2.5978238582611084,
      "rewards/margins": 0.399721622467041,
      "rewards/rejected": -2.9975457191467285,
      "step": 940
    },
    {
      "epoch": 0.99,
      "learning_rate": 4.1797599220405605e-10,
      "logits/chosen": 0.1123199313879013,
      "logits/rejected": 0.08204374462366104,
      "logps/chosen": -510.1936950683594,
      "logps/rejected": -531.1867065429688,
      "loss": 0.5859,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -2.560084819793701,
      "rewards/margins": 0.4064672589302063,
      "rewards/rejected": -2.9665520191192627,
      "step": 950
    },
    {
      "epoch": 1.0,
      "step": 955,
      "total_flos": 0.0,
      "train_loss": 0.0333975282639109,
      "train_runtime": 873.8284,
      "train_samples_per_second": 69.962,
      "train_steps_per_second": 1.093
    }
  ],
  "logging_steps": 10,
  "max_steps": 955,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}