{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9994666666666666,
  "eval_steps": 500,
  "global_step": 937,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 5.319148936170213e-08,
      "logits/chosen": 0.06159799173474312,
      "logits/rejected": 0.2503473162651062,
      "logps/chosen": -348.4295349121094,
      "logps/rejected": -269.23321533203125,
      "loss": 0.4013,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "learning_rate": 5.319148936170213e-07,
      "logits/chosen": 0.050236668437719345,
      "logits/rejected": 0.23283202946186066,
      "logps/chosen": -357.2872314453125,
      "logps/rejected": -284.6590270996094,
      "loss": 0.3533,
      "rewards/accuracies": 0.3958333432674408,
      "rewards/chosen": -9.200325621350203e-06,
      "rewards/margins": -3.7092313505127095e-06,
      "rewards/rejected": -5.491095635079546e-06,
      "step": 10
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.0638297872340427e-06,
      "logits/chosen": 0.07863714545965195,
      "logits/rejected": 0.17622847855091095,
      "logps/chosen": -305.1004638671875,
      "logps/rejected": -284.56573486328125,
      "loss": 0.3668,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -3.269618537160568e-05,
      "rewards/margins": -7.148426084313542e-05,
      "rewards/rejected": 3.878808274748735e-05,
      "step": 20
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.595744680851064e-06,
      "logits/chosen": 0.10170497745275497,
      "logits/rejected": 0.2674924433231354,
      "logps/chosen": -318.61309814453125,
      "logps/rejected": -289.11529541015625,
      "loss": 0.3582,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -2.7896196570509346e-06,
      "rewards/margins": -0.00010077792830998078,
      "rewards/rejected": 9.798832616070285e-05,
      "step": 30
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.1276595744680853e-06,
      "logits/chosen": 0.09615897387266159,
      "logits/rejected": 0.2696045935153961,
      "logps/chosen": -371.0738220214844,
      "logps/rejected": -285.8421325683594,
      "loss": 0.3338,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 4.211660780129023e-05,
      "rewards/margins": 0.0002511069178581238,
      "rewards/rejected": -0.00020899029914289713,
      "step": 40
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.6595744680851065e-06,
      "logits/chosen": 0.1058117151260376,
      "logits/rejected": 0.2665451169013977,
      "logps/chosen": -365.52001953125,
      "logps/rejected": -291.03778076171875,
      "loss": 0.3245,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0004745240439660847,
      "rewards/margins": 0.0008677773294039071,
      "rewards/rejected": -0.00039325328543782234,
      "step": 50
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.191489361702128e-06,
      "logits/chosen": 0.07869451493024826,
      "logits/rejected": 0.2602466642856598,
      "logps/chosen": -347.3607482910156,
      "logps/rejected": -290.67364501953125,
      "loss": 0.3335,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 0.0010413106065243483,
      "rewards/margins": 0.0015250012511387467,
      "rewards/rejected": -0.00048369058640673757,
      "step": 60
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.723404255319149e-06,
      "logits/chosen": 0.12400275468826294,
      "logits/rejected": 0.26277071237564087,
      "logps/chosen": -336.801025390625,
      "logps/rejected": -295.81439208984375,
      "loss": 0.3541,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.0017572289798408747,
      "rewards/margins": 0.002515274565666914,
      "rewards/rejected": -0.0007580459350720048,
      "step": 70
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.255319148936171e-06,
      "logits/chosen": 0.06505224108695984,
      "logits/rejected": 0.18910573422908783,
      "logps/chosen": -312.43511962890625,
      "logps/rejected": -274.01617431640625,
      "loss": 0.3528,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.003332150634378195,
      "rewards/margins": 0.004707558546215296,
      "rewards/rejected": -0.0013754077954217792,
      "step": 80
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.787234042553192e-06,
      "logits/chosen": 0.09570185840129852,
      "logits/rejected": 0.292137086391449,
      "logps/chosen": -320.4757080078125,
      "logps/rejected": -282.39453125,
      "loss": 0.3233,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.005724198184907436,
      "rewards/margins": 0.00849069096148014,
      "rewards/rejected": -0.002766493707895279,
      "step": 90
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.999375059004058e-06,
      "logits/chosen": 0.11051477491855621,
      "logits/rejected": 0.2542819380760193,
      "logps/chosen": -341.4449768066406,
      "logps/rejected": -303.7613220214844,
      "loss": 0.2954,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.006659829057753086,
      "rewards/margins": 0.017101887613534927,
      "rewards/rejected": -0.010442061349749565,
      "step": 100
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.9955571065548795e-06,
      "logits/chosen": 0.09644671529531479,
      "logits/rejected": 0.26117372512817383,
      "logps/chosen": -347.67059326171875,
      "logps/rejected": -339.35205078125,
      "loss": 0.3068,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.005562308244407177,
      "rewards/margins": 0.025120720267295837,
      "rewards/rejected": -0.03068302944302559,
      "step": 110
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.9882736864879e-06,
      "logits/chosen": 0.1643160879611969,
      "logits/rejected": 0.33352339267730713,
      "logps/chosen": -393.2250061035156,
      "logps/rejected": -346.75457763671875,
      "loss": 0.3154,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.026022305712103844,
      "rewards/margins": 0.04141021519899368,
      "rewards/rejected": -0.06743250787258148,
      "step": 120
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.977534912960124e-06,
      "logits/chosen": 0.14933325350284576,
      "logits/rejected": 0.36655381321907043,
      "logps/chosen": -425.4112243652344,
      "logps/rejected": -428.3099670410156,
      "loss": 0.2788,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.07423840463161469,
      "rewards/margins": 0.06771700084209442,
      "rewards/rejected": -0.1419553905725479,
      "step": 130
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.963355698422092e-06,
      "logits/chosen": 0.20979711413383484,
      "logits/rejected": 0.2947987914085388,
      "logps/chosen": -455.8238830566406,
      "logps/rejected": -478.470703125,
      "loss": 0.2889,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.11944756656885147,
      "rewards/margins": 0.08143159002065659,
      "rewards/rejected": -0.20087914168834686,
      "step": 140
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.945755732909625e-06,
      "logits/chosen": 0.22968748211860657,
      "logits/rejected": 0.3435000777244568,
      "logps/chosen": -494.1380920410156,
      "logps/rejected": -557.603271484375,
      "loss": 0.2274,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.15503188967704773,
      "rewards/margins": 0.12964466214179993,
      "rewards/rejected": -0.28467655181884766,
      "step": 150
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.924759456701167e-06,
      "logits/chosen": 0.2052285373210907,
      "logits/rejected": 0.3779992461204529,
      "logps/chosen": -555.717529296875,
      "logps/rejected": -618.7412719726562,
      "loss": 0.2945,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.22048434615135193,
      "rewards/margins": 0.12210027873516083,
      "rewards/rejected": -0.34258466958999634,
      "step": 160
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.900396026378671e-06,
      "logits/chosen": 0.19485141336917877,
      "logits/rejected": 0.35038071870803833,
      "logps/chosen": -564.8143310546875,
      "logps/rejected": -614.8881225585938,
      "loss": 0.2459,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.19514423608779907,
      "rewards/margins": 0.13954147696495056,
      "rewards/rejected": -0.33468568325042725,
      "step": 170
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.872699274339169e-06,
      "logits/chosen": 0.26888132095336914,
      "logits/rejected": 0.38210389018058777,
      "logps/chosen": -542.71875,
      "logps/rejected": -610.9818115234375,
      "loss": 0.2668,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.20494337379932404,
      "rewards/margins": 0.13571330904960632,
      "rewards/rejected": -0.34065669775009155,
      "step": 180
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.8417076618132434e-06,
      "logits/chosen": 0.27069562673568726,
      "logits/rejected": 0.3432765603065491,
      "logps/chosen": -552.3577270507812,
      "logps/rejected": -633.7227783203125,
      "loss": 0.2295,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.20474235713481903,
      "rewards/margins": 0.14836545288562775,
      "rewards/rejected": -0.35310783982276917,
      "step": 190
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.807464225455655e-06,
      "logits/chosen": 0.2959613800048828,
      "logits/rejected": 0.4290657639503479,
      "logps/chosen": -607.9371337890625,
      "logps/rejected": -658.5770263671875,
      "loss": 0.2489,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.22546908259391785,
      "rewards/margins": 0.14855849742889404,
      "rewards/rejected": -0.3740275502204895,
      "step": 200
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.770016517582283e-06,
      "logits/chosen": 0.315132200717926,
      "logits/rejected": 0.43885716795921326,
      "logps/chosen": -636.2320556640625,
      "logps/rejected": -777.607421875,
      "loss": 0.2412,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.28788623213768005,
      "rewards/margins": 0.1952207386493683,
      "rewards/rejected": -0.4831070005893707,
      "step": 210
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.7294165401363616e-06,
      "logits/chosen": 0.27698999643325806,
      "logits/rejected": 0.468730628490448,
      "logps/chosen": -641.8867797851562,
      "logps/rejected": -759.454833984375,
      "loss": 0.2575,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.26833561062812805,
      "rewards/margins": 0.1899726390838623,
      "rewards/rejected": -0.45830821990966797,
      "step": 220
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.68572067247573e-06,
      "logits/chosen": 0.2822573781013489,
      "logits/rejected": 0.42682987451553345,
      "logps/chosen": -578.1446533203125,
      "logps/rejected": -648.0927734375,
      "loss": 0.2443,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.19278240203857422,
      "rewards/margins": 0.18077878654003143,
      "rewards/rejected": -0.37356117367744446,
      "step": 230
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.638989593081364e-06,
      "logits/chosen": 0.3316207230091095,
      "logits/rejected": 0.42460235953330994,
      "logps/chosen": -508.28924560546875,
      "logps/rejected": -627.9178466796875,
      "loss": 0.2394,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18461547791957855,
      "rewards/margins": 0.17850570380687714,
      "rewards/rejected": -0.3631211817264557,
      "step": 240
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.5892881952959015e-06,
      "logits/chosen": 0.34463202953338623,
      "logits/rejected": 0.4213656485080719,
      "logps/chosen": -602.0831909179688,
      "logps/rejected": -723.1624145507812,
      "loss": 0.2229,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.22979848086833954,
      "rewards/margins": 0.1861880123615265,
      "rewards/rejected": -0.41598647832870483,
      "step": 250
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.536685497209182e-06,
      "logits/chosen": 0.2841508984565735,
      "logits/rejected": 0.4096981883049011,
      "logps/chosen": -649.3954467773438,
      "logps/rejected": -714.6544189453125,
      "loss": 0.263,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.26575830578804016,
      "rewards/margins": 0.15293100476264954,
      "rewards/rejected": -0.4186893105506897,
      "step": 260
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.481254545815943e-06,
      "logits/chosen": 0.33302605152130127,
      "logits/rejected": 0.4252694249153137,
      "logps/chosen": -652.48828125,
      "logps/rejected": -734.8269653320312,
      "loss": 0.2375,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.2703024744987488,
      "rewards/margins": 0.16575393080711365,
      "rewards/rejected": -0.43605637550354004,
      "step": 270
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.42307231557875e-06,
      "logits/chosen": 0.2720186710357666,
      "logits/rejected": 0.4189075827598572,
      "logps/chosen": -606.5438232421875,
      "logps/rejected": -711.4382934570312,
      "loss": 0.234,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.2561277151107788,
      "rewards/margins": 0.1838252693414688,
      "rewards/rejected": -0.4399529993534088,
      "step": 280
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.3622196015370305e-06,
      "logits/chosen": 0.2856540381908417,
      "logits/rejected": 0.46640047430992126,
      "logps/chosen": -581.0446166992188,
      "logps/rejected": -699.5716552734375,
      "loss": 0.2034,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.22450003027915955,
      "rewards/margins": 0.187502920627594,
      "rewards/rejected": -0.41200295090675354,
      "step": 290
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.298780907110648e-06,
      "logits/chosen": 0.30591830611228943,
      "logits/rejected": 0.4262414574623108,
      "logps/chosen": -547.7359619140625,
      "logps/rejected": -688.0494995117188,
      "loss": 0.2329,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.2417764961719513,
      "rewards/margins": 0.18638503551483154,
      "rewards/rejected": -0.42816153168678284,
      "step": 300
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.23284432675381e-06,
      "logits/chosen": 0.2821730375289917,
      "logits/rejected": 0.4414825439453125,
      "logps/chosen": -625.08984375,
      "logps/rejected": -736.9857177734375,
      "loss": 0.2392,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.29913753271102905,
      "rewards/margins": 0.1723407357931137,
      "rewards/rejected": -0.4714782238006592,
      "step": 310
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.164501423622277e-06,
      "logits/chosen": 0.3146980404853821,
      "logits/rejected": 0.5349381566047668,
      "logps/chosen": -667.1527099609375,
      "logps/rejected": -809.9267578125,
      "loss": 0.2347,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.3215456008911133,
      "rewards/margins": 0.2190614491701126,
      "rewards/rejected": -0.5406070947647095,
      "step": 320
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.0938471024237355e-06,
      "logits/chosen": 0.3459940552711487,
      "logits/rejected": 0.5182437300682068,
      "logps/chosen": -632.4589233398438,
      "logps/rejected": -797.7081298828125,
      "loss": 0.1764,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.2630694508552551,
      "rewards/margins": 0.2305426150560379,
      "rewards/rejected": -0.49361199140548706,
      "step": 330
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.020979477627907e-06,
      "logits/chosen": 0.3081711232662201,
      "logits/rejected": 0.3925454020500183,
      "logps/chosen": -592.3121948242188,
      "logps/rejected": -695.0133666992188,
      "loss": 0.2528,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.2648671865463257,
      "rewards/margins": 0.17098167538642883,
      "rewards/rejected": -0.4358488619327545,
      "step": 340
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.9459997372194105e-06,
      "logits/chosen": 0.3076136112213135,
      "logits/rejected": 0.38813453912734985,
      "logps/chosen": -642.5826416015625,
      "logps/rejected": -815.0631103515625,
      "loss": 0.1942,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.31470799446105957,
      "rewards/margins": 0.20841769874095917,
      "rewards/rejected": -0.5231257677078247,
      "step": 350
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.869012002182573e-06,
      "logits/chosen": 0.2800385057926178,
      "logits/rejected": 0.45763272047042847,
      "logps/chosen": -611.0481567382812,
      "logps/rejected": -790.9743041992188,
      "loss": 0.1966,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.275074303150177,
      "rewards/margins": 0.22857530415058136,
      "rewards/rejected": -0.5036496520042419,
      "step": 360
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.7901231819133104e-06,
      "logits/chosen": 0.2838382422924042,
      "logits/rejected": 0.4253064692020416,
      "logps/chosen": -663.5225830078125,
      "logps/rejected": -790.7160034179688,
      "loss": 0.2077,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.28215542435646057,
      "rewards/margins": 0.2109585702419281,
      "rewards/rejected": -0.49311399459838867,
      "step": 370
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.709442825758875e-06,
      "logits/chosen": 0.32005783915519714,
      "logits/rejected": 0.42395836114883423,
      "logps/chosen": -589.5636596679688,
      "logps/rejected": -722.5950317382812,
      "loss": 0.2375,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.24356071650981903,
      "rewards/margins": 0.18706892430782318,
      "rewards/rejected": -0.4306296408176422,
      "step": 380
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.6270829708916113e-06,
      "logits/chosen": 0.318645715713501,
      "logits/rejected": 0.4325946271419525,
      "logps/chosen": -611.2227783203125,
      "logps/rejected": -759.0226440429688,
      "loss": 0.2232,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.25262197852134705,
      "rewards/margins": 0.19418981671333313,
      "rewards/rejected": -0.44681182503700256,
      "step": 390
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.543157986727991e-06,
      "logits/chosen": 0.27690261602401733,
      "logits/rejected": 0.4690285325050354,
      "logps/chosen": -600.1328125,
      "logps/rejected": -722.5906982421875,
      "loss": 0.2403,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.2763115167617798,
      "rewards/margins": 0.16976875066757202,
      "rewards/rejected": -0.4460802674293518,
      "step": 400
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4577844161089614e-06,
      "logits/chosen": 0.2978890836238861,
      "logits/rejected": 0.5168454647064209,
      "logps/chosen": -631.8878173828125,
      "logps/rejected": -754.962646484375,
      "loss": 0.2448,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.3049059808254242,
      "rewards/margins": 0.16670657694339752,
      "rewards/rejected": -0.4716125428676605,
      "step": 410
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3710808134621577e-06,
      "logits/chosen": 0.31090012192726135,
      "logits/rejected": 0.4086151123046875,
      "logps/chosen": -639.5802612304688,
      "logps/rejected": -715.4155883789062,
      "loss": 0.2345,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.29435452818870544,
      "rewards/margins": 0.1740979552268982,
      "rewards/rejected": -0.46845245361328125,
      "step": 420
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2831675801707126e-06,
      "logits/chosen": 0.29356813430786133,
      "logits/rejected": 0.45901551842689514,
      "logps/chosen": -648.76806640625,
      "logps/rejected": -793.9247436523438,
      "loss": 0.195,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.30719709396362305,
      "rewards/margins": 0.2049371302127838,
      "rewards/rejected": -0.5121342539787292,
      "step": 430
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.194166797377289e-06,
      "logits/chosen": 0.35314708948135376,
      "logits/rejected": 0.4861513078212738,
      "logps/chosen": -669.8665771484375,
      "logps/rejected": -808.8753051757812,
      "loss": 0.2386,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.34487032890319824,
      "rewards/margins": 0.20408621430397034,
      "rewards/rejected": -0.548956573009491,
      "step": 440
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.104202056455501e-06,
      "logits/chosen": 0.3531484007835388,
      "logits/rejected": 0.5174172520637512,
      "logps/chosen": -657.7909545898438,
      "logps/rejected": -806.8094482421875,
      "loss": 0.2039,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.3043491244316101,
      "rewards/margins": 0.21579626202583313,
      "rewards/rejected": -0.5201453566551208,
      "step": 450
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.013398287384144e-06,
      "logits/chosen": 0.29440468549728394,
      "logits/rejected": 0.4629596769809723,
      "logps/chosen": -613.6539916992188,
      "logps/rejected": -718.9657592773438,
      "loss": 0.2514,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.28395384550094604,
      "rewards/margins": 0.17435352504253387,
      "rewards/rejected": -0.4583073556423187,
      "step": 460
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9218815852625717e-06,
      "logits/chosen": 0.26766303181648254,
      "logits/rejected": 0.4210576117038727,
      "logps/chosen": -611.0421142578125,
      "logps/rejected": -752.10546875,
      "loss": 0.2478,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.2685486972332001,
      "rewards/margins": 0.20156879723072052,
      "rewards/rejected": -0.4701175093650818,
      "step": 470
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.829779035208113e-06,
      "logits/chosen": 0.3156279921531677,
      "logits/rejected": 0.4234594702720642,
      "logps/chosen": -583.9177856445312,
      "logps/rejected": -695.6286010742188,
      "loss": 0.2687,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.2727667987346649,
      "rewards/margins": 0.16848285496234894,
      "rewards/rejected": -0.44124966859817505,
      "step": 480
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.737218535878705e-06,
      "logits/chosen": 0.2484460324048996,
      "logits/rejected": 0.3799341320991516,
      "logps/chosen": -582.5774536132812,
      "logps/rejected": -692.3199462890625,
      "loss": 0.2314,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.23775920271873474,
      "rewards/margins": 0.1904359757900238,
      "rewards/rejected": -0.42819517850875854,
      "step": 490
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.64432862186579e-06,
      "logits/chosen": 0.31521672010421753,
      "logits/rejected": 0.4720017910003662,
      "logps/chosen": -617.0424194335938,
      "logps/rejected": -763.6742553710938,
      "loss": 0.2106,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.2587849795818329,
      "rewards/margins": 0.21287491917610168,
      "rewards/rejected": -0.4716598391532898,
      "step": 500
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.551238285204126e-06,
      "logits/chosen": 0.23266558349132538,
      "logits/rejected": 0.4128551483154297,
      "logps/chosen": -645.7574462890625,
      "logps/rejected": -775.3682861328125,
      "loss": 0.2031,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.26859793066978455,
      "rewards/margins": 0.21920350193977356,
      "rewards/rejected": -0.4878014624118805,
      "step": 510
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.4580767962463688e-06,
      "logits/chosen": 0.2809138298034668,
      "logits/rejected": 0.36528998613357544,
      "logps/chosen": -635.5758666992188,
      "logps/rejected": -790.5833129882812,
      "loss": 0.2095,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.29754263162612915,
      "rewards/margins": 0.1984301209449768,
      "rewards/rejected": -0.49597278237342834,
      "step": 520
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3649735241511546e-06,
      "logits/chosen": 0.27885499596595764,
      "logits/rejected": 0.4202384352684021,
      "logps/chosen": -633.392333984375,
      "logps/rejected": -738.7381591796875,
      "loss": 0.2417,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.296310693025589,
      "rewards/margins": 0.18409204483032227,
      "rewards/rejected": -0.48040276765823364,
      "step": 530
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2720577572339914e-06,
      "logits/chosen": 0.2572694420814514,
      "logits/rejected": 0.3829544484615326,
      "logps/chosen": -661.2225341796875,
      "logps/rejected": -775.2647094726562,
      "loss": 0.2215,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.3194531500339508,
      "rewards/margins": 0.1850832998752594,
      "rewards/rejected": -0.5045364499092102,
      "step": 540
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1794585234303995e-06,
      "logits/chosen": 0.22785460948944092,
      "logits/rejected": 0.39559316635131836,
      "logps/chosen": -616.7025146484375,
      "logps/rejected": -784.1016235351562,
      "loss": 0.2179,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.28892993927001953,
      "rewards/margins": 0.21685433387756348,
      "rewards/rejected": -0.505784273147583,
      "step": 550
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0873044111206407e-06,
      "logits/chosen": 0.24686650931835175,
      "logits/rejected": 0.39195549488067627,
      "logps/chosen": -679.55859375,
      "logps/rejected": -784.8621215820312,
      "loss": 0.2101,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.3024412989616394,
      "rewards/margins": 0.1977556049823761,
      "rewards/rejected": -0.5001968741416931,
      "step": 560
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.9957233905648293e-06,
      "logits/chosen": 0.2292439192533493,
      "logits/rejected": 0.4420854151248932,
      "logps/chosen": -635.6842041015625,
      "logps/rejected": -802.7564697265625,
      "loss": 0.2054,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.2987775206565857,
      "rewards/margins": 0.22222964465618134,
      "rewards/rejected": -0.5210071206092834,
      "step": 570
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.904842636196402e-06,
      "logits/chosen": 0.24736735224723816,
      "logits/rejected": 0.4444567561149597,
      "logps/chosen": -658.0628662109375,
      "logps/rejected": -821.857421875,
      "loss": 0.2104,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.3125050663948059,
      "rewards/margins": 0.21667668223381042,
      "rewards/rejected": -0.5291817784309387,
      "step": 580
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.814788350020726e-06,
      "logits/chosen": 0.18406185507774353,
      "logits/rejected": 0.3239056468009949,
      "logps/chosen": -618.2083129882812,
      "logps/rejected": -770.7822875976562,
      "loss": 0.2304,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.2990264892578125,
      "rewards/margins": 0.19497379660606384,
      "rewards/rejected": -0.49400028586387634,
      "step": 590
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.725685586364051e-06,
      "logits/chosen": 0.23465320467948914,
      "logits/rejected": 0.41701340675354004,
      "logps/chosen": -603.99609375,
      "logps/rejected": -717.4134521484375,
      "loss": 0.2277,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.274019718170166,
      "rewards/margins": 0.18845617771148682,
      "rewards/rejected": -0.4624759256839752,
      "step": 600
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6376580782162172e-06,
      "logits/chosen": 0.2198990285396576,
      "logits/rejected": 0.3981742560863495,
      "logps/chosen": -660.04931640625,
      "logps/rejected": -776.05322265625,
      "loss": 0.2166,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.275198370218277,
      "rewards/margins": 0.21421167254447937,
      "rewards/rejected": -0.4894101023674011,
      "step": 610
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.550828065408227e-06,
      "logits/chosen": 0.21209344267845154,
      "logits/rejected": 0.35057374835014343,
      "logps/chosen": -608.8699951171875,
      "logps/rejected": -748.3526611328125,
      "loss": 0.2356,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.30509600043296814,
      "rewards/margins": 0.183022603392601,
      "rewards/rejected": -0.48811858892440796,
      "step": 620
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4653161248633053e-06,
      "logits/chosen": 0.1554277241230011,
      "logits/rejected": 0.38564786314964294,
      "logps/chosen": -655.8323974609375,
      "logps/rejected": -804.4343872070312,
      "loss": 0.2157,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.29179584980010986,
      "rewards/margins": 0.22734172642230988,
      "rewards/rejected": -0.5191375613212585,
      "step": 630
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.381241003157162e-06,
      "logits/chosen": 0.21835489571094513,
      "logits/rejected": 0.365634948015213,
      "logps/chosen": -661.9124145507812,
      "logps/rejected": -783.0175170898438,
      "loss": 0.1961,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.3171085715293884,
      "rewards/margins": 0.20071883499622345,
      "rewards/rejected": -0.5178273916244507,
      "step": 640
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.298719451619979e-06,
      "logits/chosen": 0.23657183349132538,
      "logits/rejected": 0.3743918836116791,
      "logps/chosen": -680.4862670898438,
      "logps/rejected": -835.0526123046875,
      "loss": 0.2212,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.3263990879058838,
      "rewards/margins": 0.22416400909423828,
      "rewards/rejected": -0.5505631566047668,
      "step": 650
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2178660642091036e-06,
      "logits/chosen": 0.2182171642780304,
      "logits/rejected": 0.32706716656684875,
      "logps/chosen": -675.8248901367188,
      "logps/rejected": -836.1906127929688,
      "loss": 0.2204,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3406047821044922,
      "rewards/margins": 0.20144791901111603,
      "rewards/rejected": -0.5420526266098022,
      "step": 660
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1387931183775821e-06,
      "logits/chosen": 0.21965551376342773,
      "logits/rejected": 0.377861887216568,
      "logps/chosen": -645.6212158203125,
      "logps/rejected": -850.5048828125,
      "loss": 0.2011,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.32531431317329407,
      "rewards/margins": 0.23981444537639618,
      "rewards/rejected": -0.565128743648529,
      "step": 670
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.061610419159532e-06,
      "logits/chosen": 0.23912313580513,
      "logits/rejected": 0.33457493782043457,
      "logps/chosen": -652.5158081054688,
      "logps/rejected": -804.5699462890625,
      "loss": 0.2,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.3123088479042053,
      "rewards/margins": 0.22432692348957062,
      "rewards/rejected": -0.5366358160972595,
      "step": 680
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.864251466888364e-07,
      "logits/chosen": 0.22238676249980927,
      "logits/rejected": 0.4501045346260071,
      "logps/chosen": -702.1354370117188,
      "logps/rejected": -854.2493896484375,
      "loss": 0.2429,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.3496148884296417,
      "rewards/margins": 0.21850232779979706,
      "rewards/rejected": -0.5681171417236328,
      "step": 690
    },
    {
      "epoch": 0.75,
      "learning_rate": 9.133417073629288e-07,
      "logits/chosen": 0.21312983334064484,
      "logits/rejected": 0.3885112702846527,
      "logps/chosen": -668.33447265625,
      "logps/rejected": -807.5436401367188,
      "loss": 0.236,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.3556881546974182,
      "rewards/margins": 0.1771022081375122,
      "rewards/rejected": -0.5327903628349304,
      "step": 700
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.424615888583332e-07,
      "logits/chosen": 0.17015187442302704,
      "logits/rejected": 0.3787848949432373,
      "logps/chosen": -694.0350341796875,
      "logps/rejected": -877.3869018554688,
      "loss": 0.1718,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.32832252979278564,
      "rewards/margins": 0.2480977475643158,
      "rewards/rejected": -0.5764203071594238,
      "step": 710
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.738832191993092e-07,
      "logits/chosen": 0.18861600756645203,
      "logits/rejected": 0.3507705330848694,
      "logps/chosen": -677.43701171875,
      "logps/rejected": -824.4730224609375,
      "loss": 0.1952,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.3222949504852295,
      "rewards/margins": 0.23219437897205353,
      "rewards/rejected": -0.5544893145561218,
      "step": 720
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.077018300752917e-07,
      "logits/chosen": 0.22210793197155,
      "logits/rejected": 0.3462095856666565,
      "logps/chosen": -642.8484497070312,
      "logps/rejected": -801.2487182617188,
      "loss": 0.2144,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.2975847125053406,
      "rewards/margins": 0.23604163527488708,
      "rewards/rejected": -0.53362637758255,
      "step": 730
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.440093245969342e-07,
      "logits/chosen": 0.18037883937358856,
      "logits/rejected": 0.32701772451400757,
      "logps/chosen": -619.3107299804688,
      "logps/rejected": -796.5120849609375,
      "loss": 0.2121,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2894679307937622,
      "rewards/margins": 0.22074396908283234,
      "rewards/rejected": -0.5102118849754333,
      "step": 740
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.828941496744075e-07,
      "logits/chosen": 0.21488244831562042,
      "logits/rejected": 0.3539666533470154,
      "logps/chosen": -646.1163330078125,
      "logps/rejected": -821.9992065429688,
      "loss": 0.1931,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.30917179584503174,
      "rewards/margins": 0.22842054069042206,
      "rewards/rejected": -0.537592351436615,
      "step": 750
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.244411731951671e-07,
      "logits/chosen": 0.2123112678527832,
      "logits/rejected": 0.38105306029319763,
      "logps/chosen": -718.8529663085938,
      "logps/rejected": -841.3015747070312,
      "loss": 0.2265,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.3423205316066742,
      "rewards/margins": 0.21526296436786652,
      "rewards/rejected": -0.5575835108757019,
      "step": 760
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.6873156617173594e-07,
      "logits/chosen": 0.2072085589170456,
      "logits/rejected": 0.33443737030029297,
      "logps/chosen": -687.7586669921875,
      "logps/rejected": -815.7947998046875,
      "loss": 0.2148,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.3153381943702698,
      "rewards/margins": 0.20979492366313934,
      "rewards/rejected": -0.5251331329345703,
      "step": 770
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.1584269002318653e-07,
      "logits/chosen": 0.22315290570259094,
      "logits/rejected": 0.39601173996925354,
      "logps/chosen": -641.2395629882812,
      "logps/rejected": -768.5919799804688,
      "loss": 0.2332,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.2931545674800873,
      "rewards/margins": 0.20355018973350525,
      "rewards/rejected": -0.49670475721359253,
      "step": 780
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.658479891468258e-07,
      "logits/chosen": 0.23425737023353577,
      "logits/rejected": 0.3700588047504425,
      "logps/chosen": -647.1505737304688,
      "logps/rejected": -796.8721313476562,
      "loss": 0.2232,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.3021949827671051,
      "rewards/margins": 0.2227589339017868,
      "rewards/rejected": -0.5249539017677307,
      "step": 790
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.18816888929272e-07,
      "logits/chosen": 0.21941573917865753,
      "logits/rejected": 0.38576945662498474,
      "logps/chosen": -681.2139282226562,
      "logps/rejected": -785.1143188476562,
      "loss": 0.2456,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.3359334468841553,
      "rewards/margins": 0.15878985822200775,
      "rewards/rejected": -0.4947233200073242,
      "step": 800
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.748146993385484e-07,
      "logits/chosen": 0.21103620529174805,
      "logits/rejected": 0.37812119722366333,
      "logps/chosen": -675.7420654296875,
      "logps/rejected": -835.7942504882812,
      "loss": 0.2342,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.3283590078353882,
      "rewards/margins": 0.1955871880054474,
      "rewards/rejected": -0.5239461660385132,
      "step": 810
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.3390252423108077e-07,
      "logits/chosen": 0.18820640444755554,
      "logits/rejected": 0.3453311324119568,
      "logps/chosen": -699.3671264648438,
      "logps/rejected": -808.0126342773438,
      "loss": 0.2185,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.32744988799095154,
      "rewards/margins": 0.19639787077903748,
      "rewards/rejected": -0.523847758769989,
      "step": 820
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.961371764995243e-07,
      "logits/chosen": 0.1902572214603424,
      "logits/rejected": 0.3874988257884979,
      "logps/chosen": -704.2755737304688,
      "logps/rejected": -798.5086669921875,
      "loss": 0.2272,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.3353065848350525,
      "rewards/margins": 0.1846623420715332,
      "rewards/rejected": -0.5199689269065857,
      "step": 830
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.61571099179261e-07,
      "logits/chosen": 0.2346924990415573,
      "logits/rejected": 0.35877925157546997,
      "logps/chosen": -631.2687377929688,
      "logps/rejected": -800.9693603515625,
      "loss": 0.2343,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.31699609756469727,
      "rewards/margins": 0.20049229264259338,
      "rewards/rejected": -0.517488420009613,
      "step": 840
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.3025229262312367e-07,
      "logits/chosen": 0.22632627189159393,
      "logits/rejected": 0.4042627811431885,
      "logps/chosen": -647.9192504882812,
      "logps/rejected": -805.1476440429688,
      "loss": 0.1767,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.29563456773757935,
      "rewards/margins": 0.23449036478996277,
      "rewards/rejected": -0.5301249623298645,
      "step": 850
    },
    {
      "epoch": 0.92,
      "learning_rate": 1.0222424784546853e-07,
      "logits/chosen": 0.18376657366752625,
      "logits/rejected": 0.2582804560661316,
      "logps/chosen": -614.8178100585938,
      "logps/rejected": -754.7501220703125,
      "loss": 0.2084,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.27827420830726624,
      "rewards/margins": 0.21241740882396698,
      "rewards/rejected": -0.49069157242774963,
      "step": 860
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.752588612816553e-08,
      "logits/chosen": 0.2309151142835617,
      "logits/rejected": 0.32766053080558777,
      "logps/chosen": -618.4566650390625,
      "logps/rejected": -769.9896850585938,
      "loss": 0.2188,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2959611415863037,
      "rewards/margins": 0.19887039065361023,
      "rewards/rejected": -0.49483156204223633,
      "step": 870
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.619150497236991e-08,
      "logits/chosen": 0.24140353500843048,
      "logits/rejected": 0.29053181409835815,
      "logps/chosen": -647.7692260742188,
      "logps/rejected": -806.162109375,
      "loss": 0.2179,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.31588608026504517,
      "rewards/margins": 0.21005694568157196,
      "rewards/rejected": -0.5259430408477783,
      "step": 880
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.825073047112743e-08,
      "logits/chosen": 0.21158334612846375,
      "logits/rejected": 0.31529536843299866,
      "logps/chosen": -656.6387939453125,
      "logps/rejected": -829.8762817382812,
      "loss": 0.2066,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.31754207611083984,
      "rewards/margins": 0.2269022911787033,
      "rewards/rejected": -0.5444443821907043,
      "step": 890
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.372847616895685e-08,
      "logits/chosen": 0.18535788357257843,
      "logits/rejected": 0.3380703330039978,
      "logps/chosen": -669.4324340820312,
      "logps/rejected": -809.5096435546875,
      "loss": 0.2213,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.33157414197921753,
      "rewards/margins": 0.20540854334831238,
      "rewards/rejected": -0.5369827151298523,
      "step": 900
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.264490846553279e-08,
      "logits/chosen": 0.21935626864433289,
      "logits/rejected": 0.3299446702003479,
      "logps/chosen": -643.43310546875,
      "logps/rejected": -753.556884765625,
      "loss": 0.2527,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.3096220791339874,
      "rewards/margins": 0.17697684466838837,
      "rewards/rejected": -0.4865988790988922,
      "step": 910
    },
    {
      "epoch": 0.98,
      "learning_rate": 5.015418611516165e-09,
      "logits/chosen": 0.20347245037555695,
      "logits/rejected": 0.37708914279937744,
      "logps/chosen": -650.4703979492188,
      "logps/rejected": -769.6436767578125,
      "loss": 0.2711,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.3225279450416565,
      "rewards/margins": 0.16174957156181335,
      "rewards/rejected": -0.48427754640579224,
      "step": 920
    },
    {
      "epoch": 0.99,
      "learning_rate": 8.506013354186993e-10,
      "logits/chosen": 0.22470831871032715,
      "logits/rejected": 0.3196450173854828,
      "logps/chosen": -647.2747192382812,
      "logps/rejected": -821.5018310546875,
      "loss": 0.2047,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.3270157277584076,
      "rewards/margins": 0.21042361855506897,
      "rewards/rejected": -0.5374393463134766,
      "step": 930
    },
    {
      "epoch": 1.0,
      "step": 937,
      "total_flos": 0.0,
      "train_loss": 0.2409534176996449,
      "train_runtime": 8445.1148,
      "train_samples_per_second": 1.776,
      "train_steps_per_second": 0.111
    }
  ],
  "logging_steps": 10,
  "max_steps": 937,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}