{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9984301412872841,
  "eval_steps": 100,
  "global_step": 477,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 1.0416666666666666e-08,
      "logits/chosen": 0.07429073005914688,
      "logits/rejected": 0.11598015576601028,
      "logps/chosen": -422.86004638671875,
      "logps/rejected": -414.8901672363281,
      "loss": 0.1089,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.0416666666666667e-07,
      "logits/chosen": 0.30265143513679504,
      "logits/rejected": 0.37510383129119873,
      "logps/chosen": -376.6183166503906,
      "logps/rejected": -346.707275390625,
      "loss": 0.1038,
      "rewards/accuracies": 0.3958333432674408,
      "rewards/chosen": -0.0007193544879555702,
      "rewards/margins": -0.00018897267000284046,
      "rewards/rejected": -0.0005303816869854927,
      "step": 10
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.0833333333333333e-07,
      "logits/chosen": 0.19012752175331116,
      "logits/rejected": 0.29147377610206604,
      "logps/chosen": -427.39190673828125,
      "logps/rejected": -371.12261962890625,
      "loss": 0.107,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0001767327485140413,
      "rewards/margins": 0.0014846216654404998,
      "rewards/rejected": -0.0013078890042379498,
      "step": 20
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": 0.20171578228473663,
      "logits/rejected": 0.31732800602912903,
      "logps/chosen": -399.6023254394531,
      "logps/rejected": -374.3086853027344,
      "loss": 0.1094,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.0014916183426976204,
      "rewards/margins": 0.001707876450382173,
      "rewards/rejected": -0.0031994946766644716,
      "step": 30
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.1666666666666667e-07,
      "logits/chosen": 0.24505770206451416,
      "logits/rejected": 0.3264174163341522,
      "logps/chosen": -363.1432189941406,
      "logps/rejected": -371.2718200683594,
      "loss": 0.1094,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0007695726817473769,
      "rewards/margins": 0.006053343415260315,
      "rewards/rejected": -0.006822916213423014,
      "step": 40
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999731868769026e-07,
      "logits/chosen": 0.26892322301864624,
      "logits/rejected": 0.34712380170822144,
      "logps/chosen": -378.1469421386719,
      "logps/rejected": -348.55352783203125,
      "loss": 0.105,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.0007855228031985462,
      "rewards/margins": 0.022909890860319138,
      "rewards/rejected": -0.023695414885878563,
      "step": 50
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.990353313429303e-07,
      "logits/chosen": 0.3134162127971649,
      "logits/rejected": 0.36972466111183167,
      "logps/chosen": -317.57965087890625,
      "logps/rejected": -317.5392761230469,
      "loss": 0.1049,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.008349133655428886,
      "rewards/margins": 0.035110391676425934,
      "rewards/rejected": -0.04345952719449997,
      "step": 60
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.967625656594781e-07,
      "logits/chosen": 0.23508644104003906,
      "logits/rejected": 0.30452126264572144,
      "logps/chosen": -359.3031311035156,
      "logps/rejected": -345.0860900878906,
      "loss": 0.1082,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.01529479306191206,
      "rewards/margins": 0.044887155294418335,
      "rewards/rejected": -0.06018194556236267,
      "step": 70
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.93167072587771e-07,
      "logits/chosen": 0.33254843950271606,
      "logits/rejected": 0.3818221688270569,
      "logps/chosen": -367.01129150390625,
      "logps/rejected": -355.1100158691406,
      "loss": 0.1028,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.01488255150616169,
      "rewards/margins": 0.08553680032491684,
      "rewards/rejected": -0.10041935741901398,
      "step": 80
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.882681251368548e-07,
      "logits/chosen": 0.3798341751098633,
      "logits/rejected": 0.4554978311061859,
      "logps/chosen": -369.4306640625,
      "logps/rejected": -350.7769470214844,
      "loss": 0.1031,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.04546991363167763,
      "rewards/margins": 0.09270831197500229,
      "rewards/rejected": -0.1381782442331314,
      "step": 90
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.820919832540181e-07,
      "logits/chosen": 0.4522795081138611,
      "logits/rejected": 0.5922520756721497,
      "logps/chosen": -379.4673767089844,
      "logps/rejected": -347.5188903808594,
      "loss": 0.0993,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.062339335680007935,
      "rewards/margins": 0.1438979208469391,
      "rewards/rejected": -0.20623722672462463,
      "step": 100
    },
    {
      "epoch": 0.21,
      "eval_logits/chosen": 0.5129754543304443,
      "eval_logits/rejected": 0.5706519484519958,
      "eval_logps/chosen": -366.9024963378906,
      "eval_logps/rejected": -390.411865234375,
      "eval_loss": 0.10182792693376541,
      "eval_rewards/accuracies": 0.6875,
      "eval_rewards/chosen": -0.07451306283473969,
      "eval_rewards/margins": 0.17251166701316833,
      "eval_rewards/rejected": -0.2470247447490692,
      "eval_runtime": 75.2158,
      "eval_samples_per_second": 26.59,
      "eval_steps_per_second": 0.425,
      "step": 100
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.7467175306295647e-07,
      "logits/chosen": 0.47333478927612305,
      "logits/rejected": 0.5572434663772583,
      "logps/chosen": -408.17938232421875,
      "logps/rejected": -431.2904357910156,
      "loss": 0.0955,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.12534171342849731,
      "rewards/margins": 0.16846705973148346,
      "rewards/rejected": -0.2938087582588196,
      "step": 110
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.6604720940421207e-07,
      "logits/chosen": 0.49355560541152954,
      "logits/rejected": 0.625954270362854,
      "logps/chosen": -393.81842041015625,
      "logps/rejected": -401.51202392578125,
      "loss": 0.0822,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.14783725142478943,
      "rewards/margins": 0.23524054884910583,
      "rewards/rejected": -0.38307780027389526,
      "step": 120
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.5626458262912735e-07,
      "logits/chosen": 0.5241672396659851,
      "logits/rejected": 0.8548004031181335,
      "logps/chosen": -435.3441467285156,
      "logps/rejected": -357.9606018066406,
      "loss": 0.0797,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.31069421768188477,
      "rewards/margins": 0.1076047271490097,
      "rewards/rejected": -0.41829895973205566,
      "step": 130
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.453763107901675e-07,
      "logits/chosen": 0.5490397810935974,
      "logits/rejected": 0.7539095282554626,
      "logps/chosen": -413.84002685546875,
      "logps/rejected": -386.32379150390625,
      "loss": 0.0682,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3854388892650604,
      "rewards/margins": 0.1845283955335617,
      "rewards/rejected": -0.5699672698974609,
      "step": 140
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.3344075855595097e-07,
      "logits/chosen": 0.6215056777000427,
      "logits/rejected": 0.7831140756607056,
      "logps/chosen": -419.662841796875,
      "logps/rejected": -391.9927673339844,
      "loss": 0.0704,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.35133233666419983,
      "rewards/margins": 0.202161505818367,
      "rewards/rejected": -0.553493857383728,
      "step": 150
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.2052190435769554e-07,
      "logits/chosen": 0.7069424390792847,
      "logits/rejected": 0.8614177703857422,
      "logps/chosen": -384.68463134765625,
      "logps/rejected": -374.79144287109375,
      "loss": 0.0684,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.2964687943458557,
      "rewards/margins": 0.2469368427991867,
      "rewards/rejected": -0.5434056520462036,
      "step": 160
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.0668899744407567e-07,
      "logits/chosen": 0.6795724034309387,
      "logits/rejected": 0.788532555103302,
      "logps/chosen": -407.76092529296875,
      "logps/rejected": -429.6249084472656,
      "loss": 0.065,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.38756096363067627,
      "rewards/margins": 0.3297034204006195,
      "rewards/rejected": -0.7172644734382629,
      "step": 170
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.920161866827889e-07,
      "logits/chosen": 0.6546531319618225,
      "logits/rejected": 0.8610761761665344,
      "logps/chosen": -399.92608642578125,
      "logps/rejected": -425.089599609375,
      "loss": 0.059,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.45196419954299927,
      "rewards/margins": 0.27189016342163086,
      "rewards/rejected": -0.7238543033599854,
      "step": 180
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.765821230985757e-07,
      "logits/chosen": 0.694263756275177,
      "logits/rejected": 0.9027630090713501,
      "logps/chosen": -411.06268310546875,
      "logps/rejected": -397.81097412109375,
      "loss": 0.0616,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.46890345215797424,
      "rewards/margins": 0.21999216079711914,
      "rewards/rejected": -0.688895583152771,
      "step": 190
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.604695382782159e-07,
      "logits/chosen": 0.6648334264755249,
      "logits/rejected": 0.9266846776008606,
      "logps/chosen": -450.61871337890625,
      "logps/rejected": -451.17596435546875,
      "loss": 0.0553,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.5313616394996643,
      "rewards/margins": 0.4010355472564697,
      "rewards/rejected": -0.9323970675468445,
      "step": 200
    },
    {
      "epoch": 0.42,
      "eval_logits/chosen": 0.7755414247512817,
      "eval_logits/rejected": 0.8994145393371582,
      "eval_logps/chosen": -410.08349609375,
      "eval_logps/rejected": -459.32171630859375,
      "eval_loss": 0.05395492538809776,
      "eval_rewards/accuracies": 0.73046875,
      "eval_rewards/chosen": -0.5063228607177734,
      "eval_rewards/margins": 0.42980003356933594,
      "eval_rewards/rejected": -0.9361228942871094,
      "eval_runtime": 75.0793,
      "eval_samples_per_second": 26.638,
      "eval_steps_per_second": 0.426,
      "step": 200
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4376480090239047e-07,
      "logits/chosen": 0.740880012512207,
      "logits/rejected": 0.9735954999923706,
      "logps/chosen": -407.00970458984375,
      "logps/rejected": -405.5915832519531,
      "loss": 0.0517,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.6327391266822815,
      "rewards/margins": 0.24419736862182617,
      "rewards/rejected": -0.8769365549087524,
      "step": 210
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.265574537815398e-07,
      "logits/chosen": 0.5885998606681824,
      "logits/rejected": 0.8313006162643433,
      "logps/chosen": -449.39251708984375,
      "logps/rejected": -427.76641845703125,
      "loss": 0.0537,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.5039619207382202,
      "rewards/margins": 0.33774858713150024,
      "rewards/rejected": -0.8417104482650757,
      "step": 220
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.0893973387735683e-07,
      "logits/chosen": 0.5277591347694397,
      "logits/rejected": 0.7385606169700623,
      "logps/chosen": -428.235595703125,
      "logps/rejected": -417.846923828125,
      "loss": 0.054,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.5545467138290405,
      "rewards/margins": 0.3285568654537201,
      "rewards/rejected": -0.8831035494804382,
      "step": 230
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.910060778827554e-07,
      "logits/chosen": 0.5636624693870544,
      "logits/rejected": 0.7239071130752563,
      "logps/chosen": -410.8172302246094,
      "logps/rejected": -453.8780822753906,
      "loss": 0.0517,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6257103085517883,
      "rewards/margins": 0.3102150559425354,
      "rewards/rejected": -0.9359253644943237,
      "step": 240
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7285261601056697e-07,
      "logits/chosen": 0.6507786512374878,
      "logits/rejected": 0.7937926054000854,
      "logps/chosen": -399.25531005859375,
      "logps/rejected": -428.21173095703125,
      "loss": 0.0482,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.5876737236976624,
      "rewards/margins": 0.42669692635536194,
      "rewards/rejected": -1.0143706798553467,
      "step": 250
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5457665670441937e-07,
      "logits/chosen": 0.7192268967628479,
      "logits/rejected": 0.9365060925483704,
      "logps/chosen": -442.73931884765625,
      "logps/rejected": -429.482421875,
      "loss": 0.0515,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.7237201929092407,
      "rewards/margins": 0.23805689811706543,
      "rewards/rejected": -0.9617769122123718,
      "step": 260
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3627616503391812e-07,
      "logits/chosen": 0.8510295748710632,
      "logits/rejected": 1.0050238370895386,
      "logps/chosen": -445.4356384277344,
      "logps/rejected": -501.38287353515625,
      "loss": 0.0505,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.6589037179946899,
      "rewards/margins": 0.45235753059387207,
      "rewards/rejected": -1.111261248588562,
      "step": 270
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1804923757009882e-07,
      "logits/chosen": 0.7827587723731995,
      "logits/rejected": 0.8751026391983032,
      "logps/chosen": -394.4527282714844,
      "logps/rejected": -438.7657165527344,
      "loss": 0.0514,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.6062650084495544,
      "rewards/margins": 0.3552294373512268,
      "rewards/rejected": -0.9614945650100708,
      "step": 280
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.9999357655598891e-07,
      "logits/chosen": 0.6437442898750305,
      "logits/rejected": 0.9078620672225952,
      "logps/chosen": -437.314697265625,
      "logps/rejected": -448.2724609375,
      "loss": 0.0493,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.6567361950874329,
      "rewards/margins": 0.38017868995666504,
      "rewards/rejected": -1.0369149446487427,
      "step": 290
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8220596619089573e-07,
      "logits/chosen": 0.6364871263504028,
      "logits/rejected": 0.8500644564628601,
      "logps/chosen": -384.7347412109375,
      "logps/rejected": -401.3032531738281,
      "loss": 0.0503,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.5907383561134338,
      "rewards/margins": 0.31186026334762573,
      "rewards/rejected": -0.9025986790657043,
      "step": 300
    },
    {
      "epoch": 0.63,
      "eval_logits/chosen": 0.6691160202026367,
      "eval_logits/rejected": 0.8094965219497681,
      "eval_logps/chosen": -412.98126220703125,
      "eval_logps/rejected": -468.3688049316406,
      "eval_loss": 0.05308157950639725,
      "eval_rewards/accuracies": 0.74609375,
      "eval_rewards/chosen": -0.5353004932403564,
      "eval_rewards/margins": 0.4912930727005005,
      "eval_rewards/rejected": -1.0265936851501465,
      "eval_runtime": 75.2339,
      "eval_samples_per_second": 26.584,
      "eval_steps_per_second": 0.425,
      "step": 300
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.647817538357072e-07,
      "logits/chosen": 0.5804156064987183,
      "logits/rejected": 0.8023425936698914,
      "logps/chosen": -431.28790283203125,
      "logps/rejected": -459.8777770996094,
      "loss": 0.0547,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5415506958961487,
      "rewards/margins": 0.43213915824890137,
      "rewards/rejected": -0.97368985414505,
      "step": 310
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.478143389201113e-07,
      "logits/chosen": 0.5641556978225708,
      "logits/rejected": 0.7904598712921143,
      "logps/chosen": -419.69329833984375,
      "logps/rejected": -420.63629150390625,
      "loss": 0.0512,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.5348724126815796,
      "rewards/margins": 0.3764938712120056,
      "rewards/rejected": -0.9113662838935852,
      "step": 320
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3139467229135998e-07,
      "logits/chosen": 0.7080197334289551,
      "logits/rejected": 0.8454129099845886,
      "logps/chosen": -395.1232604980469,
      "logps/rejected": -457.27978515625,
      "loss": 0.0486,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.5784841775894165,
      "rewards/margins": 0.39547181129455566,
      "rewards/rejected": -0.9739561080932617,
      "step": 330
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1561076868822755e-07,
      "logits/chosen": 0.7032380700111389,
      "logits/rejected": 0.8845696449279785,
      "logps/chosen": -421.2705078125,
      "logps/rejected": -432.5623474121094,
      "loss": 0.0532,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.6043194532394409,
      "rewards/margins": 0.37815985083580017,
      "rewards/rejected": -0.9824793934822083,
      "step": 340
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0054723495346482e-07,
      "logits/chosen": 0.7004996538162231,
      "logits/rejected": 0.8182412981987,
      "logps/chosen": -392.1104431152344,
      "logps/rejected": -416.94415283203125,
      "loss": 0.0467,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.6616944074630737,
      "rewards/margins": 0.3355887532234192,
      "rewards/rejected": -0.9972831606864929,
      "step": 350
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.628481651367875e-08,
      "logits/chosen": 0.6653555631637573,
      "logits/rejected": 0.8546744585037231,
      "logps/chosen": -370.65008544921875,
      "logps/rejected": -413.5162048339844,
      "loss": 0.0484,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.529220461845398,
      "rewards/margins": 0.42656245827674866,
      "rewards/rejected": -0.9557830095291138,
      "step": 360
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.289996455765748e-08,
      "logits/chosen": 0.6534770131111145,
      "logits/rejected": 0.7723512053489685,
      "logps/chosen": -436.49346923828125,
      "logps/rejected": -463.48577880859375,
      "loss": 0.0486,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.5268007516860962,
      "rewards/margins": 0.5095833539962769,
      "rewards/rejected": -1.0363839864730835,
      "step": 370
    },
    {
      "epoch": 0.8,
      "learning_rate": 6.046442623320145e-08,
      "logits/chosen": 0.7045032978057861,
      "logits/rejected": 0.7946165800094604,
      "logps/chosen": -398.9028625488281,
      "logps/rejected": -436.78515625,
      "loss": 0.0499,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5992251634597778,
      "rewards/margins": 0.39836931228637695,
      "rewards/rejected": -0.9975944757461548,
      "step": 380
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.904486005914027e-08,
      "logits/chosen": 0.5793179869651794,
      "logits/rejected": 0.7902762293815613,
      "logps/chosen": -427.8744201660156,
      "logps/rejected": -456.980224609375,
      "loss": 0.0472,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.541319727897644,
      "rewards/margins": 0.55488520860672,
      "rewards/rejected": -1.0962049961090088,
      "step": 390
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.8702478614051345e-08,
      "logits/chosen": 0.6567720174789429,
      "logits/rejected": 0.8759796023368835,
      "logps/chosen": -413.21124267578125,
      "logps/rejected": -466.8826599121094,
      "loss": 0.0537,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.5818170309066772,
      "rewards/margins": 0.48379120230674744,
      "rewards/rejected": -1.065608263015747,
      "step": 400
    },
    {
      "epoch": 0.84,
      "eval_logits/chosen": 0.7434472441673279,
      "eval_logits/rejected": 0.8845732808113098,
      "eval_logps/chosen": -416.42877197265625,
      "eval_logps/rejected": -473.2190246582031,
      "eval_loss": 0.05153993144631386,
      "eval_rewards/accuracies": 0.73046875,
      "eval_rewards/chosen": -0.5697759985923767,
      "eval_rewards/margins": 0.5053197145462036,
      "eval_rewards/rejected": -1.075095772743225,
      "eval_runtime": 76.6604,
      "eval_samples_per_second": 26.089,
      "eval_steps_per_second": 0.417,
      "step": 400
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.9492720416985e-08,
      "logits/chosen": 0.6941822171211243,
      "logits/rejected": 0.8047625422477722,
      "logps/chosen": -401.1028137207031,
      "logps/rejected": -427.03302001953125,
      "loss": 0.0478,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.576189398765564,
      "rewards/margins": 0.39370283484458923,
      "rewards/rejected": -0.969892144203186,
      "step": 410
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.1464952759020856e-08,
      "logits/chosen": 0.7099225521087646,
      "logits/rejected": 0.8760135769844055,
      "logps/chosen": -388.6665954589844,
      "logps/rejected": -421.422607421875,
      "loss": 0.0477,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.5949358344078064,
      "rewards/margins": 0.3995293378829956,
      "rewards/rejected": -0.9944650530815125,
      "step": 420
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.4662207078575684e-08,
      "logits/chosen": 0.6152527332305908,
      "logits/rejected": 0.7688241600990295,
      "logps/chosen": -453.6767578125,
      "logps/rejected": -463.40863037109375,
      "loss": 0.0527,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.613703727722168,
      "rewards/margins": 0.43121272325515747,
      "rewards/rejected": -1.0449163913726807,
      "step": 430
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.12094829893642e-09,
      "logits/chosen": 0.7038220167160034,
      "logits/rejected": 0.8598071932792664,
      "logps/chosen": -436.53082275390625,
      "logps/rejected": -434.2837829589844,
      "loss": 0.0484,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.6220607161521912,
      "rewards/margins": 0.44049739837646484,
      "rewards/rejected": -1.0625581741333008,
      "step": 440
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.8708793644441086e-09,
      "logits/chosen": 0.6375268697738647,
      "logits/rejected": 0.8610206842422485,
      "logps/chosen": -462.42449951171875,
      "logps/rejected": -464.8294372558594,
      "loss": 0.0474,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.685294508934021,
      "rewards/margins": 0.4343951344490051,
      "rewards/rejected": -1.119689702987671,
      "step": 450
    },
    {
      "epoch": 0.96,
      "learning_rate": 1.9347820230782295e-09,
      "logits/chosen": 0.6665740013122559,
      "logits/rejected": 0.8705039024353027,
      "logps/chosen": -409.6141662597656,
      "logps/rejected": -419.2763671875,
      "loss": 0.0457,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.5975238084793091,
      "rewards/margins": 0.3504991829395294,
      "rewards/rejected": -0.9480230212211609,
      "step": 460
    },
    {
      "epoch": 0.98,
      "learning_rate": 3.2839470889836627e-10,
      "logits/chosen": 0.5994333028793335,
      "logits/rejected": 0.8693232536315918,
      "logps/chosen": -446.9803771972656,
      "logps/rejected": -442.191650390625,
      "loss": 0.0495,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.5732464790344238,
      "rewards/margins": 0.4613564610481262,
      "rewards/rejected": -1.0346029996871948,
      "step": 470
    },
    {
      "epoch": 1.0,
      "step": 477,
      "total_flos": 0.0,
      "train_loss": 0.06614070340397472,
      "train_runtime": 4423.243,
      "train_samples_per_second": 13.821,
      "train_steps_per_second": 0.108
    }
  ],
  "logging_steps": 10,
  "max_steps": 477,
  "num_train_epochs": 1,
  "save_steps": 1000,
  "total_flos": 0.0,
  "trial_name": null,
  "trial_params": null
}