{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9994666666666666,
  "eval_steps": 500,
  "global_step": 937,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 5.319148936170213e-08,
      "logits/chosen": -0.07764632999897003,
      "logits/rejected": 0.0654495358467102,
      "logps/chosen": -548.1927490234375,
      "logps/rejected": -445.62200927734375,
      "loss": 0.21,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "learning_rate": 5.319148936170213e-07,
      "logits/chosen": 0.013010119087994099,
      "logits/rejected": 0.14687097072601318,
      "logps/chosen": -445.48004150390625,
      "logps/rejected": -474.8713073730469,
      "loss": 0.2064,
      "rewards/accuracies": 0.3333333432674408,
      "rewards/chosen": 0.0016270489431917667,
      "rewards/margins": -3.060871677007526e-05,
      "rewards/rejected": 0.0016576575580984354,
      "step": 10
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.0638297872340427e-06,
      "logits/chosen": 0.04200610890984535,
      "logits/rejected": 0.11701512336730957,
      "logps/chosen": -480.87042236328125,
      "logps/rejected": -453.9954528808594,
      "loss": 0.2121,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 0.002087645698338747,
      "rewards/margins": 4.953762254444882e-05,
      "rewards/rejected": 0.002038107719272375,
      "step": 20
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.595744680851064e-06,
      "logits/chosen": 0.08031486719846725,
      "logits/rejected": 0.17916838824748993,
      "logps/chosen": -533.2267456054688,
      "logps/rejected": -509.03192138671875,
      "loss": 0.2044,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": 0.001751051051542163,
      "rewards/margins": 0.0012214130256325006,
      "rewards/rejected": 0.0005296378512866795,
      "step": 30
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.1276595744680853e-06,
      "logits/chosen": 0.02258063293993473,
      "logits/rejected": 0.07948300242424011,
      "logps/chosen": -482.404296875,
      "logps/rejected": -482.5491638183594,
      "loss": 0.2131,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.0007890533888712525,
      "rewards/margins": 0.002143475692719221,
      "rewards/rejected": -0.002932528965175152,
      "step": 40
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.6595744680851065e-06,
      "logits/chosen": 0.12140221893787384,
      "logits/rejected": 0.09028098732233047,
      "logps/chosen": -442.23797607421875,
      "logps/rejected": -465.88214111328125,
      "loss": 0.2105,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.009099994786083698,
      "rewards/margins": 0.002986655803397298,
      "rewards/rejected": -0.012086650356650352,
      "step": 50
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.191489361702128e-06,
      "logits/chosen": 0.053414009511470795,
      "logits/rejected": 0.18009646236896515,
      "logps/chosen": -528.6683959960938,
      "logps/rejected": -486.89361572265625,
      "loss": 0.1982,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.015702595934271812,
      "rewards/margins": 0.0040222653187811375,
      "rewards/rejected": -0.019724860787391663,
      "step": 60
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.723404255319149e-06,
      "logits/chosen": 0.07392764091491699,
      "logits/rejected": 0.11563147604465485,
      "logps/chosen": -534.7530517578125,
      "logps/rejected": -519.1510009765625,
      "loss": 0.2073,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.02576013281941414,
      "rewards/margins": 0.006931398995220661,
      "rewards/rejected": -0.032691530883312225,
      "step": 70
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.255319148936171e-06,
      "logits/chosen": 0.0789504274725914,
      "logits/rejected": -0.028738999739289284,
      "logps/chosen": -516.7589111328125,
      "logps/rejected": -550.5140991210938,
      "loss": 0.1998,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02832980453968048,
      "rewards/margins": 0.013306397013366222,
      "rewards/rejected": -0.04163619875907898,
      "step": 80
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.787234042553192e-06,
      "logits/chosen": 0.007517285645008087,
      "logits/rejected": 0.06739212572574615,
      "logps/chosen": -530.9830932617188,
      "logps/rejected": -524.5928955078125,
      "loss": 0.2029,
      "rewards/accuracies": 0.39375001192092896,
      "rewards/chosen": -0.06454873085021973,
      "rewards/margins": 0.008291224017739296,
      "rewards/rejected": -0.07283996045589447,
      "step": 90
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.999375059004058e-06,
      "logits/chosen": 0.023702986538410187,
      "logits/rejected": 0.037184469401836395,
      "logps/chosen": -489.89581298828125,
      "logps/rejected": -484.0779724121094,
      "loss": 0.1966,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -0.07171402126550674,
      "rewards/margins": 0.024490345269441605,
      "rewards/rejected": -0.09620436280965805,
      "step": 100
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.9955571065548795e-06,
      "logits/chosen": 0.09514528512954712,
      "logits/rejected": 0.029327446594834328,
      "logps/chosen": -453.439697265625,
      "logps/rejected": -483.3421325683594,
      "loss": 0.1919,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.06944920122623444,
      "rewards/margins": 0.02075176127254963,
      "rewards/rejected": -0.09020096063613892,
      "step": 110
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.9882736864879e-06,
      "logits/chosen": 0.052310071885585785,
      "logits/rejected": 0.023372933268547058,
      "logps/chosen": -465.06622314453125,
      "logps/rejected": -487.2718200683594,
      "loss": 0.1983,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.045129794627428055,
      "rewards/margins": 0.017120003700256348,
      "rewards/rejected": -0.0622497983276844,
      "step": 120
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.977534912960124e-06,
      "logits/chosen": 0.040262121707201004,
      "logits/rejected": 0.2257479429244995,
      "logps/chosen": -475.408935546875,
      "logps/rejected": -475.36669921875,
      "loss": 0.1949,
      "rewards/accuracies": 0.41874998807907104,
      "rewards/chosen": -0.03786056488752365,
      "rewards/margins": 0.022892726585268974,
      "rewards/rejected": -0.060753293335437775,
      "step": 130
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.963355698422092e-06,
      "logits/chosen": 0.16925258934497833,
      "logits/rejected": 0.07987479865550995,
      "logps/chosen": -481.04083251953125,
      "logps/rejected": -517.6348876953125,
      "loss": 0.1979,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.07286442816257477,
      "rewards/margins": 0.0315692201256752,
      "rewards/rejected": -0.10443364083766937,
      "step": 140
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.945755732909625e-06,
      "logits/chosen": 0.021633749827742577,
      "logits/rejected": 0.06488915532827377,
      "logps/chosen": -476.21661376953125,
      "logps/rejected": -496.6123046875,
      "loss": 0.1908,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.10675158351659775,
      "rewards/margins": 0.03962003067135811,
      "rewards/rejected": -0.14637163281440735,
      "step": 150
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.924759456701167e-06,
      "logits/chosen": 0.05234859511256218,
      "logits/rejected": 0.02733474411070347,
      "logps/chosen": -506.60382080078125,
      "logps/rejected": -550.2049560546875,
      "loss": 0.1971,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.12149013578891754,
      "rewards/margins": 0.03788003325462341,
      "rewards/rejected": -0.15937016904354095,
      "step": 160
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.900396026378671e-06,
      "logits/chosen": 0.00539384875446558,
      "logits/rejected": 0.001964741852134466,
      "logps/chosen": -469.29840087890625,
      "logps/rejected": -486.98504638671875,
      "loss": 0.2055,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0949864536523819,
      "rewards/margins": 0.023784717544913292,
      "rewards/rejected": -0.11877117305994034,
      "step": 170
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.872699274339169e-06,
      "logits/chosen": 0.03078017197549343,
      "logits/rejected": 0.06562785059213638,
      "logps/chosen": -467.913818359375,
      "logps/rejected": -488.88165283203125,
      "loss": 0.1832,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.08011925220489502,
      "rewards/margins": 0.03867987170815468,
      "rewards/rejected": -0.1187991127371788,
      "step": 180
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.8417076618132434e-06,
      "logits/chosen": -0.012808884494006634,
      "logits/rejected": 0.07382214069366455,
      "logps/chosen": -470.2506408691406,
      "logps/rejected": -483.8114318847656,
      "loss": 0.2019,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.08619646728038788,
      "rewards/margins": 0.021857351064682007,
      "rewards/rejected": -0.10805382579565048,
      "step": 190
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.807464225455655e-06,
      "logits/chosen": 0.11153273284435272,
      "logits/rejected": 0.0497104674577713,
      "logps/chosen": -451.3377380371094,
      "logps/rejected": -478.8421936035156,
      "loss": 0.1942,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.0883895754814148,
      "rewards/margins": 0.030626222491264343,
      "rewards/rejected": -0.11901579797267914,
      "step": 200
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.770016517582283e-06,
      "logits/chosen": 0.08737102895975113,
      "logits/rejected": 0.11837353557348251,
      "logps/chosen": -519.548828125,
      "logps/rejected": -513.87255859375,
      "loss": 0.1924,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.11379089206457138,
      "rewards/margins": 0.03358360007405281,
      "rewards/rejected": -0.1473744958639145,
      "step": 210
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.7294165401363616e-06,
      "logits/chosen": 0.14104892313480377,
      "logits/rejected": 0.05087919905781746,
      "logps/chosen": -525.2891235351562,
      "logps/rejected": -516.625,
      "loss": 0.2024,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.10455243289470673,
      "rewards/margins": 0.0242080669850111,
      "rewards/rejected": -0.12876050174236298,
      "step": 220
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.68572067247573e-06,
      "logits/chosen": 0.09732341766357422,
      "logits/rejected": 0.06271795928478241,
      "logps/chosen": -514.5999755859375,
      "logps/rejected": -543.45068359375,
      "loss": 0.2006,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.1029946357011795,
      "rewards/margins": 0.033023733645677567,
      "rewards/rejected": -0.13601836562156677,
      "step": 230
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.638989593081364e-06,
      "logits/chosen": 0.06812410056591034,
      "logits/rejected": 0.025554204359650612,
      "logps/chosen": -516.6173095703125,
      "logps/rejected": -508.93511962890625,
      "loss": 0.2079,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.0961836650967598,
      "rewards/margins": 0.020848505198955536,
      "rewards/rejected": -0.11703218519687653,
      "step": 240
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.5892881952959015e-06,
      "logits/chosen": 0.052641429007053375,
      "logits/rejected": 0.11277129501104355,
      "logps/chosen": -499.9371032714844,
      "logps/rejected": -524.9378662109375,
      "loss": 0.2003,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.11769168078899384,
      "rewards/margins": 0.02954215370118618,
      "rewards/rejected": -0.14723384380340576,
      "step": 250
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.536685497209182e-06,
      "logits/chosen": 0.1458149254322052,
      "logits/rejected": 0.18058298528194427,
      "logps/chosen": -461.63262939453125,
      "logps/rejected": -501.23077392578125,
      "loss": 0.195,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -0.1235838383436203,
      "rewards/margins": 0.02654830738902092,
      "rewards/rejected": -0.15013214945793152,
      "step": 260
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.481254545815943e-06,
      "logits/chosen": 0.07014019787311554,
      "logits/rejected": 0.16694627702236176,
      "logps/chosen": -472.4525451660156,
      "logps/rejected": -473.22857666015625,
      "loss": 0.1957,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.12277780473232269,
      "rewards/margins": 0.01745077781379223,
      "rewards/rejected": -0.14022858440876007,
      "step": 270
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.42307231557875e-06,
      "logits/chosen": 0.11594029515981674,
      "logits/rejected": 0.14667686820030212,
      "logps/chosen": -454.9984436035156,
      "logps/rejected": -468.27764892578125,
      "loss": 0.1927,
      "rewards/accuracies": 0.4312500059604645,
      "rewards/chosen": -0.14300423860549927,
      "rewards/margins": 0.037807125598192215,
      "rewards/rejected": -0.18081137537956238,
      "step": 280
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.3622196015370305e-06,
      "logits/chosen": 0.08191246539354324,
      "logits/rejected": 0.13855530321598053,
      "logps/chosen": -496.906982421875,
      "logps/rejected": -526.3932495117188,
      "loss": 0.1933,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.12783119082450867,
      "rewards/margins": 0.03418290615081787,
      "rewards/rejected": -0.16201409697532654,
      "step": 290
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.298780907110648e-06,
      "logits/chosen": 0.10729117691516876,
      "logits/rejected": 0.1382167488336563,
      "logps/chosen": -525.2513427734375,
      "logps/rejected": -557.9173583984375,
      "loss": 0.1848,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.10007178783416748,
      "rewards/margins": 0.03654087707400322,
      "rewards/rejected": -0.136612668633461,
      "step": 300
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.23284432675381e-06,
      "logits/chosen": 0.03605717048048973,
      "logits/rejected": 0.08247358351945877,
      "logps/chosen": -476.7731018066406,
      "logps/rejected": -517.1722412109375,
      "loss": 0.1901,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.0743183121085167,
      "rewards/margins": 0.028351595625281334,
      "rewards/rejected": -0.10266990959644318,
      "step": 310
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.164501423622277e-06,
      "logits/chosen": 0.03332200273871422,
      "logits/rejected": 0.09907984733581543,
      "logps/chosen": -449.68212890625,
      "logps/rejected": -509.12554931640625,
      "loss": 0.1926,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.05537894368171692,
      "rewards/margins": 0.05774994567036629,
      "rewards/rejected": -0.11312887817621231,
      "step": 320
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.0938471024237355e-06,
      "logits/chosen": 0.05158273130655289,
      "logits/rejected": 0.10417857021093369,
      "logps/chosen": -498.09197998046875,
      "logps/rejected": -511.01611328125,
      "loss": 0.1957,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.06420507282018661,
      "rewards/margins": 0.012455371208488941,
      "rewards/rejected": -0.07666043937206268,
      "step": 330
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.020979477627907e-06,
      "logits/chosen": 0.015417411923408508,
      "logits/rejected": 0.09142233431339264,
      "logps/chosen": -502.4671936035156,
      "logps/rejected": -528.1393432617188,
      "loss": 0.187,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.0401633158326149,
      "rewards/margins": 0.03752883896231651,
      "rewards/rejected": -0.07769216597080231,
      "step": 340
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.9459997372194105e-06,
      "logits/chosen": 0.07565109431743622,
      "logits/rejected": 0.14677348732948303,
      "logps/chosen": -490.3994140625,
      "logps/rejected": -483.0618591308594,
      "loss": 0.182,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.000983755337074399,
      "rewards/margins": 0.04362579435110092,
      "rewards/rejected": -0.04460955038666725,
      "step": 350
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.869012002182573e-06,
      "logits/chosen": -0.012587164528667927,
      "logits/rejected": 0.17615702748298645,
      "logps/chosen": -460.7539978027344,
      "logps/rejected": -496.0394592285156,
      "loss": 0.1765,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.004288173280656338,
      "rewards/margins": 0.045940086245536804,
      "rewards/rejected": -0.05022825673222542,
      "step": 360
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.7901231819133104e-06,
      "logits/chosen": 0.05247374624013901,
      "logits/rejected": 0.06078363209962845,
      "logps/chosen": -500.44384765625,
      "logps/rejected": -511.1734313964844,
      "loss": 0.189,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.0024426232557743788,
      "rewards/margins": 0.04215018078684807,
      "rewards/rejected": -0.03970755636692047,
      "step": 370
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.709442825758875e-06,
      "logits/chosen": 0.04354017972946167,
      "logits/rejected": 0.12157300859689713,
      "logps/chosen": -477.10601806640625,
      "logps/rejected": -487.3301696777344,
      "loss": 0.1834,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.022890402004122734,
      "rewards/margins": 0.03842080011963844,
      "rewards/rejected": -0.01553039439022541,
      "step": 380
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.6270829708916113e-06,
      "logits/chosen": 0.04744056984782219,
      "logits/rejected": 0.07940800487995148,
      "logps/chosen": -472.5918884277344,
      "logps/rejected": -496.16387939453125,
      "loss": 0.1833,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.019969001412391663,
      "rewards/margins": 0.04812890291213989,
      "rewards/rejected": -0.02815990522503853,
      "step": 390
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.543157986727991e-06,
      "logits/chosen": 0.04268989711999893,
      "logits/rejected": 0.07916353642940521,
      "logps/chosen": -486.46746826171875,
      "logps/rejected": -520.3671875,
      "loss": 0.1851,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": 0.029743488878011703,
      "rewards/margins": 0.04279039055109024,
      "rewards/rejected": -0.013046903535723686,
      "step": 400
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4577844161089614e-06,
      "logits/chosen": -0.027468422427773476,
      "logits/rejected": -0.04074520990252495,
      "logps/chosen": -461.34814453125,
      "logps/rejected": -480.4091796875,
      "loss": 0.1842,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": 0.058516185730695724,
      "rewards/margins": 0.04239815101027489,
      "rewards/rejected": 0.016118034720420837,
      "step": 410
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3710808134621577e-06,
      "logits/chosen": 0.029957780614495277,
      "logits/rejected": 0.1293373703956604,
      "logps/chosen": -476.6888732910156,
      "logps/rejected": -474.581298828125,
      "loss": 0.1863,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": 0.06844775378704071,
      "rewards/margins": 0.0345512256026268,
      "rewards/rejected": 0.03389652818441391,
      "step": 420
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2831675801707126e-06,
      "logits/chosen": 0.12380461394786835,
      "logits/rejected": 0.08030427992343903,
      "logps/chosen": -501.05419921875,
      "logps/rejected": -540.5864868164062,
      "loss": 0.1833,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.08108590543270111,
      "rewards/margins": 0.04042986035346985,
      "rewards/rejected": 0.040656037628650665,
      "step": 430
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.194166797377289e-06,
      "logits/chosen": 0.07574252784252167,
      "logits/rejected": -0.00224049249663949,
      "logps/chosen": -482.531982421875,
      "logps/rejected": -499.04180908203125,
      "loss": 0.1822,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.0877564549446106,
      "rewards/margins": 0.03748675435781479,
      "rewards/rejected": 0.05026970058679581,
      "step": 440
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.104202056455501e-06,
      "logits/chosen": 0.10215386003255844,
      "logits/rejected": 0.025258684530854225,
      "logps/chosen": -455.39825439453125,
      "logps/rejected": -464.11016845703125,
      "loss": 0.1929,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": 0.11565859615802765,
      "rewards/margins": 0.04002625495195389,
      "rewards/rejected": 0.07563233375549316,
      "step": 450
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.013398287384144e-06,
      "logits/chosen": 0.07208961248397827,
      "logits/rejected": 0.05293772369623184,
      "logps/chosen": -422.0474548339844,
      "logps/rejected": -466.1214904785156,
      "loss": 0.1867,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.13345687091350555,
      "rewards/margins": 0.04373367875814438,
      "rewards/rejected": 0.08972318470478058,
      "step": 460
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9218815852625717e-06,
      "logits/chosen": 0.08753691613674164,
      "logits/rejected": 0.13366851210594177,
      "logps/chosen": -493.6244201660156,
      "logps/rejected": -481.61505126953125,
      "loss": 0.1971,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.15995556116104126,
      "rewards/margins": 0.030286097899079323,
      "rewards/rejected": 0.1296694576740265,
      "step": 470
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.829779035208113e-06,
      "logits/chosen": 0.040917035192251205,
      "logits/rejected": 0.049655430018901825,
      "logps/chosen": -496.69207763671875,
      "logps/rejected": -510.538818359375,
      "loss": 0.1827,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.16031843423843384,
      "rewards/margins": 0.047999344766139984,
      "rewards/rejected": 0.11231908947229385,
      "step": 480
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.737218535878705e-06,
      "logits/chosen": -0.02139192819595337,
      "logits/rejected": 0.07283684611320496,
      "logps/chosen": -455.2607421875,
      "logps/rejected": -484.5707092285156,
      "loss": 0.2065,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.13160817325115204,
      "rewards/margins": 0.03165828809142113,
      "rewards/rejected": 0.09994987398386002,
      "step": 490
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.64432862186579e-06,
      "logits/chosen": 0.043560974299907684,
      "logits/rejected": 0.0819135308265686,
      "logps/chosen": -448.69793701171875,
      "logps/rejected": -472.7060546875,
      "loss": 0.1844,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.10212703794240952,
      "rewards/margins": 0.038218505680561066,
      "rewards/rejected": 0.06390853226184845,
      "step": 500
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.551238285204126e-06,
      "logits/chosen": 0.005092448554933071,
      "logits/rejected": 0.10958313941955566,
      "logps/chosen": -484.2552185058594,
      "logps/rejected": -514.481689453125,
      "loss": 0.1942,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": 0.09142176806926727,
      "rewards/margins": 0.05243723839521408,
      "rewards/rejected": 0.038984514772892,
      "step": 510
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.4580767962463688e-06,
      "logits/chosen": 0.09447602182626724,
      "logits/rejected": 0.08024080842733383,
      "logps/chosen": -485.6512756347656,
      "logps/rejected": -482.0237731933594,
      "loss": 0.1891,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.07477296888828278,
      "rewards/margins": 0.05883296579122543,
      "rewards/rejected": 0.015940004959702492,
      "step": 520
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3649735241511546e-06,
      "logits/chosen": 0.03767072409391403,
      "logits/rejected": 0.02741827443242073,
      "logps/chosen": -440.016357421875,
      "logps/rejected": -472.63873291015625,
      "loss": 0.1917,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.08402599394321442,
      "rewards/margins": 0.061398398131132126,
      "rewards/rejected": 0.02262760140001774,
      "step": 530
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2720577572339914e-06,
      "logits/chosen": -0.00708337128162384,
      "logits/rejected": 0.055642373859882355,
      "logps/chosen": -446.92498779296875,
      "logps/rejected": -466.16912841796875,
      "loss": 0.1827,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": 0.07480621337890625,
      "rewards/margins": 0.033216286450624466,
      "rewards/rejected": 0.04158991575241089,
      "step": 540
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1794585234303995e-06,
      "logits/chosen": 0.035645630210638046,
      "logits/rejected": 0.019252488389611244,
      "logps/chosen": -421.6669921875,
      "logps/rejected": -462.30279541015625,
      "loss": 0.1782,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.08298784494400024,
      "rewards/margins": 0.05089961364865303,
      "rewards/rejected": 0.032088227570056915,
      "step": 550
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0873044111206407e-06,
      "logits/chosen": 0.03386590629816055,
      "logits/rejected": 0.0352010652422905,
      "logps/chosen": -497.4388732910156,
      "logps/rejected": -539.6199951171875,
      "loss": 0.184,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.07756824791431427,
      "rewards/margins": 0.06193417310714722,
      "rewards/rejected": 0.01563408225774765,
      "step": 560
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.9957233905648293e-06,
      "logits/chosen": 0.06176334619522095,
      "logits/rejected": 0.05909004807472229,
      "logps/chosen": -479.58502197265625,
      "logps/rejected": -504.15118408203125,
      "loss": 0.179,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": 0.07643002271652222,
      "rewards/margins": 0.055131517350673676,
      "rewards/rejected": 0.021298503503203392,
      "step": 570
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.904842636196402e-06,
      "logits/chosen": 0.10232093185186386,
      "logits/rejected": 0.03354436904191971,
      "logps/chosen": -504.35552978515625,
      "logps/rejected": -500.55242919921875,
      "loss": 0.188,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0535406656563282,
      "rewards/margins": 0.045547787100076675,
      "rewards/rejected": 0.007992876693606377,
      "step": 580
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.814788350020726e-06,
      "logits/chosen": 0.08256170898675919,
      "logits/rejected": 0.011359182186424732,
      "logps/chosen": -423.5184631347656,
      "logps/rejected": -462.12921142578125,
      "loss": 0.1847,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.06936874985694885,
      "rewards/margins": 0.04546703025698662,
      "rewards/rejected": 0.023901721462607384,
      "step": 590
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.725685586364051e-06,
      "logits/chosen": 0.05957316234707832,
      "logits/rejected": 0.02687280997633934,
      "logps/chosen": -463.92340087890625,
      "logps/rejected": -488.68524169921875,
      "loss": 0.1841,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.093163400888443,
      "rewards/margins": 0.05278082564473152,
      "rewards/rejected": 0.04038257524371147,
      "step": 600
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6376580782162172e-06,
      "logits/chosen": 0.03663691133260727,
      "logits/rejected": 0.06973805278539658,
      "logps/chosen": -460.896728515625,
      "logps/rejected": -508.50408935546875,
      "loss": 0.1778,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": 0.0800352543592453,
      "rewards/margins": 0.05085815116763115,
      "rewards/rejected": 0.029177110642194748,
      "step": 610
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.550828065408227e-06,
      "logits/chosen": 0.028588274493813515,
      "logits/rejected": 0.081044040620327,
      "logps/chosen": -504.3495178222656,
      "logps/rejected": -515.7532958984375,
      "loss": 0.1676,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.07437646389007568,
      "rewards/margins": 0.04987211152911186,
      "rewards/rejected": 0.02450435608625412,
      "step": 620
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4653161248633053e-06,
      "logits/chosen": 0.03259013965725899,
      "logits/rejected": 0.0049851625226438046,
      "logps/chosen": -485.429443359375,
      "logps/rejected": -495.66802978515625,
      "loss": 0.1771,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.07386799156665802,
      "rewards/margins": 0.05358237773180008,
      "rewards/rejected": 0.02028561569750309,
      "step": 630
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.381241003157162e-06,
      "logits/chosen": 0.040917474776506424,
      "logits/rejected": 0.03524303436279297,
      "logps/chosen": -474.40252685546875,
      "logps/rejected": -492.69415283203125,
      "loss": 0.1848,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.08967775106430054,
      "rewards/margins": 0.04113636165857315,
      "rewards/rejected": 0.04854138568043709,
      "step": 640
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.298719451619979e-06,
      "logits/chosen": -0.0005787253612652421,
      "logits/rejected": 0.05223802477121353,
      "logps/chosen": -469.7098693847656,
      "logps/rejected": -505.01739501953125,
      "loss": 0.1979,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.08548668771982193,
      "rewards/margins": 0.057396382093429565,
      "rewards/rejected": 0.028090301901102066,
      "step": 650
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2178660642091036e-06,
      "logits/chosen": 0.09496835619211197,
      "logits/rejected": -0.0715503841638565,
      "logps/chosen": -422.630859375,
      "logps/rejected": -483.22479248046875,
      "loss": 0.1824,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.08553356677293777,
      "rewards/margins": 0.059073131531476974,
      "rewards/rejected": 0.02646043337881565,
      "step": 660
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1387931183775821e-06,
      "logits/chosen": -0.00498700188472867,
      "logits/rejected": 0.013168242760002613,
      "logps/chosen": -445.13861083984375,
      "logps/rejected": -466.5567932128906,
      "loss": 0.1776,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.10062942653894424,
      "rewards/margins": 0.04562296345829964,
      "rewards/rejected": 0.055006466805934906,
      "step": 670
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.061610419159532e-06,
      "logits/chosen": 0.0634564682841301,
      "logits/rejected": 0.03239821270108223,
      "logps/chosen": -454.5580139160156,
      "logps/rejected": -471.24456787109375,
      "loss": 0.1788,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.11137165874242783,
      "rewards/margins": 0.06377272307872772,
      "rewards/rejected": 0.0475989505648613,
      "step": 680
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.864251466888364e-07,
      "logits/chosen": 0.13198313117027283,
      "logits/rejected": -0.011327924206852913,
      "logps/chosen": -447.65301513671875,
      "logps/rejected": -485.073974609375,
      "loss": 0.1763,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.11758910119533539,
      "rewards/margins": 0.05051087588071823,
      "rewards/rejected": 0.06707822531461716,
      "step": 690
    },
    {
      "epoch": 0.75,
      "learning_rate": 9.133417073629288e-07,
      "logits/chosen": 0.01844317652285099,
      "logits/rejected": 0.046270035207271576,
      "logps/chosen": -461.8832092285156,
      "logps/rejected": -497.8064880371094,
      "loss": 0.1902,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": 0.11230479180812836,
      "rewards/margins": 0.038510359823703766,
      "rewards/rejected": 0.07379443198442459,
      "step": 700
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.424615888583332e-07,
      "logits/chosen": -0.0005433745682239532,
      "logits/rejected": 0.07239210605621338,
      "logps/chosen": -440.7948303222656,
      "logps/rejected": -482.3118591308594,
      "loss": 0.1805,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.11701339483261108,
      "rewards/margins": 0.05948741361498833,
      "rewards/rejected": 0.05752598121762276,
      "step": 710
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.738832191993092e-07,
      "logits/chosen": 0.004731953144073486,
      "logits/rejected": 0.06949619203805923,
      "logps/chosen": -495.1181640625,
      "logps/rejected": -494.14727783203125,
      "loss": 0.1907,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.11087504774332047,
      "rewards/margins": 0.05067601799964905,
      "rewards/rejected": 0.060199033468961716,
      "step": 720
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.077018300752917e-07,
      "logits/chosen": 0.05472816899418831,
      "logits/rejected": 0.12369577586650848,
      "logps/chosen": -454.3465270996094,
      "logps/rejected": -466.5943298339844,
      "loss": 0.1862,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.09320452064275742,
      "rewards/margins": 0.04919914901256561,
      "rewards/rejected": 0.0440053753554821,
      "step": 730
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.440093245969342e-07,
      "logits/chosen": 0.049330659210681915,
      "logits/rejected": -0.05037214607000351,
      "logps/chosen": -426.7115173339844,
      "logps/rejected": -468.99169921875,
      "loss": 0.178,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": 0.1173645481467247,
      "rewards/margins": 0.06448864191770554,
      "rewards/rejected": 0.05287591367959976,
      "step": 740
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.828941496744075e-07,
      "logits/chosen": 0.018861666321754456,
      "logits/rejected": 0.04406842216849327,
      "logps/chosen": -471.93438720703125,
      "logps/rejected": -496.49993896484375,
      "loss": 0.1862,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.11358505487442017,
      "rewards/margins": 0.046424929052591324,
      "rewards/rejected": 0.06716012954711914,
      "step": 750
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.244411731951671e-07,
      "logits/chosen": 0.009539002552628517,
      "logits/rejected": 0.11306466907262802,
      "logps/chosen": -520.7066650390625,
      "logps/rejected": -506.4190368652344,
      "loss": 0.1811,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.11439801752567291,
      "rewards/margins": 0.042585574090480804,
      "rewards/rejected": 0.07181242108345032,
      "step": 760
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.6873156617173594e-07,
      "logits/chosen": 0.05899236351251602,
      "logits/rejected": -0.012183750048279762,
      "logps/chosen": -465.54583740234375,
      "logps/rejected": -492.99627685546875,
      "loss": 0.1822,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.11408071219921112,
      "rewards/margins": 0.05036580562591553,
      "rewards/rejected": 0.06371492147445679,
      "step": 770
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.1584269002318653e-07,
      "logits/chosen": 0.04782986640930176,
      "logits/rejected": 0.0820665955543518,
      "logps/chosen": -446.2923278808594,
      "logps/rejected": -473.13330078125,
      "loss": 0.1747,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": 0.10825562477111816,
      "rewards/margins": 0.04990454763174057,
      "rewards/rejected": 0.058351077139377594,
      "step": 780
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.658479891468258e-07,
      "logits/chosen": -0.04199203476309776,
      "logits/rejected": 0.034929946064949036,
      "logps/chosen": -446.6978454589844,
      "logps/rejected": -435.03521728515625,
      "loss": 0.1749,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": 0.09576048702001572,
      "rewards/margins": 0.045824047178030014,
      "rewards/rejected": 0.0499364398419857,
      "step": 790
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.18816888929272e-07,
      "logits/chosen": 0.026381831616163254,
      "logits/rejected": 0.07602212578058243,
      "logps/chosen": -467.7432556152344,
      "logps/rejected": -516.2023315429688,
      "loss": 0.1863,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.11635438352823257,
      "rewards/margins": 0.06682081520557404,
      "rewards/rejected": 0.04953356832265854,
      "step": 800
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.748146993385484e-07,
      "logits/chosen": 0.04555773735046387,
      "logits/rejected": 0.0836203470826149,
      "logps/chosen": -432.5563049316406,
      "logps/rejected": -469.57196044921875,
      "loss": 0.1816,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.11761243641376495,
      "rewards/margins": 0.05147726088762283,
      "rewards/rejected": 0.06613515317440033,
      "step": 810
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.3390252423108077e-07,
      "logits/chosen": 0.06100771576166153,
      "logits/rejected": -0.052108317613601685,
      "logps/chosen": -411.35906982421875,
      "logps/rejected": -439.3312072753906,
      "loss": 0.1838,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.12352752685546875,
      "rewards/margins": 0.057254206389188766,
      "rewards/rejected": 0.06627332419157028,
      "step": 820
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.961371764995243e-07,
      "logits/chosen": 0.0011749922996386886,
      "logits/rejected": -0.003739739302545786,
      "logps/chosen": -458.92919921875,
      "logps/rejected": -491.05615234375,
      "loss": 0.1983,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.11504600197076797,
      "rewards/margins": 0.04097086191177368,
      "rewards/rejected": 0.0740751400589943,
      "step": 830
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.61571099179261e-07,
      "logits/chosen": 0.06734688580036163,
      "logits/rejected": 0.08023635298013687,
      "logps/chosen": -494.57342529296875,
      "logps/rejected": -526.6209716796875,
      "loss": 0.1857,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.1077314019203186,
      "rewards/margins": 0.04536820203065872,
      "rewards/rejected": 0.06236319988965988,
      "step": 840
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.3025229262312367e-07,
      "logits/chosen": 0.03414052724838257,
      "logits/rejected": 0.0967903584241867,
      "logps/chosen": -406.6197204589844,
      "logps/rejected": -463.4610900878906,
      "loss": 0.1849,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": 0.12588708102703094,
      "rewards/margins": 0.05215496942400932,
      "rewards/rejected": 0.07373212277889252,
      "step": 850
    },
    {
      "epoch": 0.92,
      "learning_rate": 1.0222424784546853e-07,
      "logits/chosen": 0.046077877283096313,
      "logits/rejected": -0.010932741686701775,
      "logps/chosen": -486.12933349609375,
      "logps/rejected": -490.4742736816406,
      "loss": 0.1796,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": 0.11952760070562363,
      "rewards/margins": 0.052067436277866364,
      "rewards/rejected": 0.06746017187833786,
      "step": 860
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.752588612816553e-08,
      "logits/chosen": 0.08364848792552948,
      "logits/rejected": -0.016565704718232155,
      "logps/chosen": -421.2782287597656,
      "logps/rejected": -448.32061767578125,
      "loss": 0.1673,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": 0.12084163725376129,
      "rewards/margins": 0.05717577412724495,
      "rewards/rejected": 0.06366586685180664,
      "step": 870
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.619150497236991e-08,
      "logits/chosen": 0.05045370012521744,
      "logits/rejected": -0.015437754802405834,
      "logps/chosen": -436.0224609375,
      "logps/rejected": -480.992431640625,
      "loss": 0.1805,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.1131979450583458,
      "rewards/margins": 0.05461297556757927,
      "rewards/rejected": 0.058584969490766525,
      "step": 880
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.825073047112743e-08,
      "logits/chosen": 0.0034905076026916504,
      "logits/rejected": 0.10833680629730225,
      "logps/chosen": -497.405517578125,
      "logps/rejected": -531.6438598632812,
      "loss": 0.1902,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": 0.10887560993432999,
      "rewards/margins": 0.053155817091464996,
      "rewards/rejected": 0.05571979284286499,
      "step": 890
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.372847616895685e-08,
      "logits/chosen": 0.09158419072628021,
      "logits/rejected": 0.12686944007873535,
      "logps/chosen": -453.55206298828125,
      "logps/rejected": -507.8880920410156,
      "loss": 0.1797,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.11705918610095978,
      "rewards/margins": 0.061758268624544144,
      "rewards/rejected": 0.055300913751125336,
      "step": 900
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.264490846553279e-08,
      "logits/chosen": -0.005747838411480188,
      "logits/rejected": 0.02690596505999565,
      "logps/chosen": -485.0928649902344,
      "logps/rejected": -488.77349853515625,
      "loss": 0.1824,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.10394562780857086,
      "rewards/margins": 0.04496193677186966,
      "rewards/rejected": 0.058983683586120605,
      "step": 910
    },
    {
      "epoch": 0.98,
      "learning_rate": 5.015418611516165e-09,
      "logits/chosen": 0.04409351199865341,
      "logits/rejected": 0.023466479033231735,
      "logps/chosen": -516.257080078125,
      "logps/rejected": -529.1635131835938,
      "loss": 0.186,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.11666563898324966,
      "rewards/margins": 0.0654011145234108,
      "rewards/rejected": 0.051264531910419464,
      "step": 920
    },
    {
      "epoch": 0.99,
      "learning_rate": 8.506013354186993e-10,
      "logits/chosen": 0.029077816754579544,
      "logits/rejected": 0.1013563722372055,
      "logps/chosen": -449.57379150390625,
      "logps/rejected": -469.8511657714844,
      "loss": 0.1967,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": 0.10130287706851959,
      "rewards/margins": 0.050037987530231476,
      "rewards/rejected": 0.05126488208770752,
      "step": 930
    },
    {
      "epoch": 1.0,
      "step": 937,
      "total_flos": 0.0,
      "train_loss": 0.1892308688214647,
      "train_runtime": 7960.6994,
      "train_samples_per_second": 3.769,
      "train_steps_per_second": 0.118
    }
  ],
  "logging_steps": 10,
  "max_steps": 937,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}