{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 625,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 7.936507936507937e-08,
      "logits/chosen": 0.16854308545589447,
      "logits/rejected": 0.17400208115577698,
      "logps/chosen": -327.90460205078125,
      "logps/rejected": -253.07960510253906,
      "loss": 0.4067,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.02,
      "learning_rate": 7.936507936507937e-07,
      "logits/chosen": 0.08604434132575989,
      "logits/rejected": 0.1809283345937729,
      "logps/chosen": -327.1376647949219,
      "logps/rejected": -306.0588684082031,
      "loss": 0.3864,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -5.549057823373005e-05,
      "rewards/margins": -0.0001069721911335364,
      "rewards/rejected": 5.148161653778516e-05,
      "step": 10
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.5873015873015873e-06,
      "logits/chosen": 0.08089748024940491,
      "logits/rejected": 0.2491210699081421,
      "logps/chosen": -362.65435791015625,
      "logps/rejected": -352.3981018066406,
      "loss": 0.3729,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": 2.957274773507379e-05,
      "rewards/margins": 5.038862582296133e-05,
      "rewards/rejected": -2.0815870811929926e-05,
      "step": 20
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.380952380952381e-06,
      "logits/chosen": 0.18664827942848206,
      "logits/rejected": 0.14790232479572296,
      "logps/chosen": -312.6676940917969,
      "logps/rejected": -324.85516357421875,
      "loss": 0.3789,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 8.431552851106972e-05,
      "rewards/margins": 0.00019130071450490505,
      "rewards/rejected": -0.00010698519326979294,
      "step": 30
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.1746031746031746e-06,
      "logits/chosen": 0.08645667880773544,
      "logits/rejected": 0.2045074999332428,
      "logps/chosen": -347.3286437988281,
      "logps/rejected": -375.4449768066406,
      "loss": 0.3765,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.0001862197823356837,
      "rewards/margins": 6.710310117341578e-05,
      "rewards/rejected": -0.000253322854405269,
      "step": 40
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.968253968253968e-06,
      "logits/chosen": 0.11011010408401489,
      "logits/rejected": 0.1821037232875824,
      "logps/chosen": -385.66693115234375,
      "logps/rejected": -349.8896789550781,
      "loss": 0.3738,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 4.727130362880416e-05,
      "rewards/margins": 0.00021036244288552552,
      "rewards/rejected": -0.00016309114289470017,
      "step": 50
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.761904761904762e-06,
      "logits/chosen": 0.07650097459554672,
      "logits/rejected": 0.177745059132576,
      "logps/chosen": -343.5960998535156,
      "logps/rejected": -331.03912353515625,
      "loss": 0.3796,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.0009819746483117342,
      "rewards/margins": 0.0015515821287408471,
      "rewards/rejected": -0.0005696073058061302,
      "step": 60
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.998086282661188e-06,
      "logits/chosen": 0.11013742536306381,
      "logits/rejected": 0.16680167615413666,
      "logps/chosen": -361.571044921875,
      "logps/rejected": -318.3884582519531,
      "loss": 0.3614,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.0014994305092841387,
      "rewards/margins": 0.0018448818009346724,
      "rewards/rejected": -0.00034545123344287276,
      "step": 70
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.988720025682995e-06,
      "logits/chosen": 0.1778658628463745,
      "logits/rejected": 0.19966575503349304,
      "logps/chosen": -292.3515625,
      "logps/rejected": -321.511962890625,
      "loss": 0.3511,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0031275060027837753,
      "rewards/margins": 0.0046959021128714085,
      "rewards/rejected": -0.0015683959936723113,
      "step": 80
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.9715789537359126e-06,
      "logits/chosen": 0.14186400175094604,
      "logits/rejected": 0.18624775111675262,
      "logps/chosen": -360.6056823730469,
      "logps/rejected": -317.9211730957031,
      "loss": 0.3683,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.004151093773543835,
      "rewards/margins": 0.006570989731699228,
      "rewards/rejected": -0.002419896889477968,
      "step": 90
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.946716615897932e-06,
      "logits/chosen": 0.11935708671808243,
      "logits/rejected": 0.18251103162765503,
      "logps/chosen": -354.6888732910156,
      "logps/rejected": -341.3345642089844,
      "loss": 0.3703,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.0003122333437204361,
      "rewards/margins": 0.010582786984741688,
      "rewards/rejected": -0.010895021259784698,
      "step": 100
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.9142106826480114e-06,
      "logits/chosen": 0.06461165845394135,
      "logits/rejected": 0.18685249984264374,
      "logps/chosen": -362.90142822265625,
      "logps/rejected": -353.4388122558594,
      "loss": 0.3406,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.007556666620075703,
      "rewards/margins": 0.0255865640938282,
      "rewards/rejected": -0.03314322978258133,
      "step": 110
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.874162703221823e-06,
      "logits/chosen": 0.07998766005039215,
      "logits/rejected": 0.17009273171424866,
      "logps/chosen": -389.65362548828125,
      "logps/rejected": -363.02374267578125,
      "loss": 0.3605,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.036845412105321884,
      "rewards/margins": 0.020548993721604347,
      "rewards/rejected": -0.05739440768957138,
      "step": 120
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.826697788369752e-06,
      "logits/chosen": 0.08132342249155045,
      "logits/rejected": 0.14768512547016144,
      "logps/chosen": -431.18109130859375,
      "logps/rejected": -458.8611755371094,
      "loss": 0.3284,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.06240760162472725,
      "rewards/margins": 0.06149229407310486,
      "rewards/rejected": -0.12389989197254181,
      "step": 130
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.7719642195082224e-06,
      "logits/chosen": 0.07962881028652191,
      "logits/rejected": 0.07184718549251556,
      "logps/chosen": -449.84393310546875,
      "logps/rejected": -531.15087890625,
      "loss": 0.2981,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.13113334774971008,
      "rewards/margins": 0.08747557550668716,
      "rewards/rejected": -0.21860890090465546,
      "step": 140
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.710132985485355e-06,
      "logits/chosen": 0.048229388892650604,
      "logits/rejected": 0.13614359498023987,
      "logps/chosen": -528.4464721679688,
      "logps/rejected": -578.7511596679688,
      "loss": 0.3243,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.18633897602558136,
      "rewards/margins": 0.07957583665847778,
      "rewards/rejected": -0.26591476798057556,
      "step": 150
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.641397248408122e-06,
      "logits/chosen": 0.11920128762722015,
      "logits/rejected": 0.15292052924633026,
      "logps/chosen": -552.9255981445312,
      "logps/rejected": -630.448486328125,
      "loss": 0.3063,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.1733545958995819,
      "rewards/margins": 0.08863510191440582,
      "rewards/rejected": -0.2619897127151489,
      "step": 160
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.5659717401997655e-06,
      "logits/chosen": 0.06991209089756012,
      "logits/rejected": 0.12328378856182098,
      "logps/chosen": -470.7796325683594,
      "logps/rejected": -537.2589721679688,
      "loss": 0.3181,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.14576277136802673,
      "rewards/margins": 0.07790832221508026,
      "rewards/rejected": -0.2236711084842682,
      "step": 170
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.4840920917726425e-06,
      "logits/chosen": 0.128515362739563,
      "logits/rejected": 0.18245835602283478,
      "logps/chosen": -488.5677185058594,
      "logps/rejected": -617.6085205078125,
      "loss": 0.3053,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.15714891254901886,
      "rewards/margins": 0.1121206283569336,
      "rewards/rejected": -0.26926952600479126,
      "step": 180
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.396014096912182e-06,
      "logits/chosen": 0.06968175619840622,
      "logits/rejected": 0.1405131071805954,
      "logps/chosen": -457.48687744140625,
      "logps/rejected": -507.72491455078125,
      "loss": 0.3217,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.144833505153656,
      "rewards/margins": 0.07206179946660995,
      "rewards/rejected": -0.21689531207084656,
      "step": 190
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.302012913171584e-06,
      "logits/chosen": 0.11664710193872452,
      "logits/rejected": 0.1742321252822876,
      "logps/chosen": -498.94708251953125,
      "logps/rejected": -543.8130493164062,
      "loss": 0.3178,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.1338798701763153,
      "rewards/margins": 0.07743647694587708,
      "rewards/rejected": -0.21131634712219238,
      "step": 200
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.202382202273702e-06,
      "logits/chosen": 0.09494765847921371,
      "logits/rejected": 0.1851937472820282,
      "logps/chosen": -520.176025390625,
      "logps/rejected": -567.4844970703125,
      "loss": 0.3061,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.14064568281173706,
      "rewards/margins": 0.08944287896156311,
      "rewards/rejected": -0.23008854687213898,
      "step": 210
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.097433212705492e-06,
      "logits/chosen": 0.12784181535243988,
      "logits/rejected": 0.12376650422811508,
      "logps/chosen": -478.936767578125,
      "logps/rejected": -583.5801391601562,
      "loss": 0.3215,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.16416385769844055,
      "rewards/margins": 0.08470286428928375,
      "rewards/rejected": -0.2488667219877243,
      "step": 220
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.987493807371033e-06,
      "logits/chosen": 0.14440543949604034,
      "logits/rejected": 0.19996806979179382,
      "logps/chosen": -516.4754028320312,
      "logps/rejected": -626.3532104492188,
      "loss": 0.2979,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.1449437290430069,
      "rewards/margins": 0.12377123534679413,
      "rewards/rejected": -0.268714964389801,
      "step": 230
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.872907439340758e-06,
      "logits/chosen": 0.17143338918685913,
      "logits/rejected": 0.19446048140525818,
      "logps/chosen": -530.1805419921875,
      "logps/rejected": -642.0897827148438,
      "loss": 0.3007,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.1711062490940094,
      "rewards/margins": 0.11968924105167389,
      "rewards/rejected": -0.2907955050468445,
      "step": 240
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.75403207889666e-06,
      "logits/chosen": 0.13282717764377594,
      "logits/rejected": 0.22809620201587677,
      "logps/chosen": -522.2652587890625,
      "logps/rejected": -587.6581420898438,
      "loss": 0.3026,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.16960139572620392,
      "rewards/margins": 0.09930792450904846,
      "rewards/rejected": -0.2689093053340912,
      "step": 250
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.631239095225417e-06,
      "logits/chosen": 0.17684991657733917,
      "logits/rejected": 0.17547938227653503,
      "logps/chosen": -496.5772399902344,
      "logps/rejected": -577.0806884765625,
      "loss": 0.3156,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.1634175181388855,
      "rewards/margins": 0.09483225643634796,
      "rewards/rejected": -0.25824978947639465,
      "step": 260
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.5049120962530608e-06,
      "logits/chosen": 0.09195151180028915,
      "logits/rejected": 0.21280470490455627,
      "logps/chosen": -544.9461669921875,
      "logps/rejected": -664.9347534179688,
      "loss": 0.2795,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.18792234361171722,
      "rewards/margins": 0.1312183439731598,
      "rewards/rejected": -0.3191407024860382,
      "step": 270
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3754457302455464e-06,
      "logits/chosen": 0.14985263347625732,
      "logits/rejected": 0.22761961817741394,
      "logps/chosen": -607.9620361328125,
      "logps/rejected": -663.138427734375,
      "loss": 0.3365,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.24214263260364532,
      "rewards/margins": 0.07931643724441528,
      "rewards/rejected": -0.3214590847492218,
      "step": 280
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2432444529190714e-06,
      "logits/chosen": 0.15639743208885193,
      "logits/rejected": 0.23564644157886505,
      "logps/chosen": -586.5012817382812,
      "logps/rejected": -683.345947265625,
      "loss": 0.2713,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.21934108436107635,
      "rewards/margins": 0.14433029294013977,
      "rewards/rejected": -0.36367136240005493,
      "step": 290
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.1087212639117057e-06,
      "logits/chosen": 0.1709739863872528,
      "logits/rejected": 0.2245495617389679,
      "logps/chosen": -541.6049194335938,
      "logps/rejected": -656.5075073242188,
      "loss": 0.3202,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.21079179644584656,
      "rewards/margins": 0.11757495254278183,
      "rewards/rejected": -0.3283667266368866,
      "step": 300
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9722964165636263e-06,
      "logits/chosen": 0.14368459582328796,
      "logits/rejected": 0.2023853063583374,
      "logps/chosen": -536.142822265625,
      "logps/rejected": -626.310546875,
      "loss": 0.3197,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.1972382515668869,
      "rewards/margins": 0.10271339118480682,
      "rewards/rejected": -0.2999516427516937,
      "step": 310
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.8343961050366275e-06,
      "logits/chosen": 0.10999581962823868,
      "logits/rejected": 0.19454975426197052,
      "logps/chosen": -585.6015625,
      "logps/rejected": -633.619140625,
      "loss": 0.3227,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.18746989965438843,
      "rewards/margins": 0.09407122433185577,
      "rewards/rejected": -0.2815411686897278,
      "step": 320
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.695451132874385e-06,
      "logits/chosen": 0.12884345650672913,
      "logits/rejected": 0.21605506539344788,
      "logps/chosen": -508.71331787109375,
      "logps/rejected": -659.6421508789062,
      "loss": 0.2864,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.17948082089424133,
      "rewards/margins": 0.12334544956684113,
      "rewards/rejected": -0.30282628536224365,
      "step": 330
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5558955671628964e-06,
      "logits/chosen": 0.1270858496427536,
      "logits/rejected": 0.17638283967971802,
      "logps/chosen": -505.85235595703125,
      "logps/rejected": -656.5067138671875,
      "loss": 0.2753,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.1641542613506317,
      "rewards/margins": 0.12927937507629395,
      "rewards/rejected": -0.29343363642692566,
      "step": 340
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4161653824955654e-06,
      "logits/chosen": 0.15247094631195068,
      "logits/rejected": 0.14038711786270142,
      "logps/chosen": -518.3817138671875,
      "logps/rejected": -598.987548828125,
      "loss": 0.2873,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.181033656001091,
      "rewards/margins": 0.09822895377874374,
      "rewards/rejected": -0.27926260232925415,
      "step": 350
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2766970989791697e-06,
      "logits/chosen": 0.14109697937965393,
      "logits/rejected": 0.25879988074302673,
      "logps/chosen": -527.6212158203125,
      "logps/rejected": -657.6229248046875,
      "loss": 0.2737,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.17991268634796143,
      "rewards/margins": 0.14120043814182281,
      "rewards/rejected": -0.32111310958862305,
      "step": 360
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1379264185356545e-06,
      "logits/chosen": 0.18675974011421204,
      "logits/rejected": 0.2627638578414917,
      "logps/chosen": -579.1731567382812,
      "logps/rejected": -669.7673950195312,
      "loss": 0.2839,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.20380540192127228,
      "rewards/margins": 0.12797793745994568,
      "rewards/rejected": -0.33178335428237915,
      "step": 370
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.000286863759934e-06,
      "logits/chosen": 0.1522998809814453,
      "logits/rejected": 0.20338353514671326,
      "logps/chosen": -511.1458435058594,
      "logps/rejected": -621.6883544921875,
      "loss": 0.3078,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1741541475057602,
      "rewards/margins": 0.13219627737998962,
      "rewards/rejected": -0.306350439786911,
      "step": 380
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8642084235859764e-06,
      "logits/chosen": 0.16782529652118683,
      "logits/rejected": 0.25166839361190796,
      "logps/chosen": -559.2919921875,
      "logps/rejected": -642.5906372070312,
      "loss": 0.3013,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.17873121798038483,
      "rewards/margins": 0.09633437544107437,
      "rewards/rejected": -0.2750656008720398,
      "step": 390
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.7301162099921013e-06,
      "logits/chosen": 0.18475747108459473,
      "logits/rejected": 0.1979983150959015,
      "logps/chosen": -512.885009765625,
      "logps/rejected": -597.8982543945312,
      "loss": 0.3458,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.20149023830890656,
      "rewards/margins": 0.0695062130689621,
      "rewards/rejected": -0.2709963917732239,
      "step": 400
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.5984291299420117e-06,
      "logits/chosen": 0.13157054781913757,
      "logits/rejected": 0.21159140765666962,
      "logps/chosen": -526.1318359375,
      "logps/rejected": -639.5899658203125,
      "loss": 0.2671,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.1575133055448532,
      "rewards/margins": 0.13058245182037354,
      "rewards/rejected": -0.28809577226638794,
      "step": 410
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4695585767104092e-06,
      "logits/chosen": 0.14387401938438416,
      "logits/rejected": 0.23105745017528534,
      "logps/chosen": -597.7404174804688,
      "logps/rejected": -684.18359375,
      "loss": 0.2713,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.18249225616455078,
      "rewards/margins": 0.12386848032474518,
      "rewards/rejected": -0.30636072158813477,
      "step": 420
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3439071446815452e-06,
      "logits/chosen": 0.11251509189605713,
      "logits/rejected": 0.17909184098243713,
      "logps/chosen": -518.1268920898438,
      "logps/rejected": -573.087890625,
      "loss": 0.3273,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.1990637481212616,
      "rewards/margins": 0.07952968031167984,
      "rewards/rejected": -0.2785934507846832,
      "step": 430
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2218673716356919e-06,
      "logits/chosen": 0.11003521829843521,
      "logits/rejected": 0.21477892994880676,
      "logps/chosen": -500.889404296875,
      "logps/rejected": -645.4175415039062,
      "loss": 0.285,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.1877116858959198,
      "rewards/margins": 0.13334953784942627,
      "rewards/rejected": -0.3210611939430237,
      "step": 440
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.103820512452661e-06,
      "logits/chosen": 0.1255572885274887,
      "logits/rejected": 0.22474932670593262,
      "logps/chosen": -600.7024536132812,
      "logps/rejected": -698.1460571289062,
      "loss": 0.2723,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.19767552614212036,
      "rewards/margins": 0.14586390554904938,
      "rewards/rejected": -0.34353941679000854,
      "step": 450
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.901353480633468e-07,
      "logits/chosen": 0.13007649779319763,
      "logits/rejected": 0.18805541098117828,
      "logps/chosen": -527.2013549804688,
      "logps/rejected": -628.3349609375,
      "loss": 0.3286,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.22881360352039337,
      "rewards/margins": 0.09889401495456696,
      "rewards/rejected": -0.32770758867263794,
      "step": 460
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.811670333701544e-07,
      "logits/chosen": 0.10513252019882202,
      "logits/rejected": 0.1565053015947342,
      "logps/chosen": -563.048828125,
      "logps/rejected": -633.2500610351562,
      "loss": 0.2992,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.20568466186523438,
      "rewards/margins": 0.10016629844903946,
      "rewards/rejected": -0.3058509826660156,
      "step": 470
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.772559877354341e-07,
      "logits/chosen": 0.1835373491048813,
      "logits/rejected": 0.19298210740089417,
      "logps/chosen": -548.7342529296875,
      "logps/rejected": -617.98681640625,
      "loss": 0.3108,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.20563273131847382,
      "rewards/margins": 0.08818547427654266,
      "rewards/rejected": -0.2938181757926941,
      "step": 480
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.787268315040604e-07,
      "logits/chosen": 0.11183663457632065,
      "logits/rejected": 0.1670197993516922,
      "logps/chosen": -581.4060668945312,
      "logps/rejected": -655.2175903320312,
      "loss": 0.323,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.21741382777690887,
      "rewards/margins": 0.0863969624042511,
      "rewards/rejected": -0.3038107752799988,
      "step": 490
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.858873718824829e-07,
      "logits/chosen": 0.15304869413375854,
      "logits/rejected": 0.18997374176979065,
      "logps/chosen": -525.7789306640625,
      "logps/rejected": -628.69287109375,
      "loss": 0.2961,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.188993901014328,
      "rewards/margins": 0.11667201668024063,
      "rewards/rejected": -0.30566591024398804,
      "step": 500
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.990276413423817e-07,
      "logits/chosen": 0.1436108499765396,
      "logits/rejected": 0.22793538868427277,
      "logps/chosen": -528.6312866210938,
      "logps/rejected": -641.5194091796875,
      "loss": 0.292,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.18891510367393494,
      "rewards/margins": 0.11953170597553253,
      "rewards/rejected": -0.30844682455062866,
      "step": 510
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.184189915529796e-07,
      "logits/chosen": 0.08341806381940842,
      "logits/rejected": 0.24343731999397278,
      "logps/chosen": -505.2992248535156,
      "logps/rejected": -625.7940673828125,
      "loss": 0.3127,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.18587087094783783,
      "rewards/margins": 0.10922978073358536,
      "rewards/rejected": -0.2951006591320038,
      "step": 520
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.4431324567258176e-07,
      "logits/chosen": 0.1076665073633194,
      "logits/rejected": 0.16546323895454407,
      "logps/chosen": -564.4067993164062,
      "logps/rejected": -617.0671997070312,
      "loss": 0.3086,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.19417104125022888,
      "rewards/margins": 0.11286197602748871,
      "rewards/rejected": -0.3070330023765564,
      "step": 530
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.769419116476052e-07,
      "logits/chosen": 0.14298070967197418,
      "logits/rejected": 0.17989978194236755,
      "logps/chosen": -521.3731689453125,
      "logps/rejected": -603.6255493164062,
      "loss": 0.315,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.19311046600341797,
      "rewards/margins": 0.09155095368623734,
      "rewards/rejected": -0.2846614718437195,
      "step": 540
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.1651545897676512e-07,
      "logits/chosen": 0.14457222819328308,
      "logits/rejected": 0.1904611885547638,
      "logps/chosen": -503.63385009765625,
      "logps/rejected": -589.3573608398438,
      "loss": 0.3095,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.1882583498954773,
      "rewards/margins": 0.0975576862692833,
      "rewards/rejected": -0.2858160138130188,
      "step": 550
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.6322266119983222e-07,
      "logits/chosen": 0.1182747334241867,
      "logits/rejected": 0.21196098625659943,
      "logps/chosen": -558.52099609375,
      "logps/rejected": -705.6689453125,
      "loss": 0.2741,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.2015405148267746,
      "rewards/margins": 0.12743064761161804,
      "rewards/rejected": -0.32897114753723145,
      "step": 560
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1723000616502167e-07,
      "logits/chosen": 0.13327908515930176,
      "logits/rejected": 0.2085684984922409,
      "logps/chosen": -520.0775146484375,
      "logps/rejected": -619.95703125,
      "loss": 0.2912,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.18114188313484192,
      "rewards/margins": 0.1153794676065445,
      "rewards/rejected": -0.29652130603790283,
      "step": 570
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.868117591737585e-08,
      "logits/chosen": 0.11308556795120239,
      "logits/rejected": 0.16310641169548035,
      "logps/chosen": -502.0087890625,
      "logps/rejected": -638.6658935546875,
      "loss": 0.2833,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.1852196305990219,
      "rewards/margins": 0.13148216903209686,
      "rewards/rejected": -0.3167017996311188,
      "step": 580
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.769659783295383e-08,
      "logits/chosen": 0.1447245329618454,
      "logits/rejected": 0.17959550023078918,
      "logps/chosen": -533.7676391601562,
      "logps/rejected": -665.7962036132812,
      "loss": 0.275,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.18203890323638916,
      "rewards/margins": 0.13183028995990753,
      "rewards/rejected": -0.3138691782951355,
      "step": 590
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.4373068401120358e-08,
      "logits/chosen": 0.1246277466416359,
      "logits/rejected": 0.19313006103038788,
      "logps/chosen": -532.855224609375,
      "logps/rejected": -646.3125,
      "loss": 0.3061,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.18818099796772003,
      "rewards/margins": 0.12483028322458267,
      "rewards/rejected": -0.3130112886428833,
      "step": 600
    },
    {
      "epoch": 0.98,
      "learning_rate": 8.78345083022425e-09,
      "logits/chosen": 0.07625824213027954,
      "logits/rejected": 0.19986259937286377,
      "logps/chosen": -546.0481567382812,
      "logps/rejected": -647.2523193359375,
      "loss": 0.2853,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.207177072763443,
      "rewards/margins": 0.1102672815322876,
      "rewards/rejected": -0.3174443542957306,
      "step": 610
    },
    {
      "epoch": 0.99,
      "learning_rate": 9.764474213677654e-10,
      "logits/chosen": 0.08471451699733734,
      "logits/rejected": 0.1842333823442459,
      "logps/chosen": -539.9765014648438,
      "logps/rejected": -655.6905517578125,
      "loss": 0.2766,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18608103692531586,
      "rewards/margins": 0.13810980319976807,
      "rewards/rejected": -0.3241908550262451,
      "step": 620
    },
    {
      "epoch": 1.0,
      "step": 625,
      "total_flos": 0.0,
      "train_loss": 0.31405294857025146,
      "train_runtime": 5821.3149,
      "train_samples_per_second": 1.718,
      "train_steps_per_second": 0.107
    }
  ],
  "logging_steps": 10,
  "max_steps": 625,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}