{
  "best_metric": 1.347064733505249,
  "best_model_checkpoint": "saves/Gemma-7B-It/lora/orpo/checkpoint-1500",
  "epoch": 2.997999555456768,
  "eval_steps": 500,
  "global_step": 1686,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.017781729273171815,
      "grad_norm": 2.0664310455322266,
      "learning_rate": 4.9995745934141085e-06,
      "logits/chosen": 207.18429565429688,
      "logits/rejected": 208.7433624267578,
      "logps/chosen": -2.1626429557800293,
      "logps/rejected": -2.4889369010925293,
      "loss": 2.2441,
      "odds_ratio_loss": 0.8149965405464172,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.2162642925977707,
      "rewards/margins": 0.03262939304113388,
      "rewards/rejected": -0.24889369308948517,
      "sft_loss": 2.1626429557800293,
      "step": 10
    },
    {
      "epoch": 0.03556345854634363,
      "grad_norm": 3.5887179374694824,
      "learning_rate": 4.9982812903243405e-06,
      "logits/chosen": 207.42489624023438,
      "logits/rejected": 210.30307006835938,
      "logps/chosen": -2.3814258575439453,
      "logps/rejected": -2.279829740524292,
      "loss": 2.4841,
      "odds_ratio_loss": 1.0264532566070557,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.23814257979393005,
      "rewards/margins": -0.01015959121286869,
      "rewards/rejected": -0.2279830276966095,
      "sft_loss": 2.3814258575439453,
      "step": 20
    },
    {
      "epoch": 0.05334518781951545,
      "grad_norm": 7.692424297332764,
      "learning_rate": 4.996120496405222e-06,
      "logits/chosen": 210.8332977294922,
      "logits/rejected": 211.61099243164062,
      "logps/chosen": -2.3326590061187744,
      "logps/rejected": -2.6421866416931152,
      "loss": 2.4103,
      "odds_ratio_loss": 0.7764064073562622,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.23326590657234192,
      "rewards/margins": 0.03095281682908535,
      "rewards/rejected": -0.2642187178134918,
      "sft_loss": 2.3326590061187744,
      "step": 30
    },
    {
      "epoch": 0.07112691709268726,
      "grad_norm": 2.3221993446350098,
      "learning_rate": 4.99309296196014e-06,
      "logits/chosen": 213.6151123046875,
      "logits/rejected": 215.0422821044922,
      "logps/chosen": -2.1157479286193848,
      "logps/rejected": -2.377347469329834,
      "loss": 2.2037,
      "odds_ratio_loss": 0.8793656229972839,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.2115747630596161,
      "rewards/margins": 0.026159971952438354,
      "rewards/rejected": -0.23773476481437683,
      "sft_loss": 2.1157479286193848,
      "step": 40
    },
    {
      "epoch": 0.08890864636585907,
      "grad_norm": 3.239240884780884,
      "learning_rate": 4.989199738255166e-06,
      "logits/chosen": 215.62277221679688,
      "logits/rejected": 217.9526824951172,
      "logps/chosen": -2.122495174407959,
      "logps/rejected": -2.3831381797790527,
      "loss": 2.2102,
      "odds_ratio_loss": 0.876750648021698,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.2122495174407959,
      "rewards/margins": 0.0260643120855093,
      "rewards/rejected": -0.23831383883953094,
      "sft_loss": 2.122495174407959,
      "step": 50
    },
    {
      "epoch": 0.1066903756390309,
      "grad_norm": 2.045614719390869,
      "learning_rate": 4.984442177154031e-06,
      "logits/chosen": 218.19265747070312,
      "logits/rejected": 219.45907592773438,
      "logps/chosen": -2.157195568084717,
      "logps/rejected": -2.426860809326172,
      "loss": 2.2519,
      "odds_ratio_loss": 0.9465675354003906,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.215719535946846,
      "rewards/margins": 0.02696654573082924,
      "rewards/rejected": -0.24268610775470734,
      "sft_loss": 2.157195568084717,
      "step": 60
    },
    {
      "epoch": 0.12447210491220272,
      "grad_norm": 2.5059540271759033,
      "learning_rate": 4.978821930648704e-06,
      "logits/chosen": 222.6106414794922,
      "logits/rejected": 222.9716339111328,
      "logps/chosen": -2.108616590499878,
      "logps/rejected": -2.0369954109191895,
      "loss": 2.2226,
      "odds_ratio_loss": 1.1399357318878174,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.21086165308952332,
      "rewards/margins": -0.007162079215049744,
      "rewards/rejected": -0.20369958877563477,
      "sft_loss": 2.108616590499878,
      "step": 70
    },
    {
      "epoch": 0.14225383418537452,
      "grad_norm": 3.193876028060913,
      "learning_rate": 4.97234095028576e-06,
      "logits/chosen": 226.2631378173828,
      "logits/rejected": 227.37841796875,
      "logps/chosen": -2.011748790740967,
      "logps/rejected": -2.0834341049194336,
      "loss": 2.0967,
      "odds_ratio_loss": 0.849190354347229,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.20117488503456116,
      "rewards/margins": 0.0071685537695884705,
      "rewards/rejected": -0.20834341645240784,
      "sft_loss": 2.011748790740967,
      "step": 80
    },
    {
      "epoch": 0.16003556345854633,
      "grad_norm": 1.5080177783966064,
      "learning_rate": 4.965001486488743e-06,
      "logits/chosen": 227.5520477294922,
      "logits/rejected": 228.8264923095703,
      "logps/chosen": -1.7941173315048218,
      "logps/rejected": -2.0014424324035645,
      "loss": 1.8703,
      "odds_ratio_loss": 0.7616177797317505,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.17941176891326904,
      "rewards/margins": 0.020732494071125984,
      "rewards/rejected": -0.20014424622058868,
      "sft_loss": 1.7941173315048218,
      "step": 90
    },
    {
      "epoch": 0.17781729273171815,
      "grad_norm": 1.543541669845581,
      "learning_rate": 4.956806087776732e-06,
      "logits/chosen": 230.88284301757812,
      "logits/rejected": 231.593017578125,
      "logps/chosen": -1.7711349725723267,
      "logps/rejected": -2.1635921001434326,
      "loss": 1.8365,
      "odds_ratio_loss": 0.6534941792488098,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.17711350321769714,
      "rewards/margins": 0.039245713502168655,
      "rewards/rejected": -0.2163592129945755,
      "sft_loss": 1.7711349725723267,
      "step": 100
    },
    {
      "epoch": 0.19559902200489,
      "grad_norm": 2.5298948287963867,
      "learning_rate": 4.947757599879411e-06,
      "logits/chosen": 235.8949737548828,
      "logits/rejected": 235.8369140625,
      "logps/chosen": -1.6983522176742554,
      "logps/rejected": -1.9385782480239868,
      "loss": 1.7686,
      "odds_ratio_loss": 0.7024893760681152,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.16983522474765778,
      "rewards/margins": 0.024022620171308517,
      "rewards/rejected": -0.1938578337430954,
      "sft_loss": 1.6983522176742554,
      "step": 110
    },
    {
      "epoch": 0.2133807512780618,
      "grad_norm": 3.6936049461364746,
      "learning_rate": 4.937859164748931e-06,
      "logits/chosen": 239.0133514404297,
      "logits/rejected": 239.0744171142578,
      "logps/chosen": -1.5454410314559937,
      "logps/rejected": -1.6620423793792725,
      "loss": 1.6182,
      "odds_ratio_loss": 0.727584183216095,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.15454408526420593,
      "rewards/margins": 0.011660170741379261,
      "rewards/rejected": -0.16620425879955292,
      "sft_loss": 1.5454410314559937,
      "step": 120
    },
    {
      "epoch": 0.23116248055123362,
      "grad_norm": 2.5647566318511963,
      "learning_rate": 4.92711421946891e-06,
      "logits/chosen": 239.533203125,
      "logits/rejected": 240.0679931640625,
      "logps/chosen": -1.6095718145370483,
      "logps/rejected": -1.8200355768203735,
      "loss": 1.6813,
      "odds_ratio_loss": 0.7172408103942871,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.16095717251300812,
      "rewards/margins": 0.021046385169029236,
      "rewards/rejected": -0.18200358748435974,
      "sft_loss": 1.6095718145370483,
      "step": 130
    },
    {
      "epoch": 0.24894420982440543,
      "grad_norm": 7.456451416015625,
      "learning_rate": 4.915526495060961e-06,
      "logits/chosen": 242.9502410888672,
      "logits/rejected": 243.6327362060547,
      "logps/chosen": -1.5086013078689575,
      "logps/rejected": -1.7959527969360352,
      "loss": 1.5772,
      "odds_ratio_loss": 0.6855502128601074,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.15086011588573456,
      "rewards/margins": 0.028735166415572166,
      "rewards/rejected": -0.17959527671337128,
      "sft_loss": 1.5086013078689575,
      "step": 140
    },
    {
      "epoch": 0.26672593909757725,
      "grad_norm": 3.7102017402648926,
      "learning_rate": 4.903100015189153e-06,
      "logits/chosen": 243.65646362304688,
      "logits/rejected": 243.8603973388672,
      "logps/chosen": -1.5454883575439453,
      "logps/rejected": -1.7633510828018188,
      "loss": 1.6165,
      "odds_ratio_loss": 0.7103039026260376,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.15454885363578796,
      "rewards/margins": 0.021786261349916458,
      "rewards/rejected": -0.17633512616157532,
      "sft_loss": 1.5454883575439453,
      "step": 150
    },
    {
      "epoch": 0.28450766837074903,
      "grad_norm": 1.9714837074279785,
      "learning_rate": 4.889839094762848e-06,
      "logits/chosen": 244.4144744873047,
      "logits/rejected": 244.4231414794922,
      "logps/chosen": -1.6150789260864258,
      "logps/rejected": -1.7520729303359985,
      "loss": 1.6882,
      "odds_ratio_loss": 0.7308647036552429,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.16150791943073273,
      "rewards/margins": 0.013699382543563843,
      "rewards/rejected": -0.17520728707313538,
      "sft_loss": 1.6150789260864258,
      "step": 160
    },
    {
      "epoch": 0.3022893976439209,
      "grad_norm": 1.7966150045394897,
      "learning_rate": 4.875748338438416e-06,
      "logits/chosen": 245.6141815185547,
      "logits/rejected": 245.50735473632812,
      "logps/chosen": -1.5294561386108398,
      "logps/rejected": -1.7359291315078735,
      "loss": 1.5978,
      "odds_ratio_loss": 0.683633029460907,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.1529456079006195,
      "rewards/margins": 0.020647313445806503,
      "rewards/rejected": -0.1735929250717163,
      "sft_loss": 1.5294561386108398,
      "step": 170
    },
    {
      "epoch": 0.32007112691709266,
      "grad_norm": 4.217123031616211,
      "learning_rate": 4.8608326390203386e-06,
      "logits/chosen": 245.0269317626953,
      "logits/rejected": 245.72494506835938,
      "logps/chosen": -1.4982770681381226,
      "logps/rejected": -1.7273998260498047,
      "loss": 1.5646,
      "odds_ratio_loss": 0.663575291633606,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.1498277187347412,
      "rewards/margins": 0.022912275046110153,
      "rewards/rejected": -0.17273999750614166,
      "sft_loss": 1.4982770681381226,
      "step": 180
    },
    {
      "epoch": 0.3378528561902645,
      "grad_norm": 2.1345717906951904,
      "learning_rate": 4.845097175762251e-06,
      "logits/chosen": 246.3583984375,
      "logits/rejected": 246.6023712158203,
      "logps/chosen": -1.4693564176559448,
      "logps/rejected": -1.6270701885223389,
      "loss": 1.5391,
      "odds_ratio_loss": 0.6974608898162842,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.1469356268644333,
      "rewards/margins": 0.01577138900756836,
      "rewards/rejected": -0.16270704567432404,
      "sft_loss": 1.4693564176559448,
      "step": 190
    },
    {
      "epoch": 0.3556345854634363,
      "grad_norm": 4.533120632171631,
      "learning_rate": 4.8285474125685286e-06,
      "logits/chosen": 245.734130859375,
      "logits/rejected": 246.148193359375,
      "logps/chosen": -1.5207107067108154,
      "logps/rejected": -1.6708800792694092,
      "loss": 1.5928,
      "odds_ratio_loss": 0.7206315994262695,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.15207107365131378,
      "rewards/margins": 0.015016931109130383,
      "rewards/rejected": -0.16708801686763763,
      "sft_loss": 1.5207107067108154,
      "step": 200
    },
    {
      "epoch": 0.37341631473660813,
      "grad_norm": 2.590592861175537,
      "learning_rate": 4.811189096097025e-06,
      "logits/chosen": 244.85757446289062,
      "logits/rejected": 245.19387817382812,
      "logps/chosen": -1.4962936639785767,
      "logps/rejected": -1.7369601726531982,
      "loss": 1.5627,
      "odds_ratio_loss": 0.6638623476028442,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.1496293693780899,
      "rewards/margins": 0.024066651239991188,
      "rewards/rejected": -0.17369601130485535,
      "sft_loss": 1.4962936639785767,
      "step": 210
    },
    {
      "epoch": 0.39119804400978,
      "grad_norm": 2.847151279449463,
      "learning_rate": 4.793028253763633e-06,
      "logits/chosen": 247.3143310546875,
      "logits/rejected": 247.9826202392578,
      "logps/chosen": -1.4769468307495117,
      "logps/rejected": -1.6658426523208618,
      "loss": 1.5539,
      "odds_ratio_loss": 0.769631028175354,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.14769470691680908,
      "rewards/margins": 0.018889565020799637,
      "rewards/rejected": -0.16658425331115723,
      "sft_loss": 1.4769468307495117,
      "step": 220
    },
    {
      "epoch": 0.40897977328295176,
      "grad_norm": 2.361198902130127,
      "learning_rate": 4.774071191649352e-06,
      "logits/chosen": 247.5041961669922,
      "logits/rejected": 247.5510711669922,
      "logps/chosen": -1.3598966598510742,
      "logps/rejected": -1.6928675174713135,
      "loss": 1.421,
      "odds_ratio_loss": 0.6115107536315918,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.13598966598510742,
      "rewards/margins": 0.033297087997198105,
      "rewards/rejected": -0.16928674280643463,
      "sft_loss": 1.3598966598510742,
      "step": 230
    },
    {
      "epoch": 0.4267615025561236,
      "grad_norm": 2.774188280105591,
      "learning_rate": 4.7543244923105975e-06,
      "logits/chosen": 248.81741333007812,
      "logits/rejected": 248.56063842773438,
      "logps/chosen": -1.479175090789795,
      "logps/rejected": -1.5817492008209229,
      "loss": 1.5553,
      "odds_ratio_loss": 0.7613282799720764,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.14791752398014069,
      "rewards/margins": 0.010257410816848278,
      "rewards/rejected": -0.15817493200302124,
      "sft_loss": 1.479175090789795,
      "step": 240
    },
    {
      "epoch": 0.4445432318292954,
      "grad_norm": 1.5498669147491455,
      "learning_rate": 4.733795012493506e-06,
      "logits/chosen": 249.3889923095703,
      "logits/rejected": 249.21475219726562,
      "logps/chosen": -1.544840693473816,
      "logps/rejected": -1.6548751592636108,
      "loss": 1.6181,
      "odds_ratio_loss": 0.7324239611625671,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.15448406338691711,
      "rewards/margins": 0.011003440245985985,
      "rewards/rejected": -0.16548751294612885,
      "sft_loss": 1.544840693473816,
      "step": 250
    },
    {
      "epoch": 0.46232496110246724,
      "grad_norm": 1.862833857536316,
      "learning_rate": 4.712489880753035e-06,
      "logits/chosen": 245.6899871826172,
      "logits/rejected": 246.0701446533203,
      "logps/chosen": -1.316460132598877,
      "logps/rejected": -1.5619014501571655,
      "loss": 1.3796,
      "odds_ratio_loss": 0.631010890007019,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.13164600729942322,
      "rewards/margins": 0.02454412914812565,
      "rewards/rejected": -0.15619014203548431,
      "sft_loss": 1.316460132598877,
      "step": 260
    },
    {
      "epoch": 0.480106690375639,
      "grad_norm": 7.502230644226074,
      "learning_rate": 4.690416494977673e-06,
      "logits/chosen": 248.2025146484375,
      "logits/rejected": 249.02481079101562,
      "logps/chosen": -1.4416104555130005,
      "logps/rejected": -1.7268083095550537,
      "loss": 1.5056,
      "odds_ratio_loss": 0.6399883031845093,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.14416104555130005,
      "rewards/margins": 0.028519803658127785,
      "rewards/rejected": -0.1726808398962021,
      "sft_loss": 1.4416104555130005,
      "step": 270
    },
    {
      "epoch": 0.49788841964881086,
      "grad_norm": 1.7004640102386475,
      "learning_rate": 4.667582519820639e-06,
      "logits/chosen": 248.4412841796875,
      "logits/rejected": 248.5986328125,
      "logps/chosen": -1.4644906520843506,
      "logps/rejected": -1.6034975051879883,
      "loss": 1.5385,
      "odds_ratio_loss": 0.7402389645576477,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.14644907414913177,
      "rewards/margins": 0.013900673016905785,
      "rewards/rejected": -0.1603497415781021,
      "sft_loss": 1.4644906520843506,
      "step": 280
    },
    {
      "epoch": 0.5156701489219827,
      "grad_norm": 1.9778798818588257,
      "learning_rate": 4.643995884038443e-06,
      "logits/chosen": 249.9197540283203,
      "logits/rejected": 249.89920043945312,
      "logps/chosen": -1.390932559967041,
      "logps/rejected": -1.5962976217269897,
      "loss": 1.4582,
      "odds_ratio_loss": 0.6727808713912964,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.13909325003623962,
      "rewards/margins": 0.020536506548523903,
      "rewards/rejected": -0.15962976217269897,
      "sft_loss": 1.390932559967041,
      "step": 290
    },
    {
      "epoch": 0.5334518781951545,
      "grad_norm": 1.796635627746582,
      "learning_rate": 4.6196647777377475e-06,
      "logits/chosen": 248.5753936767578,
      "logits/rejected": 248.55288696289062,
      "logps/chosen": -1.3909634351730347,
      "logps/rejected": -1.4929519891738892,
      "loss": 1.4621,
      "odds_ratio_loss": 0.7113169431686401,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.13909634947776794,
      "rewards/margins": 0.010198858566582203,
      "rewards/rejected": -0.14929521083831787,
      "sft_loss": 1.3909634351730347,
      "step": 300
    },
    {
      "epoch": 0.5512336074683263,
      "grad_norm": 1.5784835815429688,
      "learning_rate": 4.59459764953147e-06,
      "logits/chosen": 247.842529296875,
      "logits/rejected": 248.6540069580078,
      "logps/chosen": -1.449033498764038,
      "logps/rejected": -1.587956190109253,
      "loss": 1.5181,
      "odds_ratio_loss": 0.6905879974365234,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.14490333199501038,
      "rewards/margins": 0.013892298564314842,
      "rewards/rejected": -0.15879563987255096,
      "sft_loss": 1.449033498764038,
      "step": 310
    },
    {
      "epoch": 0.5690153367414981,
      "grad_norm": 2.8430497646331787,
      "learning_rate": 4.568803203605133e-06,
      "logits/chosen": 247.75045776367188,
      "logits/rejected": 247.95272827148438,
      "logps/chosen": -1.3736222982406616,
      "logps/rejected": -1.5858898162841797,
      "loss": 1.4444,
      "odds_ratio_loss": 0.7075997591018677,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.13736224174499512,
      "rewards/margins": 0.02122673951089382,
      "rewards/rejected": -0.15858899056911469,
      "sft_loss": 1.3736222982406616,
      "step": 320
    },
    {
      "epoch": 0.58679706601467,
      "grad_norm": 8.24238395690918,
      "learning_rate": 4.542290396694462e-06,
      "logits/chosen": 250.15707397460938,
      "logits/rejected": 249.8236541748047,
      "logps/chosen": -1.4158143997192383,
      "logps/rejected": -1.5936088562011719,
      "loss": 1.4861,
      "odds_ratio_loss": 0.7032537460327148,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.1415814310312271,
      "rewards/margins": 0.017779454588890076,
      "rewards/rejected": -0.15936090052127838,
      "sft_loss": 1.4158143997192383,
      "step": 330
    },
    {
      "epoch": 0.6045787952878418,
      "grad_norm": 8.322469711303711,
      "learning_rate": 4.515068434975298e-06,
      "logits/chosen": 247.47579956054688,
      "logits/rejected": 248.01553344726562,
      "logps/chosen": -1.413419246673584,
      "logps/rejected": -1.6688693761825562,
      "loss": 1.4802,
      "odds_ratio_loss": 0.6680575013160706,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.1413419246673584,
      "rewards/margins": 0.02554500475525856,
      "rewards/rejected": -0.16688695549964905,
      "sft_loss": 1.413419246673584,
      "step": 340
    },
    {
      "epoch": 0.6223605245610135,
      "grad_norm": 1.7108134031295776,
      "learning_rate": 4.487146770866887e-06,
      "logits/chosen": 249.2637939453125,
      "logits/rejected": 249.53549194335938,
      "logps/chosen": -1.3779569864273071,
      "logps/rejected": -1.5148077011108398,
      "loss": 1.4475,
      "odds_ratio_loss": 0.6950393915176392,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.13779568672180176,
      "rewards/margins": 0.013685077428817749,
      "rewards/rejected": -0.1514807641506195,
      "sft_loss": 1.3779569864273071,
      "step": 350
    },
    {
      "epoch": 0.6401422538341853,
      "grad_norm": 1.5160853862762451,
      "learning_rate": 4.458535099749666e-06,
      "logits/chosen": 251.33633422851562,
      "logits/rejected": 251.1912841796875,
      "logps/chosen": -1.4455270767211914,
      "logps/rejected": -1.5792427062988281,
      "loss": 1.5213,
      "odds_ratio_loss": 0.7574425935745239,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.14455269277095795,
      "rewards/margins": 0.013371584005653858,
      "rewards/rejected": -0.15792429447174072,
      "sft_loss": 1.4455270767211914,
      "step": 360
    },
    {
      "epoch": 0.6579239831073572,
      "grad_norm": 2.7603657245635986,
      "learning_rate": 4.429243356598694e-06,
      "logits/chosen": 249.0806121826172,
      "logits/rejected": 249.3708038330078,
      "logps/chosen": -1.4384644031524658,
      "logps/rejected": -1.6943752765655518,
      "loss": 1.504,
      "odds_ratio_loss": 0.655007004737854,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.14384643733501434,
      "rewards/margins": 0.025591086596250534,
      "rewards/rejected": -0.16943752765655518,
      "sft_loss": 1.4384644031524658,
      "step": 370
    },
    {
      "epoch": 0.675705712380529,
      "grad_norm": 1.7461826801300049,
      "learning_rate": 4.399281712533875e-06,
      "logits/chosen": 249.9875030517578,
      "logits/rejected": 250.59976196289062,
      "logps/chosen": -1.385258674621582,
      "logps/rejected": -1.5310487747192383,
      "loss": 1.4575,
      "odds_ratio_loss": 0.7227787375450134,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.13852587342262268,
      "rewards/margins": 0.014579012989997864,
      "rewards/rejected": -0.15310488641262054,
      "sft_loss": 1.385258674621582,
      "step": 380
    },
    {
      "epoch": 0.6934874416537008,
      "grad_norm": 2.5713703632354736,
      "learning_rate": 4.368660571288192e-06,
      "logits/chosen": 249.9259490966797,
      "logits/rejected": 250.7145538330078,
      "logps/chosen": -1.3335117101669312,
      "logps/rejected": -1.4699729681015015,
      "loss": 1.404,
      "odds_ratio_loss": 0.7047192454338074,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.1333511769771576,
      "rewards/margins": 0.013646131381392479,
      "rewards/rejected": -0.14699730277061462,
      "sft_loss": 1.3335117101669312,
      "step": 390
    },
    {
      "epoch": 0.7112691709268726,
      "grad_norm": 1.9281286001205444,
      "learning_rate": 4.337390565595163e-06,
      "logits/chosen": 251.0037384033203,
      "logits/rejected": 250.80783081054688,
      "logps/chosen": -1.4610103368759155,
      "logps/rejected": -1.5656871795654297,
      "loss": 1.5334,
      "odds_ratio_loss": 0.7237542271614075,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.14610104262828827,
      "rewards/margins": 0.010467682033777237,
      "rewards/rejected": -0.1565687209367752,
      "sft_loss": 1.4610103368759155,
      "step": 400
    },
    {
      "epoch": 0.7290509002000445,
      "grad_norm": 2.2883145809173584,
      "learning_rate": 4.305482553496786e-06,
      "logits/chosen": 249.4977569580078,
      "logits/rejected": 249.899658203125,
      "logps/chosen": -1.3607113361358643,
      "logps/rejected": -1.5497020483016968,
      "loss": 1.4299,
      "odds_ratio_loss": 0.692295491695404,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.1360711306333542,
      "rewards/margins": 0.018899066373705864,
      "rewards/rejected": -0.1549702137708664,
      "sft_loss": 1.3607113361358643,
      "step": 410
    },
    {
      "epoch": 0.7468326294732163,
      "grad_norm": 1.6453748941421509,
      "learning_rate": 4.272947614573244e-06,
      "logits/chosen": 251.3318328857422,
      "logits/rejected": 251.18276977539062,
      "logps/chosen": -1.3767774105072021,
      "logps/rejected": -1.5617191791534424,
      "loss": 1.4431,
      "odds_ratio_loss": 0.662979006767273,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.13767775893211365,
      "rewards/margins": 0.01849415898323059,
      "rewards/rejected": -0.15617190301418304,
      "sft_loss": 1.3767774105072021,
      "step": 420
    },
    {
      "epoch": 0.7646143587463881,
      "grad_norm": 1.49653959274292,
      "learning_rate": 4.23979704609569e-06,
      "logits/chosen": 251.7825469970703,
      "logits/rejected": 251.8596954345703,
      "logps/chosen": -1.360251545906067,
      "logps/rejected": -1.4845999479293823,
      "loss": 1.43,
      "odds_ratio_loss": 0.6971336603164673,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.13602514564990997,
      "rewards/margins": 0.012434848584234715,
      "rewards/rejected": -0.1484600007534027,
      "sft_loss": 1.360251545906067,
      "step": 430
    },
    {
      "epoch": 0.78239608801956,
      "grad_norm": 2.6121129989624023,
      "learning_rate": 4.206042359103435e-06,
      "logits/chosen": 249.96774291992188,
      "logits/rejected": 249.6472930908203,
      "logps/chosen": -1.451643705368042,
      "logps/rejected": -1.6453460454940796,
      "loss": 1.5216,
      "odds_ratio_loss": 0.6996283531188965,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.145164355635643,
      "rewards/margins": 0.01937025412917137,
      "rewards/rejected": -0.16453461349010468,
      "sft_loss": 1.451643705368042,
      "step": 440
    },
    {
      "epoch": 0.8001778172927317,
      "grad_norm": 4.463571548461914,
      "learning_rate": 4.17169527440691e-06,
      "logits/chosen": 249.04443359375,
      "logits/rejected": 249.074951171875,
      "logps/chosen": -1.3891347646713257,
      "logps/rejected": -1.5458732843399048,
      "loss": 1.4601,
      "odds_ratio_loss": 0.7098666429519653,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.13891348242759705,
      "rewards/margins": 0.015673857182264328,
      "rewards/rejected": -0.15458734333515167,
      "sft_loss": 1.3891347646713257,
      "step": 450
    },
    {
      "epoch": 0.8179595465659035,
      "grad_norm": 3.0045888423919678,
      "learning_rate": 4.136767718517797e-06,
      "logits/chosen": 250.2001495361328,
      "logits/rejected": 250.69189453125,
      "logps/chosen": -1.2655476331710815,
      "logps/rejected": -1.5482288599014282,
      "loss": 1.3289,
      "odds_ratio_loss": 0.6339131593704224,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.12655475735664368,
      "rewards/margins": 0.02826813980937004,
      "rewards/rejected": -0.15482288599014282,
      "sft_loss": 1.2655476331710815,
      "step": 460
    },
    {
      "epoch": 0.8357412758390753,
      "grad_norm": 5.639819145202637,
      "learning_rate": 4.1012718195077196e-06,
      "logits/chosen": 250.90878295898438,
      "logits/rejected": 251.2098388671875,
      "logps/chosen": -1.3541600704193115,
      "logps/rejected": -1.4638829231262207,
      "loss": 1.4238,
      "odds_ratio_loss": 0.6962955594062805,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.13541600108146667,
      "rewards/margins": 0.01097229402512312,
      "rewards/rejected": -0.14638829231262207,
      "sft_loss": 1.3541600704193115,
      "step": 470
    },
    {
      "epoch": 0.8535230051122472,
      "grad_norm": 2.0918374061584473,
      "learning_rate": 4.065219902796953e-06,
      "logits/chosen": 250.6866455078125,
      "logits/rejected": 250.68875122070312,
      "logps/chosen": -1.3266746997833252,
      "logps/rejected": -1.5351797342300415,
      "loss": 1.3953,
      "odds_ratio_loss": 0.6859580278396606,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.13266746699810028,
      "rewards/margins": 0.02085050381720066,
      "rewards/rejected": -0.1535179764032364,
      "sft_loss": 1.3266746997833252,
      "step": 480
    },
    {
      "epoch": 0.871304734385419,
      "grad_norm": 1.7283604145050049,
      "learning_rate": 4.028624486874608e-06,
      "logits/chosen": 250.7782745361328,
      "logits/rejected": 250.49868774414062,
      "logps/chosen": -1.3391790390014648,
      "logps/rejected": -1.5556788444519043,
      "loss": 1.4069,
      "odds_ratio_loss": 0.6768004894256592,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.133917897939682,
      "rewards/margins": 0.021649986505508423,
      "rewards/rejected": -0.15556788444519043,
      "sft_loss": 1.3391790390014648,
      "step": 490
    },
    {
      "epoch": 0.8890864636585908,
      "grad_norm": 1.839385747909546,
      "learning_rate": 3.99149827895177e-06,
      "logits/chosen": 251.3027801513672,
      "logits/rejected": 250.90676879882812,
      "logps/chosen": -1.4337760210037231,
      "logps/rejected": -1.565932273864746,
      "loss": 1.5041,
      "odds_ratio_loss": 0.7032449841499329,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.14337760210037231,
      "rewards/margins": 0.01321563869714737,
      "rewards/rejected": -0.1565932184457779,
      "sft_loss": 1.4337760210037231,
      "step": 500
    },
    {
      "epoch": 0.8890864636585908,
      "eval_logits/chosen": 250.64625549316406,
      "eval_logits/rejected": 250.75489807128906,
      "eval_logps/chosen": -1.352203369140625,
      "eval_logps/rejected": -1.5643647909164429,
      "eval_loss": 1.418463945388794,
      "eval_odds_ratio_loss": 0.6626060605049133,
      "eval_rewards/accuracies": 0.5529999732971191,
      "eval_rewards/chosen": -0.13522033393383026,
      "eval_rewards/margins": 0.021216128021478653,
      "eval_rewards/rejected": -0.15643645823001862,
      "eval_runtime": 217.284,
      "eval_samples_per_second": 4.602,
      "eval_sft_loss": 1.352203369140625,
      "eval_steps_per_second": 2.301,
      "step": 500
    },
    {
      "epoch": 0.9068681929317626,
      "grad_norm": 4.217110633850098,
      "learning_rate": 3.953854170549114e-06,
      "logits/chosen": 251.8994598388672,
      "logits/rejected": 251.8975067138672,
      "logps/chosen": -1.34084153175354,
      "logps/rejected": -1.4915145635604858,
      "loss": 1.4078,
      "odds_ratio_loss": 0.6700819730758667,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.13408413529396057,
      "rewards/margins": 0.015067322179675102,
      "rewards/rejected": -0.14915145933628082,
      "sft_loss": 1.34084153175354,
      "step": 510
    },
    {
      "epoch": 0.9246499222049345,
      "grad_norm": 2.6186540126800537,
      "learning_rate": 3.91570523302051e-06,
      "logits/chosen": 250.59036254882812,
      "logits/rejected": 250.75003051757812,
      "logps/chosen": -1.3541263341903687,
      "logps/rejected": -1.4764378070831299,
      "loss": 1.427,
      "odds_ratio_loss": 0.728742241859436,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.13541266322135925,
      "rewards/margins": 0.012231158092617989,
      "rewards/rejected": -0.1476438045501709,
      "sft_loss": 1.3541263341903687,
      "step": 520
    },
    {
      "epoch": 0.9424316514781063,
      "grad_norm": 2.1178462505340576,
      "learning_rate": 3.8770647130141996e-06,
      "logits/chosen": 250.88150024414062,
      "logits/rejected": 251.63052368164062,
      "logps/chosen": -1.3341526985168457,
      "logps/rejected": -1.5821571350097656,
      "loss": 1.4029,
      "odds_ratio_loss": 0.6879525780677795,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.13341526687145233,
      "rewards/margins": 0.02480044774711132,
      "rewards/rejected": -0.1582157164812088,
      "sft_loss": 1.3341526985168457,
      "step": 530
    },
    {
      "epoch": 0.960213380751278,
      "grad_norm": 2.5758562088012695,
      "learning_rate": 3.837946027873086e-06,
      "logits/chosen": 249.52285766601562,
      "logits/rejected": 250.219970703125,
      "logps/chosen": -1.4206160306930542,
      "logps/rejected": -1.6295530796051025,
      "loss": 1.4917,
      "odds_ratio_loss": 0.7105556726455688,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.14206160604953766,
      "rewards/margins": 0.020893706008791924,
      "rewards/rejected": -0.16295531392097473,
      "sft_loss": 1.4206160306930542,
      "step": 540
    },
    {
      "epoch": 0.9779951100244498,
      "grad_norm": 2.2665178775787354,
      "learning_rate": 3.7983627609757713e-06,
      "logits/chosen": 250.9306640625,
      "logits/rejected": 251.22189331054688,
      "logps/chosen": -1.3822170495986938,
      "logps/rejected": -1.51272714138031,
      "loss": 1.4502,
      "odds_ratio_loss": 0.679997444152832,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.13822171092033386,
      "rewards/margins": 0.01305101066827774,
      "rewards/rejected": -0.1512727290391922,
      "sft_loss": 1.3822170495986938,
      "step": 550
    },
    {
      "epoch": 0.9957768392976217,
      "grad_norm": 9.529622077941895,
      "learning_rate": 3.758328657019924e-06,
      "logits/chosen": 252.7086639404297,
      "logits/rejected": 252.86575317382812,
      "logps/chosen": -1.3466806411743164,
      "logps/rejected": -1.4755653142929077,
      "loss": 1.4192,
      "odds_ratio_loss": 0.7249861359596252,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.13466808199882507,
      "rewards/margins": 0.012888476252555847,
      "rewards/rejected": -0.14755654335021973,
      "sft_loss": 1.3466806411743164,
      "step": 560
    },
    {
      "epoch": 1.0135585685707935,
      "grad_norm": 8.696991920471191,
      "learning_rate": 3.717857617249642e-06,
      "logits/chosen": 250.4936981201172,
      "logits/rejected": 250.49435424804688,
      "logps/chosen": -1.4108977317810059,
      "logps/rejected": -1.591428279876709,
      "loss": 1.4822,
      "odds_ratio_loss": 0.7131791710853577,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.1410897672176361,
      "rewards/margins": 0.018053067848086357,
      "rewards/rejected": -0.1591428518295288,
      "sft_loss": 1.4108977317810059,
      "step": 570
    },
    {
      "epoch": 1.0313402978439654,
      "grad_norm": 1.4178659915924072,
      "learning_rate": 3.6769636946284543e-06,
      "logits/chosen": 252.4293975830078,
      "logits/rejected": 252.90542602539062,
      "logps/chosen": -1.2866795063018799,
      "logps/rejected": -1.4288690090179443,
      "loss": 1.3561,
      "odds_ratio_loss": 0.694588840007782,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.12866798043251038,
      "rewards/margins": 0.01421893946826458,
      "rewards/rejected": -0.1428869068622589,
      "sft_loss": 1.2866795063018799,
      "step": 580
    },
    {
      "epoch": 1.049122027117137,
      "grad_norm": 2.9459097385406494,
      "learning_rate": 3.6356610889596355e-06,
      "logits/chosen": 251.82577514648438,
      "logits/rejected": 252.0190887451172,
      "logps/chosen": -1.3364391326904297,
      "logps/rejected": -1.4779332876205444,
      "loss": 1.4062,
      "odds_ratio_loss": 0.6974235773086548,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.13364391028881073,
      "rewards/margins": 0.014149405062198639,
      "rewards/rejected": -0.14779332280158997,
      "sft_loss": 1.3364391326904297,
      "step": 590
    },
    {
      "epoch": 1.066903756390309,
      "grad_norm": 2.037022590637207,
      "learning_rate": 3.593964141955541e-06,
      "logits/chosen": 251.60153198242188,
      "logits/rejected": 252.28738403320312,
      "logps/chosen": -1.3261809349060059,
      "logps/rejected": -1.4387245178222656,
      "loss": 1.3975,
      "odds_ratio_loss": 0.7132095098495483,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.13261809945106506,
      "rewards/margins": 0.011254330165684223,
      "rewards/rejected": -0.1438724398612976,
      "sft_loss": 1.3261809349060059,
      "step": 600
    },
    {
      "epoch": 1.0846854856634809,
      "grad_norm": 1.648507833480835,
      "learning_rate": 3.5518873322576573e-06,
      "logits/chosen": 251.1070556640625,
      "logits/rejected": 251.15121459960938,
      "logps/chosen": -1.3120733499526978,
      "logps/rejected": -1.5062081813812256,
      "loss": 1.3823,
      "odds_ratio_loss": 0.7022742033004761,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.13120731711387634,
      "rewards/margins": 0.019413486123085022,
      "rewards/rejected": -0.15062081813812256,
      "sft_loss": 1.3120733499526978,
      "step": 610
    },
    {
      "epoch": 1.1024672149366526,
      "grad_norm": 1.7841641902923584,
      "learning_rate": 3.5094452704091143e-06,
      "logits/chosen": 253.28305053710938,
      "logits/rejected": 253.123046875,
      "logps/chosen": -1.3503539562225342,
      "logps/rejected": -1.5104092359542847,
      "loss": 1.4195,
      "odds_ratio_loss": 0.6916577219963074,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.13503538072109222,
      "rewards/margins": 0.016005536541342735,
      "rewards/rejected": -0.1510409265756607,
      "sft_loss": 1.3503539562225342,
      "step": 620
    },
    {
      "epoch": 1.1202489442098245,
      "grad_norm": 4.146026611328125,
      "learning_rate": 3.46665269378139e-06,
      "logits/chosen": 251.23709106445312,
      "logits/rejected": 251.81686401367188,
      "logps/chosen": -1.3730881214141846,
      "logps/rejected": -1.4982974529266357,
      "loss": 1.4452,
      "odds_ratio_loss": 0.7208770513534546,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.13730882108211517,
      "rewards/margins": 0.012520933523774147,
      "rewards/rejected": -0.14982974529266357,
      "sft_loss": 1.3730881214141846,
      "step": 630
    },
    {
      "epoch": 1.1380306734829961,
      "grad_norm": 3.3884165287017822,
      "learning_rate": 3.4235244614569794e-06,
      "logits/chosen": 250.8208770751953,
      "logits/rejected": 251.1712646484375,
      "logps/chosen": -1.3837242126464844,
      "logps/rejected": -1.5583369731903076,
      "loss": 1.4557,
      "odds_ratio_loss": 0.7200697660446167,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.13837242126464844,
      "rewards/margins": 0.017461290583014488,
      "rewards/rejected": -0.15583372116088867,
      "sft_loss": 1.3837242126464844,
      "step": 640
    },
    {
      "epoch": 1.155812402756168,
      "grad_norm": 1.377685546875,
      "learning_rate": 3.3800755490698008e-06,
      "logits/chosen": 252.3297576904297,
      "logits/rejected": 252.5504150390625,
      "logps/chosen": -1.310136079788208,
      "logps/rejected": -1.5744563341140747,
      "loss": 1.3736,
      "odds_ratio_loss": 0.6345242857933044,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.13101361691951752,
      "rewards/margins": 0.026432007551193237,
      "rewards/rejected": -0.15744563937187195,
      "sft_loss": 1.310136079788208,
      "step": 650
    },
    {
      "epoch": 1.17359413202934,
      "grad_norm": 2.116525411605835,
      "learning_rate": 3.3363210436051287e-06,
      "logits/chosen": 252.10305786132812,
      "logits/rejected": 252.2167205810547,
      "logps/chosen": -1.348901629447937,
      "logps/rejected": -1.5241731405258179,
      "loss": 1.4188,
      "odds_ratio_loss": 0.6987627148628235,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.13489016890525818,
      "rewards/margins": 0.017527148127555847,
      "rewards/rejected": -0.15241730213165283,
      "sft_loss": 1.348901629447937,
      "step": 660
    },
    {
      "epoch": 1.1913758613025116,
      "grad_norm": 1.8333208560943604,
      "learning_rate": 3.292276138160867e-06,
      "logits/chosen": 251.9469757080078,
      "logits/rejected": 252.2017059326172,
      "logps/chosen": -1.331968069076538,
      "logps/rejected": -1.551393747329712,
      "loss": 1.4003,
      "odds_ratio_loss": 0.6828904151916504,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -0.13319680094718933,
      "rewards/margins": 0.021942555904388428,
      "rewards/rejected": -0.15513937175273895,
      "sft_loss": 1.331968069076538,
      "step": 670
    },
    {
      "epoch": 1.2091575905756835,
      "grad_norm": 2.233184576034546,
      "learning_rate": 3.2479561266719694e-06,
      "logits/chosen": 252.11386108398438,
      "logits/rejected": 252.0362548828125,
      "logps/chosen": -1.346163272857666,
      "logps/rejected": -1.5309796333312988,
      "loss": 1.4146,
      "odds_ratio_loss": 0.6838955283164978,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.13461633026599884,
      "rewards/margins": 0.01848164014518261,
      "rewards/rejected": -0.1530979573726654,
      "sft_loss": 1.346163272857666,
      "step": 680
    },
    {
      "epoch": 1.2269393198488552,
      "grad_norm": 4.622295379638672,
      "learning_rate": 3.2033763985998533e-06,
      "logits/chosen": 253.1451416015625,
      "logits/rejected": 253.5478973388672,
      "logps/chosen": -1.2959039211273193,
      "logps/rejected": -1.597389578819275,
      "loss": 1.3583,
      "odds_ratio_loss": 0.6243518590927124,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.12959036231040955,
      "rewards/margins": 0.03014858439564705,
      "rewards/rejected": -0.15973897278308868,
      "sft_loss": 1.2959039211273193,
      "step": 690
    },
    {
      "epoch": 1.244721049122027,
      "grad_norm": 2.6373684406280518,
      "learning_rate": 3.1585524335886335e-06,
      "logits/chosen": 252.57833862304688,
      "logits/rejected": 252.81259155273438,
      "logps/chosen": -1.2123761177062988,
      "logps/rejected": -1.4513623714447021,
      "loss": 1.2775,
      "odds_ratio_loss": 0.6515756845474243,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.121237613260746,
      "rewards/margins": 0.023898636922240257,
      "rewards/rejected": -0.1451362520456314,
      "sft_loss": 1.2123761177062988,
      "step": 700
    },
    {
      "epoch": 1.262502778395199,
      "grad_norm": 2.7742574214935303,
      "learning_rate": 3.1134997960900536e-06,
      "logits/chosen": 250.22891235351562,
      "logits/rejected": 251.8517608642578,
      "logps/chosen": -1.2099064588546753,
      "logps/rejected": -1.5609335899353027,
      "loss": 1.2689,
      "odds_ratio_loss": 0.5895074009895325,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.12099063396453857,
      "rewards/margins": 0.0351027250289917,
      "rewards/rejected": -0.15609335899353027,
      "sft_loss": 1.2099064588546753,
      "step": 710
    },
    {
      "epoch": 1.2802845076683709,
      "grad_norm": 1.7436786890029907,
      "learning_rate": 3.0682341299589583e-06,
      "logits/chosen": 252.24365234375,
      "logits/rejected": 252.2627410888672,
      "logps/chosen": -1.2945445775985718,
      "logps/rejected": -1.419683814048767,
      "loss": 1.3646,
      "odds_ratio_loss": 0.7010321617126465,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -0.12945446372032166,
      "rewards/margins": 0.012513910420238972,
      "rewards/rejected": -0.14196836948394775,
      "sft_loss": 1.2945445775985718,
      "step": 720
    },
    {
      "epoch": 1.2980662369415426,
      "grad_norm": 1.6442652940750122,
      "learning_rate": 3.022771153021201e-06,
      "logits/chosen": 252.22427368164062,
      "logits/rejected": 252.268310546875,
      "logps/chosen": -1.2626879215240479,
      "logps/rejected": -1.530016541481018,
      "loss": 1.3271,
      "odds_ratio_loss": 0.644324779510498,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.12626877427101135,
      "rewards/margins": 0.026732871308922768,
      "rewards/rejected": -0.15300165116786957,
      "sft_loss": 1.2626879215240479,
      "step": 730
    },
    {
      "epoch": 1.3158479662147144,
      "grad_norm": 2.3948917388916016,
      "learning_rate": 2.9771266516158625e-06,
      "logits/chosen": 252.8442840576172,
      "logits/rejected": 253.3193359375,
      "logps/chosen": -1.2903095483779907,
      "logps/rejected": -1.4887399673461914,
      "loss": 1.3586,
      "odds_ratio_loss": 0.682421088218689,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.12903094291687012,
      "rewards/margins": 0.019843045622110367,
      "rewards/rejected": -0.14887399971485138,
      "sft_loss": 1.2903095483779907,
      "step": 740
    },
    {
      "epoch": 1.3336296954878861,
      "grad_norm": 1.7583057880401611,
      "learning_rate": 2.9313164751136802e-06,
      "logits/chosen": 251.74462890625,
      "logits/rejected": 252.4144287109375,
      "logps/chosen": -1.3303029537200928,
      "logps/rejected": -1.4901565313339233,
      "loss": 1.3975,
      "odds_ratio_loss": 0.6716746091842651,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.13303029537200928,
      "rewards/margins": 0.015985365957021713,
      "rewards/rejected": -0.1490156650543213,
      "sft_loss": 1.3303029537200928,
      "step": 750
    },
    {
      "epoch": 1.351411424761058,
      "grad_norm": 2.620116710662842,
      "learning_rate": 2.8853565304135956e-06,
      "logits/chosen": 254.57931518554688,
      "logits/rejected": 254.56857299804688,
      "logps/chosen": -1.3150755167007446,
      "logps/rejected": -1.3985707759857178,
      "loss": 1.3882,
      "odds_ratio_loss": 0.730746865272522,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.13150756061077118,
      "rewards/margins": 0.008349532261490822,
      "rewards/rejected": -0.13985709846019745,
      "sft_loss": 1.3150755167007446,
      "step": 760
    },
    {
      "epoch": 1.36919315403423,
      "grad_norm": 6.955394744873047,
      "learning_rate": 2.839262776419313e-06,
      "logits/chosen": 253.0518035888672,
      "logits/rejected": 253.3127899169922,
      "logps/chosen": -1.2819963693618774,
      "logps/rejected": -1.5402740240097046,
      "loss": 1.346,
      "odds_ratio_loss": 0.6400705575942993,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.12819963693618774,
      "rewards/margins": 0.02582777664065361,
      "rewards/rejected": -0.15402741730213165,
      "sft_loss": 1.2819963693618774,
      "step": 770
    },
    {
      "epoch": 1.3869748833074016,
      "grad_norm": 2.157224416732788,
      "learning_rate": 2.793051218497817e-06,
      "logits/chosen": 253.73080444335938,
      "logits/rejected": 253.3892822265625,
      "logps/chosen": -1.324377179145813,
      "logps/rejected": -1.411966323852539,
      "loss": 1.3963,
      "odds_ratio_loss": 0.7193102240562439,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.13243773579597473,
      "rewards/margins": 0.00875892210751772,
      "rewards/rejected": -0.14119663834571838,
      "sft_loss": 1.324377179145813,
      "step": 780
    },
    {
      "epoch": 1.4047566125805735,
      "grad_norm": 1.813340663909912,
      "learning_rate": 2.7467379029217437e-06,
      "logits/chosen": 253.36312866210938,
      "logits/rejected": 254.00588989257812,
      "logps/chosen": -1.2761633396148682,
      "logps/rejected": -1.5060415267944336,
      "loss": 1.343,
      "odds_ratio_loss": 0.6682227849960327,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.12761631608009338,
      "rewards/margins": 0.022987816482782364,
      "rewards/rejected": -0.15060414373874664,
      "sft_loss": 1.2761633396148682,
      "step": 790
    },
    {
      "epoch": 1.4225383418537452,
      "grad_norm": 1.6623824834823608,
      "learning_rate": 2.7003389112975546e-06,
      "logits/chosen": 253.5782470703125,
      "logits/rejected": 253.32089233398438,
      "logps/chosen": -1.3095474243164062,
      "logps/rejected": -1.491077184677124,
      "loss": 1.3757,
      "odds_ratio_loss": 0.6613184213638306,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.13095474243164062,
      "rewards/margins": 0.018152965232729912,
      "rewards/rejected": -0.14910772442817688,
      "sft_loss": 1.3095474243164062,
      "step": 800
    },
    {
      "epoch": 1.440320071126917,
      "grad_norm": 5.644952774047852,
      "learning_rate": 2.653870354981437e-06,
      "logits/chosen": 252.7183380126953,
      "logits/rejected": 253.2029571533203,
      "logps/chosen": -1.2087516784667969,
      "logps/rejected": -1.411168098449707,
      "loss": 1.2748,
      "odds_ratio_loss": 0.6601918935775757,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.12087517976760864,
      "rewards/margins": 0.020241642370820045,
      "rewards/rejected": -0.14111682772636414,
      "sft_loss": 1.2087516784667969,
      "step": 810
    },
    {
      "epoch": 1.458101800400089,
      "grad_norm": 4.477853298187256,
      "learning_rate": 2.6073483694848777e-06,
      "logits/chosen": 253.3741912841797,
      "logits/rejected": 253.51382446289062,
      "logps/chosen": -1.2232033014297485,
      "logps/rejected": -1.4460740089416504,
      "loss": 1.2895,
      "odds_ratio_loss": 0.663042426109314,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.12232033163309097,
      "rewards/margins": 0.02228708192706108,
      "rewards/rejected": -0.14460742473602295,
      "sft_loss": 1.2232033014297485,
      "step": 820
    },
    {
      "epoch": 1.4758835296732609,
      "grad_norm": 1.6548304557800293,
      "learning_rate": 2.560789108871847e-06,
      "logits/chosen": 253.0503692626953,
      "logits/rejected": 252.9963836669922,
      "logps/chosen": -1.3039335012435913,
      "logps/rejected": -1.57505202293396,
      "loss": 1.3696,
      "odds_ratio_loss": 0.6571396589279175,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.1303933560848236,
      "rewards/margins": 0.027111848816275597,
      "rewards/rejected": -0.15750519931316376,
      "sft_loss": 1.3039335012435913,
      "step": 830
    },
    {
      "epoch": 1.4936652589464325,
      "grad_norm": 5.2365007400512695,
      "learning_rate": 2.514208740149544e-06,
      "logits/chosen": 253.81640625,
      "logits/rejected": 253.57717895507812,
      "logps/chosen": -1.3866493701934814,
      "logps/rejected": -1.5284301042556763,
      "loss": 1.4596,
      "odds_ratio_loss": 0.7293277978897095,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13866494596004486,
      "rewards/margins": 0.014178059995174408,
      "rewards/rejected": -0.15284302830696106,
      "sft_loss": 1.3866493701934814,
      "step": 840
    },
    {
      "epoch": 1.5114469882196042,
      "grad_norm": 2.916973114013672,
      "learning_rate": 2.46762343765464e-06,
      "logits/chosen": 254.6901397705078,
      "logits/rejected": 254.9998016357422,
      "logps/chosen": -1.3546521663665771,
      "logps/rejected": -1.587242841720581,
      "loss": 1.4197,
      "odds_ratio_loss": 0.6507903933525085,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.13546521961688995,
      "rewards/margins": 0.02325906977057457,
      "rewards/rejected": -0.15872427821159363,
      "sft_loss": 1.3546521663665771,
      "step": 850
    },
    {
      "epoch": 1.5292287174927761,
      "grad_norm": 2.888098955154419,
      "learning_rate": 2.4210493774369903e-06,
      "logits/chosen": 253.61563110351562,
      "logits/rejected": 253.7008056640625,
      "logps/chosen": -1.3398011922836304,
      "logps/rejected": -1.5939433574676514,
      "loss": 1.4092,
      "odds_ratio_loss": 0.6944610476493835,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.13398012518882751,
      "rewards/margins": 0.02541421353816986,
      "rewards/rejected": -0.15939433872699738,
      "sft_loss": 1.3398011922836304,
      "step": 860
    },
    {
      "epoch": 1.547010446765948,
      "grad_norm": 1.7133105993270874,
      "learning_rate": 2.374502731642732e-06,
      "logits/chosen": 253.13046264648438,
      "logits/rejected": 253.28274536132812,
      "logps/chosen": -1.2972921133041382,
      "logps/rejected": -1.5040593147277832,
      "loss": 1.3657,
      "odds_ratio_loss": 0.6844374537467957,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.12972919642925262,
      "rewards/margins": 0.02067672833800316,
      "rewards/rejected": -0.15040592849254608,
      "sft_loss": 1.2972921133041382,
      "step": 870
    },
    {
      "epoch": 1.56479217603912,
      "grad_norm": 1.869072437286377,
      "learning_rate": 2.3279996628987556e-06,
      "logits/chosen": 253.9283447265625,
      "logits/rejected": 254.41073608398438,
      "logps/chosen": -1.2677370309829712,
      "logps/rejected": -1.4599510431289673,
      "loss": 1.3369,
      "odds_ratio_loss": 0.6917437314987183,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.1267736852169037,
      "rewards/margins": 0.01922140084207058,
      "rewards/rejected": -0.1459950953722,
      "sft_loss": 1.2677370309829712,
      "step": 880
    },
    {
      "epoch": 1.5825739053122916,
      "grad_norm": 2.387537717819214,
      "learning_rate": 2.281556318700474e-06,
      "logits/chosen": 253.6074676513672,
      "logits/rejected": 253.7229461669922,
      "logps/chosen": -1.230033278465271,
      "logps/rejected": -1.369767189025879,
      "loss": 1.3007,
      "odds_ratio_loss": 0.7063087224960327,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.12300334125757217,
      "rewards/margins": 0.013973374851047993,
      "rewards/rejected": -0.1369767189025879,
      "sft_loss": 1.230033278465271,
      "step": 890
    },
    {
      "epoch": 1.6003556345854635,
      "grad_norm": 7.490704536437988,
      "learning_rate": 2.2351888258048408e-06,
      "logits/chosen": 251.65274047851562,
      "logits/rejected": 251.5640106201172,
      "logps/chosen": -1.2508928775787354,
      "logps/rejected": -1.4577651023864746,
      "loss": 1.3169,
      "odds_ratio_loss": 0.6601670980453491,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.12508928775787354,
      "rewards/margins": 0.020687231793999672,
      "rewards/rejected": -0.14577652513980865,
      "sft_loss": 1.2508928775787354,
      "step": 900
    },
    {
      "epoch": 1.6181373638586352,
      "grad_norm": 1.7122284173965454,
      "learning_rate": 2.188913284630584e-06,
      "logits/chosen": 253.48501586914062,
      "logits/rejected": 253.5773468017578,
      "logps/chosen": -1.3153185844421387,
      "logps/rejected": -1.4097468852996826,
      "loss": 1.3867,
      "odds_ratio_loss": 0.7141562700271606,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.13153186440467834,
      "rewards/margins": 0.009442826732993126,
      "rewards/rejected": -0.1409747153520584,
      "sft_loss": 1.3153185844421387,
      "step": 910
    },
    {
      "epoch": 1.635919093131807,
      "grad_norm": 3.5988399982452393,
      "learning_rate": 2.1427457636675652e-06,
      "logits/chosen": 254.33633422851562,
      "logits/rejected": 253.96981811523438,
      "logps/chosen": -1.2676795721054077,
      "logps/rejected": -1.4219448566436768,
      "loss": 1.3387,
      "odds_ratio_loss": 0.7098127007484436,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.12676796317100525,
      "rewards/margins": 0.015426510944962502,
      "rewards/rejected": -0.1421944797039032,
      "sft_loss": 1.2676795721054077,
      "step": 920
    },
    {
      "epoch": 1.653700822404979,
      "grad_norm": 1.8244489431381226,
      "learning_rate": 2.096702293897247e-06,
      "logits/chosen": 254.08718872070312,
      "logits/rejected": 254.5830535888672,
      "logps/chosen": -1.283644437789917,
      "logps/rejected": -1.5567032098770142,
      "loss": 1.35,
      "odds_ratio_loss": 0.6635793447494507,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.1283644437789917,
      "rewards/margins": 0.027305880561470985,
      "rewards/rejected": -0.15567031502723694,
      "sft_loss": 1.283644437789917,
      "step": 930
    },
    {
      "epoch": 1.6714825516781509,
      "grad_norm": 3.096039056777954,
      "learning_rate": 2.0507988632261672e-06,
      "logits/chosen": 252.3622589111328,
      "logits/rejected": 252.93716430664062,
      "logps/chosen": -1.2625596523284912,
      "logps/rejected": -1.5456993579864502,
      "loss": 1.3258,
      "odds_ratio_loss": 0.6320444345474243,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.12625595927238464,
      "rewards/margins": 0.028313983231782913,
      "rewards/rejected": -0.15456993877887726,
      "sft_loss": 1.2625596523284912,
      "step": 940
    },
    {
      "epoch": 1.6892642809513225,
      "grad_norm": 3.4061944484710693,
      "learning_rate": 2.005051410934382e-06,
      "logits/chosen": 253.2592010498047,
      "logits/rejected": 253.57846069335938,
      "logps/chosen": -1.3725305795669556,
      "logps/rejected": -1.5077531337738037,
      "loss": 1.4437,
      "odds_ratio_loss": 0.7114490866661072,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.137253075838089,
      "rewards/margins": 0.013522250577807426,
      "rewards/rejected": -0.15077531337738037,
      "sft_loss": 1.3725305795669556,
      "step": 950
    },
    {
      "epoch": 1.7070460102244942,
      "grad_norm": 2.6905410289764404,
      "learning_rate": 1.9594758221407843e-06,
      "logits/chosen": 254.19900512695312,
      "logits/rejected": 254.83987426757812,
      "logps/chosen": -1.2452795505523682,
      "logps/rejected": -1.5318635702133179,
      "loss": 1.3081,
      "odds_ratio_loss": 0.6286849975585938,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.1245279535651207,
      "rewards/margins": 0.02865840494632721,
      "rewards/rejected": -0.1531863659620285,
      "sft_loss": 1.2452795505523682,
      "step": 960
    },
    {
      "epoch": 1.724827739497666,
      "grad_norm": 2.9211478233337402,
      "learning_rate": 1.9140879222872408e-06,
      "logits/chosen": 254.0661163330078,
      "logits/rejected": 254.55819702148438,
      "logps/chosen": -1.274641752243042,
      "logps/rejected": -1.3871382474899292,
      "loss": 1.3478,
      "odds_ratio_loss": 0.7320677042007446,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.1274641752243042,
      "rewards/margins": 0.011249655857682228,
      "rewards/rejected": -0.13871383666992188,
      "sft_loss": 1.274641752243042,
      "step": 970
    },
    {
      "epoch": 1.742609468770838,
      "grad_norm": 2.0924360752105713,
      "learning_rate": 1.8689034716434346e-06,
      "logits/chosen": 253.67953491210938,
      "logits/rejected": 254.02993774414062,
      "logps/chosen": -1.334486722946167,
      "logps/rejected": -1.4547626972198486,
      "loss": 1.4054,
      "odds_ratio_loss": 0.7087211012840271,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.13344867527484894,
      "rewards/margins": 0.01202760823071003,
      "rewards/rejected": -0.14547628164291382,
      "sft_loss": 1.334486722946167,
      "step": 980
    },
    {
      "epoch": 1.76039119804401,
      "grad_norm": 2.326310634613037,
      "learning_rate": 1.8239381598343576e-06,
      "logits/chosen": 255.02243041992188,
      "logits/rejected": 255.69680786132812,
      "logps/chosen": -1.28658127784729,
      "logps/rejected": -1.4356472492218018,
      "loss": 1.3565,
      "odds_ratio_loss": 0.6996346712112427,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.12865814566612244,
      "rewards/margins": 0.014906583353877068,
      "rewards/rejected": -0.14356473088264465,
      "sft_loss": 1.28658127784729,
      "step": 990
    },
    {
      "epoch": 1.7781729273171816,
      "grad_norm": 5.255044460296631,
      "learning_rate": 1.779207600392312e-06,
      "logits/chosen": 253.11575317382812,
      "logits/rejected": 253.3959197998047,
      "logps/chosen": -1.3602896928787231,
      "logps/rejected": -1.5267302989959717,
      "loss": 1.428,
      "odds_ratio_loss": 0.6775819063186646,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.1360289752483368,
      "rewards/margins": 0.016644055023789406,
      "rewards/rejected": -0.15267305076122284,
      "sft_loss": 1.3602896928787231,
      "step": 1000
    },
    {
      "epoch": 1.7781729273171816,
      "eval_logits/chosen": 253.95806884765625,
      "eval_logits/rejected": 254.13504028320312,
      "eval_logps/chosen": -1.2936838865280151,
      "eval_logps/rejected": -1.5090892314910889,
      "eval_loss": 1.3595393896102905,
      "eval_odds_ratio_loss": 0.6585569381713867,
      "eval_rewards/accuracies": 0.5600000023841858,
      "eval_rewards/chosen": -0.1293683797121048,
      "eval_rewards/margins": 0.02154054492712021,
      "eval_rewards/rejected": -0.1509089171886444,
      "eval_runtime": 217.2988,
      "eval_samples_per_second": 4.602,
      "eval_sft_loss": 1.2936838865280151,
      "eval_steps_per_second": 2.301,
      "step": 1000
    },
    {
      "epoch": 1.7959546565903532,
      "grad_norm": 2.5635573863983154,
      "learning_rate": 1.7347273253353552e-06,
      "logits/chosen": 254.41915893554688,
      "logits/rejected": 254.5049285888672,
      "logps/chosen": -1.2720413208007812,
      "logps/rejected": -1.4322283267974854,
      "loss": 1.3422,
      "odds_ratio_loss": 0.7016333341598511,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.12720413506031036,
      "rewards/margins": 0.01601869985461235,
      "rewards/rejected": -0.14322282373905182,
      "sft_loss": 1.2720413208007812,
      "step": 1010
    },
    {
      "epoch": 1.8137363858635251,
      "grad_norm": 5.554447650909424,
      "learning_rate": 1.690512779774029e-06,
      "logits/chosen": 253.4269561767578,
      "logits/rejected": 253.6225128173828,
      "logps/chosen": -1.3230665922164917,
      "logps/rejected": -1.678013563156128,
      "loss": 1.3849,
      "odds_ratio_loss": 0.6180821657180786,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.13230668008327484,
      "rewards/margins": 0.035494692623615265,
      "rewards/rejected": -0.16780135035514832,
      "sft_loss": 1.3230665922164917,
      "step": 1020
    },
    {
      "epoch": 1.831518115136697,
      "grad_norm": 2.634599447250366,
      "learning_rate": 1.6465793165482838e-06,
      "logits/chosen": 253.33926391601562,
      "logits/rejected": 253.8477020263672,
      "logps/chosen": -1.2152363061904907,
      "logps/rejected": -1.465906023979187,
      "loss": 1.2779,
      "odds_ratio_loss": 0.6263755559921265,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.12152364104986191,
      "rewards/margins": 0.025066960602998734,
      "rewards/rejected": -0.14659060537815094,
      "sft_loss": 1.2152363061904907,
      "step": 1030
    },
    {
      "epoch": 1.849299844409869,
      "grad_norm": 3.051729917526245,
      "learning_rate": 1.6029421908964305e-06,
      "logits/chosen": 252.26016235351562,
      "logits/rejected": 253.5745086669922,
      "logps/chosen": -1.2302608489990234,
      "logps/rejected": -1.62982976436615,
      "loss": 1.2943,
      "odds_ratio_loss": 0.6404699683189392,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.12302607297897339,
      "rewards/margins": 0.03995689004659653,
      "rewards/rejected": -0.1629829704761505,
      "sft_loss": 1.2302608489990234,
      "step": 1040
    },
    {
      "epoch": 1.8670815736830408,
      "grad_norm": 5.069088459014893,
      "learning_rate": 1.559616555157985e-06,
      "logits/chosen": 253.4644775390625,
      "logits/rejected": 253.29727172851562,
      "logps/chosen": -1.2889888286590576,
      "logps/rejected": -1.4968655109405518,
      "loss": 1.3589,
      "odds_ratio_loss": 0.6988304853439331,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.12889888882637024,
      "rewards/margins": 0.020787667483091354,
      "rewards/rejected": -0.1496865451335907,
      "sft_loss": 1.2889888286590576,
      "step": 1050
    },
    {
      "epoch": 1.8848633029562125,
      "grad_norm": 2.464268207550049,
      "learning_rate": 1.516617453512252e-06,
      "logits/chosen": 254.87197875976562,
      "logits/rejected": 254.7659454345703,
      "logps/chosen": -1.351043462753296,
      "logps/rejected": -1.4525654315948486,
      "loss": 1.4237,
      "odds_ratio_loss": 0.7263092398643494,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.13510434329509735,
      "rewards/margins": 0.010152206756174564,
      "rewards/rejected": -0.14525654911994934,
      "sft_loss": 1.351043462753296,
      "step": 1060
    },
    {
      "epoch": 1.9026450322293842,
      "grad_norm": 10.498387336730957,
      "learning_rate": 1.473959816754449e-06,
      "logits/chosen": 253.4375457763672,
      "logits/rejected": 254.0106201171875,
      "logps/chosen": -1.2346054315567017,
      "logps/rejected": -1.3691545724868774,
      "loss": 1.3067,
      "odds_ratio_loss": 0.721213161945343,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.123460553586483,
      "rewards/margins": 0.013454906642436981,
      "rewards/rejected": -0.13691547513008118,
      "sft_loss": 1.2346054315567017,
      "step": 1070
    },
    {
      "epoch": 1.920426761502556,
      "grad_norm": 4.5994696617126465,
      "learning_rate": 1.4316584571112213e-06,
      "logits/chosen": 256.06500244140625,
      "logits/rejected": 256.2422790527344,
      "logps/chosen": -1.3240019083023071,
      "logps/rejected": -1.4346423149108887,
      "loss": 1.3965,
      "odds_ratio_loss": 0.7249402403831482,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.13240018486976624,
      "rewards/margins": 0.011064060032367706,
      "rewards/rejected": -0.14346425235271454,
      "sft_loss": 1.3240019083023071,
      "step": 1080
    },
    {
      "epoch": 1.938208490775728,
      "grad_norm": 3.5848639011383057,
      "learning_rate": 1.389728063097306e-06,
      "logits/chosen": 255.7421112060547,
      "logits/rejected": 255.6949005126953,
      "logps/chosen": -1.3223278522491455,
      "logps/rejected": -1.6033687591552734,
      "loss": 1.3875,
      "odds_ratio_loss": 0.6514922976493835,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.13223278522491455,
      "rewards/margins": 0.028104085475206375,
      "rewards/rejected": -0.16033688187599182,
      "sft_loss": 1.3223278522491455,
      "step": 1090
    },
    {
      "epoch": 1.9559902200488999,
      "grad_norm": 2.085165500640869,
      "learning_rate": 1.348183194415179e-06,
      "logits/chosen": 254.6555938720703,
      "logits/rejected": 255.1551513671875,
      "logps/chosen": -1.2908974885940552,
      "logps/rejected": -1.5859901905059814,
      "loss": 1.3544,
      "odds_ratio_loss": 0.6346372365951538,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.12908974289894104,
      "rewards/margins": 0.029509279876947403,
      "rewards/rejected": -0.15859903395175934,
      "sft_loss": 1.2908974885940552,
      "step": 1100
    },
    {
      "epoch": 1.9737719493220716,
      "grad_norm": 2.3228931427001953,
      "learning_rate": 1.3070382768994015e-06,
      "logits/chosen": 254.2451171875,
      "logits/rejected": 254.3721160888672,
      "logps/chosen": -1.2544920444488525,
      "logps/rejected": -1.4442470073699951,
      "loss": 1.3195,
      "odds_ratio_loss": 0.6503337621688843,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.12544921040534973,
      "rewards/margins": 0.018975483253598213,
      "rewards/rejected": -0.1444246917963028,
      "sft_loss": 1.2544920444488525,
      "step": 1110
    },
    {
      "epoch": 1.9915536785952432,
      "grad_norm": 2.0866916179656982,
      "learning_rate": 1.2663075975074746e-06,
      "logits/chosen": 253.80899047851562,
      "logits/rejected": 254.31460571289062,
      "logps/chosen": -1.2685939073562622,
      "logps/rejected": -1.4882584810256958,
      "loss": 1.3361,
      "odds_ratio_loss": 0.6750525236129761,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.1268593817949295,
      "rewards/margins": 0.021966462954878807,
      "rewards/rejected": -0.14882585406303406,
      "sft_loss": 1.2685939073562622,
      "step": 1120
    },
    {
      "epoch": 2.009335407868415,
      "grad_norm": 5.936447620391846,
      "learning_rate": 1.2260052993589034e-06,
      "logits/chosen": 253.43161010742188,
      "logits/rejected": 253.4159698486328,
      "logps/chosen": -1.3823201656341553,
      "logps/rejected": -1.4824841022491455,
      "loss": 1.4548,
      "odds_ratio_loss": 0.7252174615859985,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.1382320076227188,
      "rewards/margins": 0.010016398504376411,
      "rewards/rejected": -0.14824840426445007,
      "sft_loss": 1.3823201656341553,
      "step": 1130
    },
    {
      "epoch": 2.027117137141587,
      "grad_norm": 1.858097791671753,
      "learning_rate": 1.1861453768242099e-06,
      "logits/chosen": 255.3394317626953,
      "logits/rejected": 256.12109375,
      "logps/chosen": -1.223772644996643,
      "logps/rejected": -1.5006752014160156,
      "loss": 1.2835,
      "odds_ratio_loss": 0.5976046323776245,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.12237726151943207,
      "rewards/margins": 0.027690261602401733,
      "rewards/rejected": -0.1500675231218338,
      "sft_loss": 1.223772644996643,
      "step": 1140
    },
    {
      "epoch": 2.044898866414759,
      "grad_norm": 9.576284408569336,
      "learning_rate": 1.1467416706655982e-06,
      "logits/chosen": 255.4516143798828,
      "logits/rejected": 255.5124969482422,
      "logps/chosen": -1.33961021900177,
      "logps/rejected": -1.5424822568893433,
      "loss": 1.4101,
      "odds_ratio_loss": 0.7047215700149536,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.13396099209785461,
      "rewards/margins": 0.020287221297621727,
      "rewards/rejected": -0.1542482227087021,
      "sft_loss": 1.33961021900177,
      "step": 1150
    },
    {
      "epoch": 2.062680595687931,
      "grad_norm": 2.312715768814087,
      "learning_rate": 1.1078078632309559e-06,
      "logits/chosen": 255.4163360595703,
      "logits/rejected": 256.0242004394531,
      "logps/chosen": -1.2566642761230469,
      "logps/rejected": -1.4380619525909424,
      "loss": 1.3236,
      "odds_ratio_loss": 0.6697817444801331,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.12566642463207245,
      "rewards/margins": 0.018139759078621864,
      "rewards/rejected": -0.14380618929862976,
      "sft_loss": 1.2566642761230469,
      "step": 1160
    },
    {
      "epoch": 2.0804623249611023,
      "grad_norm": 2.0991952419281006,
      "learning_rate": 1.0693574737028627e-06,
      "logits/chosen": 254.43576049804688,
      "logits/rejected": 255.13711547851562,
      "logps/chosen": -1.2939661741256714,
      "logps/rejected": -1.4546544551849365,
      "loss": 1.3642,
      "odds_ratio_loss": 0.7022632360458374,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.12939661741256714,
      "rewards/margins": 0.01606883481144905,
      "rewards/rejected": -0.14546546339988708,
      "sft_loss": 1.2939661741256714,
      "step": 1170
    },
    {
      "epoch": 2.098244054234274,
      "grad_norm": 4.281792640686035,
      "learning_rate": 1.0314038534042586e-06,
      "logits/chosen": 254.57168579101562,
      "logits/rejected": 255.064453125,
      "logps/chosen": -1.1770070791244507,
      "logps/rejected": -1.3855504989624023,
      "loss": 1.2441,
      "odds_ratio_loss": 0.6709107756614685,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.1177007183432579,
      "rewards/margins": 0.020854344591498375,
      "rewards/rejected": -0.13855504989624023,
      "sft_loss": 1.1770070791244507,
      "step": 1180
    },
    {
      "epoch": 2.116025783507446,
      "grad_norm": 1.9617539644241333,
      "learning_rate": 9.939601811623946e-07,
      "logits/chosen": 253.7810516357422,
      "logits/rejected": 254.1466827392578,
      "logps/chosen": -1.2411329746246338,
      "logps/rejected": -1.4748810529708862,
      "loss": 1.3104,
      "odds_ratio_loss": 0.692661702632904,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.1241132840514183,
      "rewards/margins": 0.02337482199072838,
      "rewards/rejected": -0.14748811721801758,
      "sft_loss": 1.2411329746246338,
      "step": 1190
    },
    {
      "epoch": 2.133807512780618,
      "grad_norm": 2.653927803039551,
      "learning_rate": 9.570394587326825e-07,
      "logits/chosen": 253.3402099609375,
      "logits/rejected": 253.52310180664062,
      "logps/chosen": -1.2719714641571045,
      "logps/rejected": -1.571843147277832,
      "loss": 1.3354,
      "odds_ratio_loss": 0.6337886452674866,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.12719716131687164,
      "rewards/margins": 0.02998717501759529,
      "rewards/rejected": -0.15718431770801544,
      "sft_loss": 1.2719714641571045,
      "step": 1200
    },
    {
      "epoch": 2.15158924205379,
      "grad_norm": 1.7235380411148071,
      "learning_rate": 9.206545062840302e-07,
      "logits/chosen": 255.1512908935547,
      "logits/rejected": 255.9069366455078,
      "logps/chosen": -1.2449194192886353,
      "logps/rejected": -1.4648315906524658,
      "loss": 1.3088,
      "odds_ratio_loss": 0.6390582323074341,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.12449195235967636,
      "rewards/margins": 0.021991217508912086,
      "rewards/rejected": -0.1464831531047821,
      "sft_loss": 1.2449194192886353,
      "step": 1210
    },
    {
      "epoch": 2.1693709713269618,
      "grad_norm": 2.0629489421844482,
      "learning_rate": 8.848179579472285e-07,
      "logits/chosen": 255.0786590576172,
      "logits/rejected": 255.07534790039062,
      "logps/chosen": -1.2237968444824219,
      "logps/rejected": -1.3279929161071777,
      "loss": 1.2941,
      "odds_ratio_loss": 0.7028827667236328,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.12237968295812607,
      "rewards/margins": 0.010419609025120735,
      "rewards/rejected": -0.13279929757118225,
      "sft_loss": 1.2237968444824219,
      "step": 1220
    },
    {
      "epoch": 2.1871527006001332,
      "grad_norm": 4.134257793426514,
      "learning_rate": 8.495422574279403e-07,
      "logits/chosen": 253.44869995117188,
      "logits/rejected": 253.70126342773438,
      "logps/chosen": -1.201407551765442,
      "logps/rejected": -1.4721620082855225,
      "loss": 1.2647,
      "odds_ratio_loss": 0.633193850517273,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.12014075368642807,
      "rewards/margins": 0.02707546018064022,
      "rewards/rejected": -0.14721623063087463,
      "sft_loss": 1.201407551765442,
      "step": 1230
    },
    {
      "epoch": 2.204934429873305,
      "grad_norm": 2.6145901679992676,
      "learning_rate": 8.148396536858063e-07,
      "logits/chosen": 255.00973510742188,
      "logits/rejected": 255.087646484375,
      "logps/chosen": -1.311307668685913,
      "logps/rejected": -1.5930455923080444,
      "loss": 1.3784,
      "odds_ratio_loss": 0.670753538608551,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.13113076984882355,
      "rewards/margins": 0.028173794969916344,
      "rewards/rejected": -0.15930457413196564,
      "sft_loss": 1.311307668685913,
      "step": 1240
    },
    {
      "epoch": 2.222716159146477,
      "grad_norm": 1.5492827892303467,
      "learning_rate": 7.807221966811815e-07,
      "logits/chosen": 255.0123291015625,
      "logits/rejected": 256.08282470703125,
      "logps/chosen": -1.222886323928833,
      "logps/rejected": -1.411444067955017,
      "loss": 1.2915,
      "odds_ratio_loss": 0.6862080097198486,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.12228864431381226,
      "rewards/margins": 0.01885577104985714,
      "rewards/rejected": -0.14114442467689514,
      "sft_loss": 1.222886323928833,
      "step": 1250
    },
    {
      "epoch": 2.240497888419649,
      "grad_norm": 1.9974987506866455,
      "learning_rate": 7.47201733190962e-07,
      "logits/chosen": 254.1110382080078,
      "logits/rejected": 253.8502197265625,
      "logps/chosen": -1.2319315671920776,
      "logps/rejected": -1.3752238750457764,
      "loss": 1.2993,
      "odds_ratio_loss": 0.6734907627105713,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.12319314479827881,
      "rewards/margins": 0.01432921551167965,
      "rewards/rejected": -0.1375223696231842,
      "sft_loss": 1.2319315671920776,
      "step": 1260
    },
    {
      "epoch": 2.258279617692821,
      "grad_norm": 2.013181209564209,
      "learning_rate": 7.142899026949721e-07,
      "logits/chosen": 255.99240112304688,
      "logits/rejected": 255.9341278076172,
      "logps/chosen": -1.2691768407821655,
      "logps/rejected": -1.4378575086593628,
      "loss": 1.3351,
      "odds_ratio_loss": 0.6596958637237549,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.12691769003868103,
      "rewards/margins": 0.016868067905306816,
      "rewards/rejected": -0.143785759806633,
      "sft_loss": 1.2691768407821655,
      "step": 1270
    },
    {
      "epoch": 2.2760613469659923,
      "grad_norm": 5.814607620239258,
      "learning_rate": 6.819981333343273e-07,
      "logits/chosen": 253.1020965576172,
      "logits/rejected": 253.30661010742188,
      "logps/chosen": -1.2039464712142944,
      "logps/rejected": -1.4708151817321777,
      "loss": 1.2691,
      "odds_ratio_loss": 0.6517861485481262,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.12039466202259064,
      "rewards/margins": 0.026686882600188255,
      "rewards/rejected": -0.14708155393600464,
      "sft_loss": 1.2039464712142944,
      "step": 1280
    },
    {
      "epoch": 2.293843076239164,
      "grad_norm": 2.745338201522827,
      "learning_rate": 6.503376379431839e-07,
      "logits/chosen": 253.5093536376953,
      "logits/rejected": 253.670654296875,
      "logps/chosen": -1.3510358333587646,
      "logps/rejected": -1.4214909076690674,
      "loss": 1.423,
      "odds_ratio_loss": 0.7199310660362244,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.13510356843471527,
      "rewards/margins": 0.007045503705739975,
      "rewards/rejected": -0.14214907586574554,
      "sft_loss": 1.3510358333587646,
      "step": 1290
    },
    {
      "epoch": 2.311624805512336,
      "grad_norm": 2.5681943893432617,
      "learning_rate": 6.193194101552502e-07,
      "logits/chosen": 256.16900634765625,
      "logits/rejected": 256.3623962402344,
      "logps/chosen": -1.24690580368042,
      "logps/rejected": -1.4972703456878662,
      "loss": 1.3081,
      "odds_ratio_loss": 0.6117135286331177,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.12469057738780975,
      "rewards/margins": 0.02503645047545433,
      "rewards/rejected": -0.14972704648971558,
      "sft_loss": 1.24690580368042,
      "step": 1300
    },
    {
      "epoch": 2.329406534785508,
      "grad_norm": 3.2992701530456543,
      "learning_rate": 5.889542205864083e-07,
      "logits/chosen": 254.740966796875,
      "logits/rejected": 255.06307983398438,
      "logps/chosen": -1.2395174503326416,
      "logps/rejected": -1.4891048669815063,
      "loss": 1.303,
      "odds_ratio_loss": 0.6343892812728882,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.1239517480134964,
      "rewards/margins": 0.024958748370409012,
      "rewards/rejected": -0.1489104926586151,
      "sft_loss": 1.2395174503326416,
      "step": 1310
    },
    {
      "epoch": 2.34718826405868,
      "grad_norm": 2.295304298400879,
      "learning_rate": 5.592526130947862e-07,
      "logits/chosen": 253.2501220703125,
      "logits/rejected": 253.6666717529297,
      "logps/chosen": -1.3131468296051025,
      "logps/rejected": -1.5006693601608276,
      "loss": 1.3841,
      "odds_ratio_loss": 0.709243893623352,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.13131467998027802,
      "rewards/margins": 0.018752261996269226,
      "rewards/rejected": -0.15006692707538605,
      "sft_loss": 1.3131468296051025,
      "step": 1320
    },
    {
      "epoch": 2.3649699933318518,
      "grad_norm": 3.0129616260528564,
      "learning_rate": 5.302249011195507e-07,
      "logits/chosen": 254.51284790039062,
      "logits/rejected": 254.08145141601562,
      "logps/chosen": -1.2322486639022827,
      "logps/rejected": -1.3891911506652832,
      "loss": 1.3008,
      "odds_ratio_loss": 0.6851211786270142,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.12322486937046051,
      "rewards/margins": 0.015694255009293556,
      "rewards/rejected": -0.13891912996768951,
      "sft_loss": 1.2322486639022827,
      "step": 1330
    },
    {
      "epoch": 2.382751722605023,
      "grad_norm": 4.828993797302246,
      "learning_rate": 5.018811640997307e-07,
      "logits/chosen": 255.24087524414062,
      "logits/rejected": 255.8361358642578,
      "logps/chosen": -1.3192299604415894,
      "logps/rejected": -1.6515556573867798,
      "loss": 1.3833,
      "odds_ratio_loss": 0.6403645873069763,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.13192300498485565,
      "rewards/margins": 0.03323255851864815,
      "rewards/rejected": -0.1651555597782135,
      "sft_loss": 1.3192299604415894,
      "step": 1340
    },
    {
      "epoch": 2.400533451878195,
      "grad_norm": 1.868326187133789,
      "learning_rate": 4.7423124397427105e-07,
      "logits/chosen": 253.5162811279297,
      "logits/rejected": 253.6891632080078,
      "logps/chosen": -1.2553585767745972,
      "logps/rejected": -1.4340143203735352,
      "loss": 1.3209,
      "odds_ratio_loss": 0.6558007001876831,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.12553584575653076,
      "rewards/margins": 0.017865588888525963,
      "rewards/rejected": -0.14340144395828247,
      "sft_loss": 1.2553585767745972,
      "step": 1350
    },
    {
      "epoch": 2.418315181151367,
      "grad_norm": 3.549006938934326,
      "learning_rate": 4.472847417645787e-07,
      "logits/chosen": 255.4973602294922,
      "logits/rejected": 255.242431640625,
      "logps/chosen": -1.3243919610977173,
      "logps/rejected": -1.665623426437378,
      "loss": 1.3842,
      "odds_ratio_loss": 0.5979620814323425,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.13243919610977173,
      "rewards/margins": 0.03412316367030144,
      "rewards/rejected": -0.16656234860420227,
      "sft_loss": 1.3243919610977173,
      "step": 1360
    },
    {
      "epoch": 2.436096910424539,
      "grad_norm": 1.720616340637207,
      "learning_rate": 4.210510142406993e-07,
      "logits/chosen": 254.47842407226562,
      "logits/rejected": 254.3418731689453,
      "logps/chosen": -1.2948418855667114,
      "logps/rejected": -1.6191186904907227,
      "loss": 1.3596,
      "odds_ratio_loss": 0.6471132040023804,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.1294841766357422,
      "rewards/margins": 0.03242768719792366,
      "rewards/rejected": -0.16191187500953674,
      "sft_loss": 1.2948418855667114,
      "step": 1370
    },
    {
      "epoch": 2.4538786396977104,
      "grad_norm": 2.2009122371673584,
      "learning_rate": 3.9553917067232966e-07,
      "logits/chosen": 255.0864715576172,
      "logits/rejected": 255.9404754638672,
      "logps/chosen": -1.2902610301971436,
      "logps/rejected": -1.4941017627716064,
      "loss": 1.3585,
      "odds_ratio_loss": 0.6828353404998779,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.12902610003948212,
      "rewards/margins": 0.02038406766951084,
      "rewards/rejected": -0.1494101732969284,
      "sft_loss": 1.2902610301971436,
      "step": 1380
    },
    {
      "epoch": 2.4716603689708823,
      "grad_norm": 2.2108042240142822,
      "learning_rate": 3.707580696657509e-07,
      "logits/chosen": 253.1954345703125,
      "logits/rejected": 253.47250366210938,
      "logps/chosen": -1.2349462509155273,
      "logps/rejected": -1.409597396850586,
      "loss": 1.3025,
      "odds_ratio_loss": 0.6752232313156128,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.12349464744329453,
      "rewards/margins": 0.017465103417634964,
      "rewards/rejected": -0.1409597247838974,
      "sft_loss": 1.2349462509155273,
      "step": 1390
    },
    {
      "epoch": 2.489442098244054,
      "grad_norm": 2.4644668102264404,
      "learning_rate": 3.4671631608781815e-07,
      "logits/chosen": 254.1881561279297,
      "logits/rejected": 254.3625030517578,
      "logps/chosen": -1.3036715984344482,
      "logps/rejected": -1.4527599811553955,
      "loss": 1.3758,
      "odds_ratio_loss": 0.721402108669281,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.13036717474460602,
      "rewards/margins": 0.014908820390701294,
      "rewards/rejected": -0.14527598023414612,
      "sft_loss": 1.3036715984344482,
      "step": 1400
    },
    {
      "epoch": 2.507223827517226,
      "grad_norm": 3.9182987213134766,
      "learning_rate": 3.234222580780405e-07,
      "logits/chosen": 254.8416290283203,
      "logits/rejected": 254.9809112548828,
      "logps/chosen": -1.2807973623275757,
      "logps/rejected": -1.4203259944915771,
      "loss": 1.3485,
      "odds_ratio_loss": 0.6770311594009399,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.12807972729206085,
      "rewards/margins": 0.013952854089438915,
      "rewards/rejected": -0.14203259348869324,
      "sft_loss": 1.2807973623275757,
      "step": 1410
    },
    {
      "epoch": 2.525005556790398,
      "grad_norm": 8.136197090148926,
      "learning_rate": 3.0088398414982375e-07,
      "logits/chosen": 255.15200805664062,
      "logits/rejected": 255.2108612060547,
      "logps/chosen": -1.269274115562439,
      "logps/rejected": -1.5304492712020874,
      "loss": 1.3364,
      "odds_ratio_loss": 0.6713584661483765,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.12692740559577942,
      "rewards/margins": 0.026117529720067978,
      "rewards/rejected": -0.1530449539422989,
      "sft_loss": 1.269274115562439,
      "step": 1420
    },
    {
      "epoch": 2.54278728606357,
      "grad_norm": 3.0494956970214844,
      "learning_rate": 2.7910932038184487e-07,
      "logits/chosen": 252.93734741210938,
      "logits/rejected": 254.24777221679688,
      "logps/chosen": -1.24881112575531,
      "logps/rejected": -1.4509551525115967,
      "loss": 1.3164,
      "odds_ratio_loss": 0.675870418548584,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.12488112598657608,
      "rewards/margins": 0.020214393734931946,
      "rewards/rejected": -0.14509549736976624,
      "sft_loss": 1.24881112575531,
      "step": 1430
    },
    {
      "epoch": 2.5605690153367417,
      "grad_norm": 3.582563638687134,
      "learning_rate": 2.5810582770057325e-07,
      "logits/chosen": 255.56686401367188,
      "logits/rejected": 255.5573272705078,
      "logps/chosen": -1.2086716890335083,
      "logps/rejected": -1.3686209917068481,
      "loss": 1.2763,
      "odds_ratio_loss": 0.6763256788253784,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.12086717784404755,
      "rewards/margins": 0.01599491387605667,
      "rewards/rejected": -0.13686209917068481,
      "sft_loss": 1.2086716890335083,
      "step": 1440
    },
    {
      "epoch": 2.578350744609913,
      "grad_norm": 2.657351016998291,
      "learning_rate": 2.3788079925484402e-07,
      "logits/chosen": 254.91213989257812,
      "logits/rejected": 254.9799346923828,
      "logps/chosen": -1.3017337322235107,
      "logps/rejected": -1.4446247816085815,
      "loss": 1.3706,
      "odds_ratio_loss": 0.6881921887397766,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.13017335534095764,
      "rewards/margins": 0.01428910344839096,
      "rewards/rejected": -0.1444624662399292,
      "sft_loss": 1.3017337322235107,
      "step": 1450
    },
    {
      "epoch": 2.596132473883085,
      "grad_norm": 3.423999309539795,
      "learning_rate": 2.1844125788342661e-07,
      "logits/chosen": 253.8058624267578,
      "logits/rejected": 254.35348510742188,
      "logps/chosen": -1.220453143119812,
      "logps/rejected": -1.6302309036254883,
      "loss": 1.284,
      "odds_ratio_loss": 0.6357576251029968,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.12204531580209732,
      "rewards/margins": 0.04097776859998703,
      "rewards/rejected": -0.16302308440208435,
      "sft_loss": 1.220453143119812,
      "step": 1460
    },
    {
      "epoch": 2.613914203156257,
      "grad_norm": 1.957571029663086,
      "learning_rate": 1.9979395367644428e-07,
      "logits/chosen": 254.90676879882812,
      "logits/rejected": 254.95101928710938,
      "logps/chosen": -1.2191929817199707,
      "logps/rejected": -1.4415347576141357,
      "loss": 1.2838,
      "odds_ratio_loss": 0.645682156085968,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.12191929668188095,
      "rewards/margins": 0.022234182804822922,
      "rewards/rejected": -0.14415349066257477,
      "sft_loss": 1.2191929817199707,
      "step": 1470
    },
    {
      "epoch": 2.631695932429429,
      "grad_norm": 4.140540599822998,
      "learning_rate": 1.81945361631512e-07,
      "logits/chosen": 254.05709838867188,
      "logits/rejected": 253.84130859375,
      "logps/chosen": -1.2751600742340088,
      "logps/rejected": -1.4497580528259277,
      "loss": 1.3463,
      "odds_ratio_loss": 0.711692214012146,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.1275160014629364,
      "rewards/margins": 0.017459798604249954,
      "rewards/rejected": -0.14497579634189606,
      "sft_loss": 1.2751600742340088,
      "step": 1480
    },
    {
      "epoch": 2.6494776617026004,
      "grad_norm": 3.2640628814697266,
      "learning_rate": 1.6490167940538343e-07,
      "logits/chosen": 254.5946807861328,
      "logits/rejected": 254.5460662841797,
      "logps/chosen": -1.2259750366210938,
      "logps/rejected": -1.4610532522201538,
      "loss": 1.289,
      "odds_ratio_loss": 0.6301438212394714,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.12259751558303833,
      "rewards/margins": 0.023507816717028618,
      "rewards/rejected": -0.1461053192615509,
      "sft_loss": 1.2259750366210938,
      "step": 1490
    },
    {
      "epoch": 2.6672593909757722,
      "grad_norm": 2.282480001449585,
      "learning_rate": 1.4866882516191339e-07,
      "logits/chosen": 253.92684936523438,
      "logits/rejected": 254.4107208251953,
      "logps/chosen": -1.259511113166809,
      "logps/rejected": -1.3850682973861694,
      "loss": 1.3302,
      "odds_ratio_loss": 0.7073375582695007,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.1259511113166809,
      "rewards/margins": 0.012555733323097229,
      "rewards/rejected": -0.13850684463977814,
      "sft_loss": 1.259511113166809,
      "step": 1500
    },
    {
      "epoch": 2.6672593909757722,
      "eval_logits/chosen": 254.46791076660156,
      "eval_logits/rejected": 254.66136169433594,
      "eval_logps/chosen": -1.2813518047332764,
      "eval_logps/rejected": -1.5003557205200195,
      "eval_loss": 1.347064733505249,
      "eval_odds_ratio_loss": 0.6571269631385803,
      "eval_rewards/accuracies": 0.5609999895095825,
      "eval_rewards/chosen": -0.12813518941402435,
      "eval_rewards/margins": 0.021900387480854988,
      "eval_rewards/rejected": -0.15003558993339539,
      "eval_runtime": 217.1974,
      "eval_samples_per_second": 4.604,
      "eval_sft_loss": 1.2813518047332764,
      "eval_steps_per_second": 2.302,
      "step": 1500
    },
    {
      "epoch": 2.685041120248944,
      "grad_norm": 6.244467258453369,
      "learning_rate": 1.3325243551706057e-07,
      "logits/chosen": 254.00894165039062,
      "logits/rejected": 254.4985809326172,
      "logps/chosen": -1.272092580795288,
      "logps/rejected": -1.6640291213989258,
      "loss": 1.3335,
      "odds_ratio_loss": 0.6145035624504089,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.12720926105976105,
      "rewards/margins": 0.03919365257024765,
      "rewards/rejected": -0.1664029359817505,
      "sft_loss": 1.272092580795288,
      "step": 1510
    },
    {
      "epoch": 2.702822849522116,
      "grad_norm": 6.613005638122559,
      "learning_rate": 1.1865786358165737e-07,
      "logits/chosen": 254.2620086669922,
      "logits/rejected": 254.74526977539062,
      "logps/chosen": -1.2863306999206543,
      "logps/rejected": -1.474635124206543,
      "loss": 1.3537,
      "odds_ratio_loss": 0.6741579174995422,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.12863309681415558,
      "rewards/margins": 0.0188304390758276,
      "rewards/rejected": -0.14746353030204773,
      "sft_loss": 1.2863306999206543,
      "step": 1520
    },
    {
      "epoch": 2.720604578795288,
      "grad_norm": 6.351752758026123,
      "learning_rate": 1.0489017710262311e-07,
      "logits/chosen": 255.4362030029297,
      "logits/rejected": 256.20306396484375,
      "logps/chosen": -1.331672191619873,
      "logps/rejected": -1.618596076965332,
      "loss": 1.4013,
      "odds_ratio_loss": 0.6964471340179443,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.13316722214221954,
      "rewards/margins": 0.028692390769720078,
      "rewards/rejected": -0.16185960173606873,
      "sft_loss": 1.331672191619873,
      "step": 1530
    },
    {
      "epoch": 2.73838630806846,
      "grad_norm": 2.5552542209625244,
      "learning_rate": 9.195415670326446e-08,
      "logits/chosen": 256.7084655761719,
      "logits/rejected": 256.827392578125,
      "logps/chosen": -1.285346269607544,
      "logps/rejected": -1.4887077808380127,
      "loss": 1.3524,
      "odds_ratio_loss": 0.6702226400375366,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.12853462994098663,
      "rewards/margins": 0.020336147397756577,
      "rewards/rejected": -0.1488707810640335,
      "sft_loss": 1.285346269607544,
      "step": 1540
    },
    {
      "epoch": 2.7561680373416317,
      "grad_norm": 4.229678630828857,
      "learning_rate": 7.985429422327384e-08,
      "logits/chosen": 254.65280151367188,
      "logits/rejected": 255.42001342773438,
      "logps/chosen": -1.2554762363433838,
      "logps/rejected": -1.3682177066802979,
      "loss": 1.3267,
      "odds_ratio_loss": 0.7126489877700806,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.1255476176738739,
      "rewards/margins": 0.011274145916104317,
      "rewards/rejected": -0.13682177662849426,
      "sft_loss": 1.2554762363433838,
      "step": 1550
    },
    {
      "epoch": 2.773949766614803,
      "grad_norm": 3.981313705444336,
      "learning_rate": 6.859479115900818e-08,
      "logits/chosen": 254.1326446533203,
      "logits/rejected": 254.6916961669922,
      "logps/chosen": -1.359695553779602,
      "logps/rejected": -1.6282466650009155,
      "loss": 1.425,
      "odds_ratio_loss": 0.6527189016342163,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.13596956431865692,
      "rewards/margins": 0.026855114847421646,
      "rewards/rejected": -0.16282466053962708,
      "sft_loss": 1.359695553779602,
      "step": 1560
    },
    {
      "epoch": 2.791731495887975,
      "grad_norm": 3.927825450897217,
      "learning_rate": 5.817955720457902e-08,
      "logits/chosen": 253.6094970703125,
      "logits/rejected": 253.5572967529297,
      "logps/chosen": -1.1961861848831177,
      "logps/rejected": -1.3642635345458984,
      "loss": 1.262,
      "odds_ratio_loss": 0.6578332185745239,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.11961860954761505,
      "rewards/margins": 0.016807744279503822,
      "rewards/rejected": -0.13642635941505432,
      "sft_loss": 1.1961861848831177,
      "step": 1570
    },
    {
      "epoch": 2.809513225161147,
      "grad_norm": 3.0377912521362305,
      "learning_rate": 4.861220889427199e-08,
      "logits/chosen": 254.80587768554688,
      "logits/rejected": 254.9072265625,
      "logps/chosen": -1.2765783071517944,
      "logps/rejected": -1.4059991836547852,
      "loss": 1.3469,
      "odds_ratio_loss": 0.7033384442329407,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.12765783071517944,
      "rewards/margins": 0.012942099943757057,
      "rewards/rejected": -0.14059992134571075,
      "sft_loss": 1.2765783071517944,
      "step": 1580
    },
    {
      "epoch": 2.827294954434319,
      "grad_norm": 1.8500066995620728,
      "learning_rate": 3.9896068346758074e-08,
      "logits/chosen": 254.78158569335938,
      "logits/rejected": 254.95730590820312,
      "logps/chosen": -1.2893197536468506,
      "logps/rejected": -1.4526543617248535,
      "loss": 1.3588,
      "odds_ratio_loss": 0.6947753429412842,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.12893196940422058,
      "rewards/margins": 0.01633346639573574,
      "rewards/rejected": -0.14526543021202087,
      "sft_loss": 1.2893197536468506,
      "step": 1590
    },
    {
      "epoch": 2.8450766837074903,
      "grad_norm": 5.4357829093933105,
      "learning_rate": 3.203416211153832e-08,
      "logits/chosen": 254.9698944091797,
      "logits/rejected": 256.3062438964844,
      "logps/chosen": -1.2634626626968384,
      "logps/rejected": -1.4395668506622314,
      "loss": 1.3344,
      "odds_ratio_loss": 0.7093390822410583,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.12634627521038055,
      "rewards/margins": 0.017610404640436172,
      "rewards/rejected": -0.14395669102668762,
      "sft_loss": 1.2634626626968384,
      "step": 1600
    },
    {
      "epoch": 2.8628584129806622,
      "grad_norm": 3.7756481170654297,
      "learning_rate": 2.5029220118019393e-08,
      "logits/chosen": 253.7996826171875,
      "logits/rejected": 254.91201782226562,
      "logps/chosen": -1.3131462335586548,
      "logps/rejected": -1.434832215309143,
      "loss": 1.3818,
      "odds_ratio_loss": 0.6866922974586487,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.13131462037563324,
      "rewards/margins": 0.012168603017926216,
      "rewards/rejected": -0.1434832215309143,
      "sft_loss": 1.3131462335586548,
      "step": 1610
    },
    {
      "epoch": 2.880640142253834,
      "grad_norm": 3.6297013759613037,
      "learning_rate": 1.8883674727586122e-08,
      "logits/chosen": 254.3390350341797,
      "logits/rejected": 254.14718627929688,
      "logps/chosen": -1.1930474042892456,
      "logps/rejected": -1.5245510339736938,
      "loss": 1.2541,
      "odds_ratio_loss": 0.6109124422073364,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.11930473893880844,
      "rewards/margins": 0.03315035253763199,
      "rewards/rejected": -0.15245509147644043,
      "sft_loss": 1.1930474042892456,
      "step": 1620
    },
    {
      "epoch": 2.898421871527006,
      "grad_norm": 1.5044124126434326,
      "learning_rate": 1.3599659889000639e-08,
      "logits/chosen": 255.17379760742188,
      "logits/rejected": 254.8286895751953,
      "logps/chosen": -1.3174787759780884,
      "logps/rejected": -1.4666922092437744,
      "loss": 1.3872,
      "odds_ratio_loss": 0.6974038481712341,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.13174788653850555,
      "rewards/margins": 0.01492134016007185,
      "rewards/rejected": -0.14666922390460968,
      "sft_loss": 1.3174787759780884,
      "step": 1630
    },
    {
      "epoch": 2.916203600800178,
      "grad_norm": 2.1138768196105957,
      "learning_rate": 9.179010397421528e-09,
      "logits/chosen": 256.32757568359375,
      "logits/rejected": 255.87295532226562,
      "logps/chosen": -1.26983642578125,
      "logps/rejected": -1.4480781555175781,
      "loss": 1.3386,
      "odds_ratio_loss": 0.6873185038566589,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.1269836574792862,
      "rewards/margins": 0.017824167385697365,
      "rewards/rejected": -0.144807830452919,
      "sft_loss": 1.26983642578125,
      "step": 1640
    },
    {
      "epoch": 2.93398533007335,
      "grad_norm": 1.8579224348068237,
      "learning_rate": 5.623261257296509e-09,
      "logits/chosen": 253.3066864013672,
      "logits/rejected": 253.75753784179688,
      "logps/chosen": -1.1514933109283447,
      "logps/rejected": -1.3553564548492432,
      "loss": 1.2157,
      "odds_ratio_loss": 0.6422222256660461,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.11514933407306671,
      "rewards/margins": 0.020386312156915665,
      "rewards/rejected": -0.13553564250469208,
      "sft_loss": 1.1514933109283447,
      "step": 1650
    },
    {
      "epoch": 2.9517670593465217,
      "grad_norm": 2.8349151611328125,
      "learning_rate": 2.933647149357122e-09,
      "logits/chosen": 254.45968627929688,
      "logits/rejected": 254.57400512695312,
      "logps/chosen": -1.300631046295166,
      "logps/rejected": -1.4870007038116455,
      "loss": 1.3705,
      "odds_ratio_loss": 0.6987138390541077,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.13006310164928436,
      "rewards/margins": 0.018636973574757576,
      "rewards/rejected": -0.14870008826255798,
      "sft_loss": 1.300631046295166,
      "step": 1660
    },
    {
      "epoch": 2.969548788619693,
      "grad_norm": 2.5957581996917725,
      "learning_rate": 1.1111020018930717e-09,
      "logits/chosen": 255.29000854492188,
      "logits/rejected": 255.3350830078125,
      "logps/chosen": -1.2563540935516357,
      "logps/rejected": -1.3871452808380127,
      "loss": 1.3264,
      "odds_ratio_loss": 0.7003098726272583,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.12563541531562805,
      "rewards/margins": 0.013079119846224785,
      "rewards/rejected": -0.1387145221233368,
      "sft_loss": 1.2563540935516357,
      "step": 1670
    },
    {
      "epoch": 2.987330517892865,
      "grad_norm": 1.7800966501235962,
      "learning_rate": 1.5625866646051813e-10,
      "logits/chosen": 255.032470703125,
      "logits/rejected": 255.3948974609375,
      "logps/chosen": -1.2115042209625244,
      "logps/rejected": -1.4695723056793213,
      "loss": 1.2721,
      "odds_ratio_loss": 0.6058895587921143,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.1211504191160202,
      "rewards/margins": 0.025806818157434464,
      "rewards/rejected": -0.14695724844932556,
      "sft_loss": 1.2115042209625244,
      "step": 1680
    },
    {
      "epoch": 2.997999555456768,
      "step": 1686,
      "total_flos": 2.0747345909474918e+18,
      "train_loss": 1.4404786094086308,
      "train_runtime": 22862.0738,
      "train_samples_per_second": 1.181,
      "train_steps_per_second": 0.074
    }
  ],
  "logging_steps": 10,
  "max_steps": 1686,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "total_flos": 2.0747345909474918e+18,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}