{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.995276334435522,
  "eval_steps": 50,
  "global_step": 704,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "dpo_loss": 0.6931471824645996,
      "epoch": 0.002834199338686821,
      "grad_norm": 18.397043918777676,
      "learning_rate": 7.042253521126761e-08,
      "logits": -1.2867579460144043,
      "logps": -84.34933471679688,
      "loss": 0.6931,
      "objective": 0.6931471824645996,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.6931471824645996,
      "step": 1,
      "wo_beta": 5.2708282470703125
    },
    {
      "dpo_loss": 0.6924155354499817,
      "epoch": 0.014170996693434105,
      "grad_norm": 18.351841049696436,
      "learning_rate": 3.521126760563381e-07,
      "logits": -1.430939793586731,
      "logps": -83.6253890991211,
      "loss": 0.6927,
      "objective": 0.6924155354499817,
      "ranking_simple": 0.4895833432674408,
      "regularize": 0.6924155354499817,
      "step": 5,
      "wo_beta": 7.649607181549072
    },
    {
      "dpo_loss": 0.6938675045967102,
      "epoch": 0.02834199338686821,
      "grad_norm": 19.803358997748763,
      "learning_rate": 7.042253521126762e-07,
      "logits": -1.4047328233718872,
      "logps": -83.41845703125,
      "loss": 0.6929,
      "objective": 0.6938675045967102,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.6938675045967102,
      "step": 10,
      "wo_beta": 5.940184593200684
    },
    {
      "dpo_loss": 0.6885419487953186,
      "epoch": 0.042512990080302314,
      "grad_norm": 19.861932182637975,
      "learning_rate": 1.0563380281690142e-06,
      "logits": -1.5388954877853394,
      "logps": -82.4270248413086,
      "loss": 0.689,
      "objective": 0.6885419487953186,
      "ranking_simple": 0.5708333253860474,
      "regularize": 0.6885419487953186,
      "step": 15,
      "wo_beta": 6.731040000915527
    },
    {
      "dpo_loss": 0.6827310919761658,
      "epoch": 0.05668398677373642,
      "grad_norm": 16.87924772685924,
      "learning_rate": 1.4084507042253523e-06,
      "logits": -1.3925108909606934,
      "logps": -80.71045684814453,
      "loss": 0.684,
      "objective": 0.6827310919761658,
      "ranking_simple": 0.4833333194255829,
      "regularize": 0.6827310919761658,
      "step": 20,
      "wo_beta": 8.312360763549805
    },
    {
      "dpo_loss": 0.6760162711143494,
      "epoch": 0.07085498346717052,
      "grad_norm": 17.86836459306927,
      "learning_rate": 1.7605633802816902e-06,
      "logits": -1.4580414295196533,
      "logps": -80.7186508178711,
      "loss": 0.6703,
      "objective": 0.6760162711143494,
      "ranking_simple": 0.5833333134651184,
      "regularize": 0.6760162711143494,
      "step": 25,
      "wo_beta": 6.106756210327148
    },
    {
      "dpo_loss": 0.6718389391899109,
      "epoch": 0.08502598016060463,
      "grad_norm": 19.355249838601942,
      "learning_rate": 2.1126760563380285e-06,
      "logits": -1.6006718873977661,
      "logps": -84.822021484375,
      "loss": 0.6727,
      "objective": 0.6718389391899109,
      "ranking_simple": 0.4958333373069763,
      "regularize": 0.6718389391899109,
      "step": 30,
      "wo_beta": 8.509283065795898
    },
    {
      "dpo_loss": 0.6751564145088196,
      "epoch": 0.09919697685403873,
      "grad_norm": 19.23258032194357,
      "learning_rate": 2.4647887323943666e-06,
      "logits": -1.6175826787948608,
      "logps": -85.66646575927734,
      "loss": 0.6624,
      "objective": 0.6751564145088196,
      "ranking_simple": 0.5416666865348816,
      "regularize": 0.6751564145088196,
      "step": 35,
      "wo_beta": 6.50329065322876
    },
    {
      "dpo_loss": 0.6747376322746277,
      "epoch": 0.11336797354747284,
      "grad_norm": 20.018488756494,
      "learning_rate": 2.8169014084507046e-06,
      "logits": -1.621884822845459,
      "logps": -84.25798797607422,
      "loss": 0.6598,
      "objective": 0.6747376322746277,
      "ranking_simple": 0.4791666567325592,
      "regularize": 0.6747376322746277,
      "step": 40,
      "wo_beta": 6.886596202850342
    },
    {
      "dpo_loss": 0.6456737518310547,
      "epoch": 0.12753897024090693,
      "grad_norm": 14.933918801623976,
      "learning_rate": 3.1690140845070427e-06,
      "logits": -1.6078789234161377,
      "logps": -81.69025421142578,
      "loss": 0.6563,
      "objective": 0.6456737518310547,
      "ranking_simple": 0.5166666507720947,
      "regularize": 0.6456737518310547,
      "step": 45,
      "wo_beta": 6.529275417327881
    },
    {
      "dpo_loss": 0.6442943811416626,
      "epoch": 0.14170996693434104,
      "grad_norm": 16.57958550585861,
      "learning_rate": 3.5211267605633804e-06,
      "logits": -1.6018227338790894,
      "logps": -82.79893493652344,
      "loss": 0.6316,
      "objective": 0.6442943811416626,
      "ranking_simple": 0.5375000238418579,
      "regularize": 0.6442943811416626,
      "step": 50,
      "wo_beta": 7.091952323913574
    },
    {
      "epoch": 0.14170996693434104,
      "eval_dpo_loss": 0.6824604868888855,
      "eval_logits": -1.587925672531128,
      "eval_logps": -90.32820892333984,
      "eval_loss": 0.6806595921516418,
      "eval_objective": 0.6824604868888855,
      "eval_ranking_simple": 0.5341615080833435,
      "eval_regularize": 0.6824604868888855,
      "eval_runtime": 367.8191,
      "eval_samples_per_second": 15.741,
      "eval_steps_per_second": 1.313,
      "eval_wo_beta": 7.8618669509887695,
      "step": 50
    },
    {
      "dpo_loss": 0.6267839074134827,
      "epoch": 0.15588096362777515,
      "grad_norm": 17.05808296065035,
      "learning_rate": 3.873239436619718e-06,
      "logits": -1.5757447481155396,
      "logps": -84.72220611572266,
      "loss": 0.6258,
      "objective": 0.6267839074134827,
      "ranking_simple": 0.5208333134651184,
      "regularize": 0.6267839074134827,
      "step": 55,
      "wo_beta": 6.998147487640381
    },
    {
      "dpo_loss": 0.6304137110710144,
      "epoch": 0.17005196032120926,
      "grad_norm": 17.359597890876504,
      "learning_rate": 4.225352112676057e-06,
      "logits": -1.5341871976852417,
      "logps": -87.07388305664062,
      "loss": 0.6204,
      "objective": 0.6304137110710144,
      "ranking_simple": 0.5708333253860474,
      "regularize": 0.6304137110710144,
      "step": 60,
      "wo_beta": 5.637550354003906
    },
    {
      "dpo_loss": 0.5928328633308411,
      "epoch": 0.18422295701464336,
      "grad_norm": 18.81033297245387,
      "learning_rate": 4.577464788732395e-06,
      "logits": -1.681604266166687,
      "logps": -88.5802001953125,
      "loss": 0.6021,
      "objective": 0.5928328633308411,
      "ranking_simple": 0.6041666865348816,
      "regularize": 0.5928328633308411,
      "step": 65,
      "wo_beta": 5.803915023803711
    },
    {
      "dpo_loss": 0.5882770419120789,
      "epoch": 0.19839395370807747,
      "grad_norm": 18.134916504512542,
      "learning_rate": 4.929577464788733e-06,
      "logits": -1.7763893604278564,
      "logps": -92.76559448242188,
      "loss": 0.5898,
      "objective": 0.5882770419120789,
      "ranking_simple": 0.5874999761581421,
      "regularize": 0.5882770419120789,
      "step": 70,
      "wo_beta": 6.3991007804870605
    },
    {
      "dpo_loss": 0.5995556712150574,
      "epoch": 0.21256495040151158,
      "grad_norm": 21.061539129772793,
      "learning_rate": 4.999507384516835e-06,
      "logits": -1.727323055267334,
      "logps": -92.68614196777344,
      "loss": 0.5904,
      "objective": 0.5995556712150574,
      "ranking_simple": 0.5874999761581421,
      "regularize": 0.5995556712150574,
      "step": 75,
      "wo_beta": 6.095489978790283
    },
    {
      "dpo_loss": 0.5429711937904358,
      "epoch": 0.22673594709494568,
      "grad_norm": 17.500250856825723,
      "learning_rate": 4.997506466835171e-06,
      "logits": -1.7568250894546509,
      "logps": -89.59777069091797,
      "loss": 0.5696,
      "objective": 0.5429711937904358,
      "ranking_simple": 0.6625000238418579,
      "regularize": 0.5429711937904358,
      "step": 80,
      "wo_beta": 4.702788829803467
    },
    {
      "dpo_loss": 0.5587875247001648,
      "epoch": 0.2409069437883798,
      "grad_norm": 14.610779362874785,
      "learning_rate": 4.9939676896203576e-06,
      "logits": -1.7483054399490356,
      "logps": -89.5750961303711,
      "loss": 0.5788,
      "objective": 0.5587875247001648,
      "ranking_simple": 0.6291666626930237,
      "regularize": 0.5587875247001648,
      "step": 85,
      "wo_beta": 5.067751407623291
    },
    {
      "dpo_loss": 0.4884372353553772,
      "epoch": 0.25507794048181387,
      "grad_norm": 16.59773892580396,
      "learning_rate": 4.9888932319026994e-06,
      "logits": -1.829767107963562,
      "logps": -88.12342071533203,
      "loss": 0.5584,
      "objective": 0.4884372353553772,
      "ranking_simple": 0.6541666388511658,
      "regularize": 0.4884372353553772,
      "step": 90,
      "wo_beta": 4.8033833503723145
    },
    {
      "dpo_loss": 0.5380887985229492,
      "epoch": 0.269248937175248,
      "grad_norm": 16.942898511925186,
      "learning_rate": 4.982286218320023e-06,
      "logits": -1.8157219886779785,
      "logps": -89.4225082397461,
      "loss": 0.548,
      "objective": 0.5380887985229492,
      "ranking_simple": 0.6208333373069763,
      "regularize": 0.5380887985229492,
      "step": 95,
      "wo_beta": 6.339680194854736
    },
    {
      "dpo_loss": 0.5714857578277588,
      "epoch": 0.2834199338686821,
      "grad_norm": 16.553483764105664,
      "learning_rate": 4.974150717193654e-06,
      "logits": -1.7562583684921265,
      "logps": -88.6811294555664,
      "loss": 0.5922,
      "objective": 0.5714857578277588,
      "ranking_simple": 0.6083333492279053,
      "regularize": 0.5714857578277588,
      "step": 100,
      "wo_beta": 5.948981285095215
    },
    {
      "epoch": 0.2834199338686821,
      "eval_dpo_loss": 0.6818667054176331,
      "eval_logits": -1.7963634729385376,
      "eval_logps": -95.815185546875,
      "eval_loss": 0.6793084740638733,
      "eval_objective": 0.6818667054176331,
      "eval_ranking_simple": 0.5486542582511902,
      "eval_regularize": 0.6818667054176331,
      "eval_runtime": 368.0636,
      "eval_samples_per_second": 15.731,
      "eval_steps_per_second": 1.312,
      "eval_wo_beta": 7.707693576812744,
      "step": 100
    },
    {
      "dpo_loss": 0.5925873517990112,
      "epoch": 0.2975909305621162,
      "grad_norm": 16.87738475184783,
      "learning_rate": 4.964491738023321e-06,
      "logits": -1.8753679990768433,
      "logps": -90.80838775634766,
      "loss": 0.5707,
      "objective": 0.5925873517990112,
      "ranking_simple": 0.6291666626930237,
      "regularize": 0.5925873517990112,
      "step": 105,
      "wo_beta": 5.3273797035217285
    },
    {
      "dpo_loss": 0.5388583540916443,
      "epoch": 0.3117619272555503,
      "grad_norm": 15.466132286249575,
      "learning_rate": 4.953315228402512e-06,
      "logits": -1.7556500434875488,
      "logps": -88.66165924072266,
      "loss": 0.5374,
      "objective": 0.5388583540916443,
      "ranking_simple": 0.5916666388511658,
      "regularize": 0.5388583540916443,
      "step": 110,
      "wo_beta": 5.21142053604126
    },
    {
      "dpo_loss": 0.5987796187400818,
      "epoch": 0.32593292394898443,
      "grad_norm": 16.57925719769902,
      "learning_rate": 4.9406280703561944e-06,
      "logits": -1.6699596643447876,
      "logps": -87.46602630615234,
      "loss": 0.5713,
      "objective": 0.5987796187400818,
      "ranking_simple": 0.6208333373069763,
      "regularize": 0.5987796187400818,
      "step": 115,
      "wo_beta": 5.849599838256836
    },
    {
      "dpo_loss": 0.5472472906112671,
      "epoch": 0.3401039206424185,
      "grad_norm": 16.2014631256649,
      "learning_rate": 4.926438076103162e-06,
      "logits": -1.6490483283996582,
      "logps": -87.74899291992188,
      "loss": 0.5603,
      "objective": 0.5472472906112671,
      "ranking_simple": 0.6333333253860474,
      "regularize": 0.5472472906112671,
      "step": 120,
      "wo_beta": 6.010056495666504
    },
    {
      "dpo_loss": 0.5509154796600342,
      "epoch": 0.35427491733585265,
      "grad_norm": 14.950351510543216,
      "learning_rate": 4.910753983245589e-06,
      "logits": -1.7191225290298462,
      "logps": -88.99519348144531,
      "loss": 0.5397,
      "objective": 0.5509154796600342,
      "ranking_simple": 0.6083333492279053,
      "regularize": 0.5509154796600342,
      "step": 125,
      "wo_beta": 5.906139850616455
    },
    {
      "dpo_loss": 0.5123094916343689,
      "epoch": 0.3684459140292867,
      "grad_norm": 13.21141785871735,
      "learning_rate": 4.893585449388786e-06,
      "logits": -1.695233702659607,
      "logps": -85.64789581298828,
      "loss": 0.5398,
      "objective": 0.5123094916343689,
      "ranking_simple": 0.6583333611488342,
      "regularize": 0.5123094916343689,
      "step": 130,
      "wo_beta": 4.518141269683838
    },
    {
      "dpo_loss": 0.5679463744163513,
      "epoch": 0.3826169107227208,
      "grad_norm": 16.522848387628404,
      "learning_rate": 4.8749430461944536e-06,
      "logits": -1.6519335508346558,
      "logps": -88.2005615234375,
      "loss": 0.5526,
      "objective": 0.5679463744163513,
      "ranking_simple": 0.6333333253860474,
      "regularize": 0.5679463744163513,
      "step": 135,
      "wo_beta": 5.8905439376831055
    },
    {
      "dpo_loss": 0.5832223296165466,
      "epoch": 0.39678790741615494,
      "grad_norm": 18.436163706658995,
      "learning_rate": 4.854838252871097e-06,
      "logits": -1.5592352151870728,
      "logps": -90.67977142333984,
      "loss": 0.5534,
      "objective": 0.5832223296165466,
      "ranking_simple": 0.625,
      "regularize": 0.5832223296165466,
      "step": 140,
      "wo_beta": 5.7765278816223145
    },
    {
      "dpo_loss": 0.510637640953064,
      "epoch": 0.410958904109589,
      "grad_norm": 15.85144306565166,
      "learning_rate": 4.833283449105609e-06,
      "logits": -1.5676114559173584,
      "logps": -90.87916564941406,
      "loss": 0.527,
      "objective": 0.510637640953064,
      "ranking_simple": 0.699999988079071,
      "regularize": 0.510637640953064,
      "step": 145,
      "wo_beta": 4.243090629577637
    },
    {
      "dpo_loss": 0.5151103138923645,
      "epoch": 0.42512990080302315,
      "grad_norm": 14.430458288203226,
      "learning_rate": 4.810291907440382e-06,
      "logits": -1.5757466554641724,
      "logps": -91.76609802246094,
      "loss": 0.5002,
      "objective": 0.5151103138923645,
      "ranking_simple": 0.6291666626930237,
      "regularize": 0.5151103138923645,
      "step": 150,
      "wo_beta": 6.819777011871338
    },
    {
      "epoch": 0.42512990080302315,
      "eval_dpo_loss": 0.6748862862586975,
      "eval_logits": -1.5950791835784912,
      "eval_logps": -96.20237731933594,
      "eval_loss": 0.6815473437309265,
      "eval_objective": 0.6748862862586975,
      "eval_ranking_simple": 0.5496894121170044,
      "eval_regularize": 0.6748862862586975,
      "eval_runtime": 367.9317,
      "eval_samples_per_second": 15.737,
      "eval_steps_per_second": 1.313,
      "eval_wo_beta": 7.438036918640137,
      "step": 150
    },
    {
      "dpo_loss": 0.49763983488082886,
      "epoch": 0.43930089749645723,
      "grad_norm": 17.361740656063542,
      "learning_rate": 4.785877785100633e-06,
      "logits": -1.6784894466400146,
      "logps": -92.56346130371094,
      "loss": 0.5198,
      "objective": 0.49763983488082886,
      "ranking_simple": 0.6499999761581421,
      "regularize": 0.49763983488082886,
      "step": 155,
      "wo_beta": 5.170775890350342
    },
    {
      "dpo_loss": 0.5579937100410461,
      "epoch": 0.45347189418989137,
      "grad_norm": 15.652756053936857,
      "learning_rate": 4.7600561152769795e-06,
      "logits": -1.5714988708496094,
      "logps": -92.15592956542969,
      "loss": 0.5328,
      "objective": 0.5579937100410461,
      "ranking_simple": 0.6416666507720947,
      "regularize": 0.5579937100410461,
      "step": 160,
      "wo_beta": 5.608686447143555
    },
    {
      "dpo_loss": 0.5186927318572998,
      "epoch": 0.46764289088332545,
      "grad_norm": 14.630193025898393,
      "learning_rate": 4.732842797868631e-06,
      "logits": -1.6691575050354004,
      "logps": -91.15839385986328,
      "loss": 0.5189,
      "objective": 0.5186927318572998,
      "ranking_simple": 0.625,
      "regularize": 0.5186927318572998,
      "step": 165,
      "wo_beta": 5.489006996154785
    },
    {
      "dpo_loss": 0.4966394603252411,
      "epoch": 0.4818138875767596,
      "grad_norm": 13.19317387340531,
      "learning_rate": 4.704254589692903e-06,
      "logits": -1.7252763509750366,
      "logps": -92.86217498779297,
      "loss": 0.4959,
      "objective": 0.4966394603252411,
      "ranking_simple": 0.6416666507720947,
      "regularize": 0.4966394603252411,
      "step": 170,
      "wo_beta": 5.683476448059082
    },
    {
      "dpo_loss": 0.4829941689968109,
      "epoch": 0.49598488427019366,
      "grad_norm": 16.695755417688215,
      "learning_rate": 4.6743090941670675e-06,
      "logits": -1.609352946281433,
      "logps": -95.68805694580078,
      "loss": 0.4892,
      "objective": 0.4829941689968109,
      "ranking_simple": 0.675000011920929,
      "regularize": 0.4829941689968109,
      "step": 175,
      "wo_beta": 4.217517852783203
    },
    {
      "dpo_loss": 0.5080674886703491,
      "epoch": 0.5101558809636277,
      "grad_norm": 18.149531395287664,
      "learning_rate": 4.643024750468913e-06,
      "logits": -1.6799732446670532,
      "logps": -99.82926177978516,
      "loss": 0.468,
      "objective": 0.5080674886703491,
      "ranking_simple": 0.6333333253860474,
      "regularize": 0.5080674886703491,
      "step": 180,
      "wo_beta": 5.674283504486084
    },
    {
      "dpo_loss": 0.4599061608314514,
      "epoch": 0.5243268776570619,
      "grad_norm": 16.418473362321556,
      "learning_rate": 4.610420822182671e-06,
      "logits": -1.6607011556625366,
      "logps": -99.13754272460938,
      "loss": 0.4631,
      "objective": 0.4599061608314514,
      "ranking_simple": 0.699999988079071,
      "regularize": 0.4599061608314514,
      "step": 185,
      "wo_beta": 4.34508752822876
    },
    {
      "dpo_loss": 0.4760186970233917,
      "epoch": 0.538497874350496,
      "grad_norm": 16.5909593130195,
      "learning_rate": 4.576517385437315e-06,
      "logits": -1.7211116552352905,
      "logps": -96.94883728027344,
      "loss": 0.4859,
      "objective": 0.4760186970233917,
      "ranking_simple": 0.6958333253860474,
      "regularize": 0.4760186970233917,
      "step": 190,
      "wo_beta": 4.704507827758789
    },
    {
      "dpo_loss": 0.4567195773124695,
      "epoch": 0.5526688710439301,
      "grad_norm": 17.67231005570377,
      "learning_rate": 4.541335316544514e-06,
      "logits": -1.7492233514785767,
      "logps": -95.30302429199219,
      "loss": 0.5112,
      "objective": 0.4567195773124695,
      "ranking_simple": 0.7208333611488342,
      "regularize": 0.4567195773124695,
      "step": 195,
      "wo_beta": 4.662184238433838
    },
    {
      "dpo_loss": 0.4767034947872162,
      "epoch": 0.5668398677373642,
      "grad_norm": 14.561592123615519,
      "learning_rate": 4.5048962791438885e-06,
      "logits": -1.7373807430267334,
      "logps": -95.8047866821289,
      "loss": 0.4735,
      "objective": 0.4767034947872162,
      "ranking_simple": 0.6583333611488342,
      "regularize": 0.4767034947872162,
      "step": 200,
      "wo_beta": 5.47299861907959
    },
    {
      "epoch": 0.5668398677373642,
      "eval_dpo_loss": 0.691116213798523,
      "eval_logits": -1.7563871145248413,
      "eval_logps": -98.91757202148438,
      "eval_loss": 0.6950607299804688,
      "eval_objective": 0.691116213798523,
      "eval_ranking_simple": 0.5569358468055725,
      "eval_regularize": 0.691116213798523,
      "eval_runtime": 367.7852,
      "eval_samples_per_second": 15.743,
      "eval_steps_per_second": 1.313,
      "eval_wo_beta": 7.524092197418213,
      "step": 200
    },
    {
      "dpo_loss": 0.49286583065986633,
      "epoch": 0.5810108644307983,
      "grad_norm": 15.609742263801415,
      "learning_rate": 4.467222710863444e-06,
      "logits": -1.6046305894851685,
      "logps": -95.40666961669922,
      "loss": 0.4728,
      "objective": 0.49286583065986633,
      "ranking_simple": 0.6833333373069763,
      "regularize": 0.49286583065986633,
      "step": 205,
      "wo_beta": 5.8981099128723145
    },
    {
      "dpo_loss": 0.42898985743522644,
      "epoch": 0.5951818611242324,
      "grad_norm": 16.532108905835056,
      "learning_rate": 4.428337809503425e-06,
      "logits": -1.6189254522323608,
      "logps": -94.09720611572266,
      "loss": 0.4362,
      "objective": 0.42898985743522644,
      "ranking_simple": 0.6916666626930237,
      "regularize": 0.42898985743522644,
      "step": 210,
      "wo_beta": 3.8531105518341064
    },
    {
      "dpo_loss": 0.44433167576789856,
      "epoch": 0.6093528578176665,
      "grad_norm": 18.51464695126941,
      "learning_rate": 4.388265518752085e-06,
      "logits": -1.7230619192123413,
      "logps": -92.92915344238281,
      "loss": 0.4621,
      "objective": 0.44433167576789856,
      "ranking_simple": 0.6791666746139526,
      "regularize": 0.44433167576789856,
      "step": 215,
      "wo_beta": 5.033292770385742
    },
    {
      "dpo_loss": 0.44531288743019104,
      "epoch": 0.6235238545111006,
      "grad_norm": 14.674842226422456,
      "learning_rate": 4.347030513442168e-06,
      "logits": -1.7578327655792236,
      "logps": -91.2856674194336,
      "loss": 0.4332,
      "objective": 0.44531288743019104,
      "ranking_simple": 0.6916666626930237,
      "regularize": 0.44531288743019104,
      "step": 220,
      "wo_beta": 5.138680934906006
    },
    {
      "dpo_loss": 0.47229692339897156,
      "epoch": 0.6376948512045347,
      "grad_norm": 13.123177972096014,
      "learning_rate": 4.304658184357186e-06,
      "logits": -1.8197827339172363,
      "logps": -92.20543670654297,
      "loss": 0.4692,
      "objective": 0.47229692339897156,
      "ranking_simple": 0.6541666388511658,
      "regularize": 0.47229692339897156,
      "step": 225,
      "wo_beta": 5.442239284515381
    },
    {
      "dpo_loss": 0.4127563536167145,
      "epoch": 0.6518658478979689,
      "grad_norm": 13.818617424678937,
      "learning_rate": 4.261174622596835e-06,
      "logits": -1.6802526712417603,
      "logps": -90.7798843383789,
      "loss": 0.4559,
      "objective": 0.4127563536167145,
      "ranking_simple": 0.7333333492279053,
      "regularize": 0.4127563536167145,
      "step": 230,
      "wo_beta": 3.521521806716919
    },
    {
      "dpo_loss": 0.45779237151145935,
      "epoch": 0.6660368445914029,
      "grad_norm": 15.036299488311245,
      "learning_rate": 4.216606603511202e-06,
      "logits": -1.6339088678359985,
      "logps": -90.51241302490234,
      "loss": 0.4578,
      "objective": 0.45779237151145935,
      "ranking_simple": 0.6666666865348816,
      "regularize": 0.45779237151145935,
      "step": 235,
      "wo_beta": 4.571218967437744
    },
    {
      "dpo_loss": 0.4738180935382843,
      "epoch": 0.680207841284837,
      "grad_norm": 13.614428839916116,
      "learning_rate": 4.170981570213621e-06,
      "logits": -1.8103351593017578,
      "logps": -92.29689025878906,
      "loss": 0.4481,
      "objective": 0.4738180935382843,
      "ranking_simple": 0.6958333253860474,
      "regularize": 0.4738180935382843,
      "step": 240,
      "wo_beta": 6.207835674285889
    },
    {
      "dpo_loss": 0.5174158215522766,
      "epoch": 0.6943788379782712,
      "grad_norm": 16.01438129051185,
      "learning_rate": 4.124327616682362e-06,
      "logits": -1.7986476421356201,
      "logps": -91.10508728027344,
      "loss": 0.4642,
      "objective": 0.5174158215522766,
      "ranking_simple": 0.6958333253860474,
      "regularize": 0.5174158215522766,
      "step": 245,
      "wo_beta": 4.900957107543945
    },
    {
      "dpo_loss": 0.44342610239982605,
      "epoch": 0.7085498346717053,
      "grad_norm": 13.738305838918832,
      "learning_rate": 4.076673470461538e-06,
      "logits": -1.5667024850845337,
      "logps": -88.29222106933594,
      "loss": 0.4626,
      "objective": 0.44342610239982605,
      "ranking_simple": 0.7041666507720947,
      "regularize": 0.44342610239982605,
      "step": 250,
      "wo_beta": 4.291601181030273
    },
    {
      "epoch": 0.7085498346717053,
      "eval_dpo_loss": 0.694493293762207,
      "eval_logits": -1.7985897064208984,
      "eval_logps": -93.47747039794922,
      "eval_loss": 0.6976169347763062,
      "eval_objective": 0.694493293762207,
      "eval_ranking_simple": 0.5579710006713867,
      "eval_regularize": 0.694493293762207,
      "eval_runtime": 367.886,
      "eval_samples_per_second": 15.739,
      "eval_steps_per_second": 1.313,
      "eval_wo_beta": 7.902660369873047,
      "step": 250
    },
    {
      "dpo_loss": 0.43279239535331726,
      "epoch": 0.7227208313651393,
      "grad_norm": 12.486423165060444,
      "learning_rate": 4.028048474971889e-06,
      "logits": -1.860019564628601,
      "logps": -87.4556884765625,
      "loss": 0.4459,
      "objective": 0.43279239535331726,
      "ranking_simple": 0.6833333373069763,
      "regularize": 0.43279239535331726,
      "step": 255,
      "wo_beta": 4.541534900665283
    },
    {
      "dpo_loss": 0.41317591071128845,
      "epoch": 0.7368918280585735,
      "grad_norm": 13.93218309283749,
      "learning_rate": 3.978482571442339e-06,
      "logits": -1.8741662502288818,
      "logps": -91.13224029541016,
      "loss": 0.4561,
      "objective": 0.41317591071128845,
      "ranking_simple": 0.6625000238418579,
      "regularize": 0.41317591071128845,
      "step": 260,
      "wo_beta": 5.1168012619018555
    },
    {
      "dpo_loss": 0.39290040731430054,
      "epoch": 0.7510628247520076,
      "grad_norm": 13.220960314976885,
      "learning_rate": 3.928006280473445e-06,
      "logits": -1.8604073524475098,
      "logps": -92.02545928955078,
      "loss": 0.419,
      "objective": 0.39290040731430054,
      "ranking_simple": 0.7708333134651184,
      "regularize": 0.39290040731430054,
      "step": 265,
      "wo_beta": 4.408606052398682
    },
    {
      "dpo_loss": 0.47771012783050537,
      "epoch": 0.7652338214454416,
      "grad_norm": 14.906696639905949,
      "learning_rate": 3.876650683244093e-06,
      "logits": -1.997718095779419,
      "logps": -94.74840545654297,
      "loss": 0.4365,
      "objective": 0.47771012783050537,
      "ranking_simple": 0.6666666865348816,
      "regularize": 0.47771012783050537,
      "step": 270,
      "wo_beta": 4.766172409057617
    },
    {
      "dpo_loss": 0.4372769594192505,
      "epoch": 0.7794048181388757,
      "grad_norm": 14.602546448634548,
      "learning_rate": 3.8244474023730155e-06,
      "logits": -1.8585816621780396,
      "logps": -94.3137435913086,
      "loss": 0.4293,
      "objective": 0.4372769594192505,
      "ranking_simple": 0.6791666746139526,
      "regularize": 0.4372769594192505,
      "step": 275,
      "wo_beta": 5.203604698181152
    },
    {
      "dpo_loss": 0.3766806721687317,
      "epoch": 0.7935758148323099,
      "grad_norm": 15.405745134115882,
      "learning_rate": 3.771428582446908e-06,
      "logits": -1.9468127489089966,
      "logps": -95.09542846679688,
      "loss": 0.4039,
      "objective": 0.3766806721687317,
      "ranking_simple": 0.7208333611488342,
      "regularize": 0.3766806721687317,
      "step": 280,
      "wo_beta": 4.175257205963135
    },
    {
      "dpo_loss": 0.41625434160232544,
      "epoch": 0.807746811525744,
      "grad_norm": 15.243116510039572,
      "learning_rate": 3.7176268702271468e-06,
      "logits": -1.9459937810897827,
      "logps": -95.90043640136719,
      "loss": 0.407,
      "objective": 0.41625434160232544,
      "ranking_simple": 0.699999988079071,
      "regularize": 0.41625434160232544,
      "step": 285,
      "wo_beta": 5.091909408569336
    },
    {
      "dpo_loss": 0.3713260293006897,
      "epoch": 0.821917808219178,
      "grad_norm": 16.703078330526107,
      "learning_rate": 3.6630753945472854e-06,
      "logits": -1.963159203529358,
      "logps": -95.55049133300781,
      "loss": 0.3981,
      "objective": 0.3713260293006897,
      "ranking_simple": 0.7791666388511658,
      "regularize": 0.3713260293006897,
      "step": 290,
      "wo_beta": 3.84537935256958
    },
    {
      "dpo_loss": 0.4009644687175751,
      "epoch": 0.8360888049126122,
      "grad_norm": 13.199801209484807,
      "learning_rate": 3.6078077459137097e-06,
      "logits": -1.99600350856781,
      "logps": -98.70610809326172,
      "loss": 0.4243,
      "objective": 0.4009644687175751,
      "ranking_simple": 0.7166666388511658,
      "regularize": 0.4009644687175751,
      "step": 295,
      "wo_beta": 5.210625171661377
    },
    {
      "dpo_loss": 0.42688027024269104,
      "epoch": 0.8502598016060463,
      "grad_norm": 12.622555129903681,
      "learning_rate": 3.5518579558220144e-06,
      "logits": -1.9770207405090332,
      "logps": -95.8087387084961,
      "loss": 0.4214,
      "objective": 0.42688027024269104,
      "ranking_simple": 0.7458333373069763,
      "regularize": 0.42688027024269104,
      "step": 300,
      "wo_beta": 3.2817904949188232
    },
    {
      "epoch": 0.8502598016060463,
      "eval_dpo_loss": 0.6865138411521912,
      "eval_logits": -2.0138111114501953,
      "eval_logps": -104.43373107910156,
      "eval_loss": 0.693064272403717,
      "eval_objective": 0.6865138411521912,
      "eval_ranking_simple": 0.5615941882133484,
      "eval_regularize": 0.6865138411521912,
      "eval_runtime": 367.6512,
      "eval_samples_per_second": 15.749,
      "eval_steps_per_second": 1.314,
      "eval_wo_beta": 7.581406593322754,
      "step": 300
    },
    {
      "dpo_loss": 0.44818738102912903,
      "epoch": 0.8644307982994804,
      "grad_norm": 13.23517832354566,
      "learning_rate": 3.495260475801841e-06,
      "logits": -1.9494545459747314,
      "logps": -97.299560546875,
      "loss": 0.3903,
      "objective": 0.44818738102912903,
      "ranking_simple": 0.7041666507720947,
      "regularize": 0.44818738102912903,
      "step": 305,
      "wo_beta": 4.3523712158203125
    },
    {
      "dpo_loss": 0.34828221797943115,
      "epoch": 0.8786017949929145,
      "grad_norm": 14.41832502286332,
      "learning_rate": 3.4380501562030704e-06,
      "logits": -1.9428808689117432,
      "logps": -98.20413970947266,
      "loss": 0.3807,
      "objective": 0.34828221797943115,
      "ranking_simple": 0.7333333492279053,
      "regularize": 0.34828221797943115,
      "step": 310,
      "wo_beta": 3.6427719593048096
    },
    {
      "dpo_loss": 0.3642140030860901,
      "epoch": 0.8927727916863486,
      "grad_norm": 12.300830909289896,
      "learning_rate": 3.3802622247364446e-06,
      "logits": -1.9570696353912354,
      "logps": -98.85526275634766,
      "loss": 0.4042,
      "objective": 0.3642140030860901,
      "ranking_simple": 0.699999988079071,
      "regularize": 0.3642140030860901,
      "step": 315,
      "wo_beta": 4.256704330444336
    },
    {
      "dpo_loss": 0.399863064289093,
      "epoch": 0.9069437883797827,
      "grad_norm": 15.886821504052966,
      "learning_rate": 3.321932264781822e-06,
      "logits": -1.9135253429412842,
      "logps": -101.23651885986328,
      "loss": 0.4395,
      "objective": 0.399863064289093,
      "ranking_simple": 0.7708333134651184,
      "regularize": 0.399863064289093,
      "step": 320,
      "wo_beta": 2.9900147914886475
    },
    {
      "dpo_loss": 0.37714484333992004,
      "epoch": 0.9211147850732169,
      "grad_norm": 14.587168312549423,
      "learning_rate": 3.2630961934774265e-06,
      "logits": -1.9540404081344604,
      "logps": -100.98246002197266,
      "loss": 0.3745,
      "objective": 0.37714484333992004,
      "ranking_simple": 0.7250000238418579,
      "regularize": 0.37714484333992004,
      "step": 325,
      "wo_beta": 4.075650691986084
    },
    {
      "dpo_loss": 0.35718733072280884,
      "epoch": 0.9352857817666509,
      "grad_norm": 15.295489026198931,
      "learning_rate": 3.203790239603583e-06,
      "logits": -1.7621917724609375,
      "logps": -101.14175415039062,
      "loss": 0.4011,
      "objective": 0.35718733072280884,
      "ranking_simple": 0.737500011920929,
      "regularize": 0.35718733072280884,
      "step": 330,
      "wo_beta": 3.9375758171081543
    },
    {
      "dpo_loss": 0.3431912660598755,
      "epoch": 0.949456778460085,
      "grad_norm": 13.496480338032299,
      "learning_rate": 3.1440509212745584e-06,
      "logits": -1.7355188131332397,
      "logps": -100.78395080566406,
      "loss": 0.3733,
      "objective": 0.3431912660598755,
      "ranking_simple": 0.7416666746139526,
      "regularize": 0.3431912660598755,
      "step": 335,
      "wo_beta": 3.3042349815368652
    },
    {
      "dpo_loss": 0.43064969778060913,
      "epoch": 0.9636277751535192,
      "grad_norm": 13.67781828181165,
      "learning_rate": 3.0839150234522404e-06,
      "logits": -1.781424641609192,
      "logps": -99.20060729980469,
      "loss": 0.4067,
      "objective": 0.43064969778060913,
      "ranking_simple": 0.7083333134651184,
      "regularize": 0.43064969778060913,
      "step": 340,
      "wo_beta": 4.711233615875244
    },
    {
      "dpo_loss": 0.3512551188468933,
      "epoch": 0.9777987718469532,
      "grad_norm": 13.324791495929361,
      "learning_rate": 3.0234195752955032e-06,
      "logits": -1.8463162183761597,
      "logps": -95.25973510742188,
      "loss": 0.3729,
      "objective": 0.3512551188468933,
      "ranking_simple": 0.7541666626930237,
      "regularize": 0.3512551188468933,
      "step": 345,
      "wo_beta": 2.9623959064483643
    },
    {
      "dpo_loss": 0.32169008255004883,
      "epoch": 0.9919697685403873,
      "grad_norm": 15.127441089438493,
      "learning_rate": 2.962601827359208e-06,
      "logits": -1.78915274143219,
      "logps": -97.9096450805664,
      "loss": 0.3652,
      "objective": 0.32169008255004883,
      "ranking_simple": 0.824999988079071,
      "regularize": 0.32169008255004883,
      "step": 350,
      "wo_beta": 2.600872039794922
    },
    {
      "epoch": 0.9919697685403873,
      "eval_dpo_loss": 0.6984499096870422,
      "eval_logits": -1.9094278812408447,
      "eval_logps": -102.83055114746094,
      "eval_loss": 0.7074127793312073,
      "eval_objective": 0.6984499096870422,
      "eval_ranking_simple": 0.5559006333351135,
      "eval_regularize": 0.6984499096870422,
      "eval_runtime": 367.6345,
      "eval_samples_per_second": 15.749,
      "eval_steps_per_second": 1.314,
      "eval_wo_beta": 7.8343892097473145,
      "step": 350
    },
    {
      "dpo_loss": 0.2995939552783966,
      "epoch": 1.0061407652338215,
      "grad_norm": 9.31634152713278,
      "learning_rate": 2.9014992286568773e-06,
      "logits": -2.007425546646118,
      "logps": -96.57454681396484,
      "loss": 0.3042,
      "objective": 0.2995939552783966,
      "ranking_simple": 0.8041666746139526,
      "regularize": 0.2995939552783966,
      "step": 355,
      "wo_beta": 2.7125356197357178
    },
    {
      "dpo_loss": 0.24162109196186066,
      "epoch": 1.0203117619272555,
      "grad_norm": 11.90366408091981,
      "learning_rate": 2.840149403601166e-06,
      "logits": -1.9152239561080933,
      "logps": -101.0685043334961,
      "loss": 0.2301,
      "objective": 0.24162109196186066,
      "ranking_simple": 0.8125,
      "regularize": 0.24162109196186066,
      "step": 360,
      "wo_beta": 2.7127444744110107
    },
    {
      "dpo_loss": 0.19792620837688446,
      "epoch": 1.0344827586206897,
      "grad_norm": 9.884538723237807,
      "learning_rate": 2.7785901288363253e-06,
      "logits": -1.9689671993255615,
      "logps": -107.58961486816406,
      "loss": 0.2082,
      "objective": 0.19792620837688446,
      "ranking_simple": 0.8291666507720947,
      "regularize": 0.19792620837688446,
      "step": 365,
      "wo_beta": 1.9694886207580566
    },
    {
      "dpo_loss": 0.23545877635478973,
      "epoch": 1.0486537553141237,
      "grad_norm": 16.911618406294945,
      "learning_rate": 2.7168593099769414e-06,
      "logits": -1.9311782121658325,
      "logps": -106.47748565673828,
      "loss": 0.2379,
      "objective": 0.23545877635478973,
      "ranking_simple": 0.8125,
      "regularize": 0.23545877635478973,
      "step": 370,
      "wo_beta": 3.021015167236328
    },
    {
      "dpo_loss": 0.22429493069648743,
      "epoch": 1.0628247520075578,
      "grad_norm": 15.795921555252965,
      "learning_rate": 2.654994958267241e-06,
      "logits": -1.9985809326171875,
      "logps": -108.60234832763672,
      "loss": 0.2364,
      "objective": 0.22429493069648743,
      "ranking_simple": 0.8374999761581421,
      "regularize": 0.22429493069648743,
      "step": 375,
      "wo_beta": 1.6982978582382202
    },
    {
      "dpo_loss": 0.20513677597045898,
      "epoch": 1.076995748700992,
      "grad_norm": 15.164372148185517,
      "learning_rate": 2.5930351671753707e-06,
      "logits": -2.0427591800689697,
      "logps": -109.7289047241211,
      "loss": 0.2153,
      "objective": 0.20513677597045898,
      "ranking_simple": 0.8583333492279053,
      "regularize": 0.20513677597045898,
      "step": 380,
      "wo_beta": 1.7354587316513062
    },
    {
      "dpo_loss": 0.24022004008293152,
      "epoch": 1.091166745394426,
      "grad_norm": 13.938458603767867,
      "learning_rate": 2.5310180889370374e-06,
      "logits": -2.007528781890869,
      "logps": -109.47885131835938,
      "loss": 0.2371,
      "objective": 0.24022004008293152,
      "ranking_simple": 0.8291666507720947,
      "regularize": 0.24022004008293152,
      "step": 385,
      "wo_beta": 1.9898384809494019
    },
    {
      "dpo_loss": 0.22819384932518005,
      "epoch": 1.10533774208786,
      "grad_norm": 10.75019956346705,
      "learning_rate": 2.468981911062964e-06,
      "logits": -1.9904738664627075,
      "logps": -110.48689270019531,
      "loss": 0.2092,
      "objective": 0.22819384932518005,
      "ranking_simple": 0.8291666507720947,
      "regularize": 0.22819384932518005,
      "step": 390,
      "wo_beta": 2.570746660232544
    },
    {
      "dpo_loss": 0.2051982581615448,
      "epoch": 1.1195087387812943,
      "grad_norm": 10.957538026383743,
      "learning_rate": 2.4069648328246305e-06,
      "logits": -2.0448696613311768,
      "logps": -108.89817810058594,
      "loss": 0.2315,
      "objective": 0.2051982581615448,
      "ranking_simple": 0.8666666746139526,
      "regularize": 0.2051982581615448,
      "step": 395,
      "wo_beta": 2.1264781951904297
    },
    {
      "dpo_loss": 0.20951204001903534,
      "epoch": 1.1336797354747283,
      "grad_norm": 10.831310704591798,
      "learning_rate": 2.3450050417327593e-06,
      "logits": -2.0998401641845703,
      "logps": -106.39730072021484,
      "loss": 0.2206,
      "objective": 0.20951204001903534,
      "ranking_simple": 0.8500000238418579,
      "regularize": 0.20951204001903534,
      "step": 400,
      "wo_beta": 1.4239428043365479
    },
    {
      "epoch": 1.1336797354747283,
      "eval_dpo_loss": 0.729559600353241,
      "eval_logits": -2.090850591659546,
      "eval_logps": -113.60484313964844,
      "eval_loss": 0.7347043752670288,
      "eval_objective": 0.729559600353241,
      "eval_ranking_simple": 0.5502070188522339,
      "eval_regularize": 0.729559600353241,
      "eval_runtime": 367.8652,
      "eval_samples_per_second": 15.739,
      "eval_steps_per_second": 1.313,
      "eval_wo_beta": 8.675128936767578,
      "step": 400
    },
    {
      "dpo_loss": 0.24946197867393494,
      "epoch": 1.1478507321681626,
      "grad_norm": 13.639203958436303,
      "learning_rate": 2.2831406900230586e-06,
      "logits": -2.0489606857299805,
      "logps": -109.95741271972656,
      "loss": 0.2302,
      "objective": 0.24946197867393494,
      "ranking_simple": 0.8208333253860474,
      "regularize": 0.24946197867393494,
      "step": 405,
      "wo_beta": 3.3028647899627686
    },
    {
      "dpo_loss": 0.22408606112003326,
      "epoch": 1.1620217288615966,
      "grad_norm": 9.982686874740121,
      "learning_rate": 2.221409871163675e-06,
      "logits": -2.026094913482666,
      "logps": -112.10523986816406,
      "loss": 0.204,
      "objective": 0.22408606112003326,
      "ranking_simple": 0.824999988079071,
      "regularize": 0.22408606112003326,
      "step": 410,
      "wo_beta": 3.6203742027282715
    },
    {
      "dpo_loss": 0.24647466838359833,
      "epoch": 1.1761927255550306,
      "grad_norm": 14.127702730262909,
      "learning_rate": 2.1598505963988354e-06,
      "logits": -1.9576979875564575,
      "logps": -112.045654296875,
      "loss": 0.2487,
      "objective": 0.24647466838359833,
      "ranking_simple": 0.8041666746139526,
      "regularize": 0.24647466838359833,
      "step": 415,
      "wo_beta": 3.0596923828125
    },
    {
      "dpo_loss": 0.2189057618379593,
      "epoch": 1.1903637222484649,
      "grad_norm": 12.453759090133827,
      "learning_rate": 2.098500771343124e-06,
      "logits": -1.9014623165130615,
      "logps": -112.163330078125,
      "loss": 0.224,
      "objective": 0.2189057618379593,
      "ranking_simple": 0.8333333134651184,
      "regularize": 0.2189057618379593,
      "step": 420,
      "wo_beta": 2.8976945877075195
    },
    {
      "dpo_loss": 0.24521614611148834,
      "epoch": 1.204534718941899,
      "grad_norm": 11.810902908919543,
      "learning_rate": 2.037398172640793e-06,
      "logits": -1.939537525177002,
      "logps": -111.03417205810547,
      "loss": 0.225,
      "objective": 0.24521614611148834,
      "ranking_simple": 0.8166666626930237,
      "regularize": 0.24521614611148834,
      "step": 425,
      "wo_beta": 2.2815327644348145
    },
    {
      "dpo_loss": 0.25871187448501587,
      "epoch": 1.2187057156353331,
      "grad_norm": 13.805740179315583,
      "learning_rate": 1.976580424704498e-06,
      "logits": -1.908257246017456,
      "logps": -114.54412078857422,
      "loss": 0.2315,
      "objective": 0.25871187448501587,
      "ranking_simple": 0.8208333253860474,
      "regularize": 0.25871187448501587,
      "step": 430,
      "wo_beta": 2.5115749835968018
    },
    {
      "dpo_loss": 0.20695915818214417,
      "epoch": 1.2328767123287672,
      "grad_norm": 13.189182575578958,
      "learning_rate": 1.9160849765477604e-06,
      "logits": -1.845086932182312,
      "logps": -111.52816772460938,
      "loss": 0.2502,
      "objective": 0.20695915818214417,
      "ranking_simple": 0.8666666746139526,
      "regularize": 0.20695915818214417,
      "step": 435,
      "wo_beta": 2.1831676959991455
    },
    {
      "dpo_loss": 0.2140309065580368,
      "epoch": 1.2470477090222012,
      "grad_norm": 15.231839264333766,
      "learning_rate": 1.8559490787254423e-06,
      "logits": -1.8013054132461548,
      "logps": -112.4487533569336,
      "loss": 0.2241,
      "objective": 0.2140309065580368,
      "ranking_simple": 0.8500000238418579,
      "regularize": 0.2140309065580368,
      "step": 440,
      "wo_beta": 2.4584310054779053
    },
    {
      "dpo_loss": 0.2538544535636902,
      "epoch": 1.2612187057156352,
      "grad_norm": 11.902330497163605,
      "learning_rate": 1.7962097603964177e-06,
      "logits": -1.8283072710037231,
      "logps": -111.75948333740234,
      "loss": 0.2393,
      "objective": 0.2538544535636902,
      "ranking_simple": 0.8291666507720947,
      "regularize": 0.2538544535636902,
      "step": 445,
      "wo_beta": 2.5012738704681396
    },
    {
      "dpo_loss": 0.21088647842407227,
      "epoch": 1.2753897024090695,
      "grad_norm": 12.114049593362662,
      "learning_rate": 1.7369038065225743e-06,
      "logits": -1.9961200952529907,
      "logps": -110.46500396728516,
      "loss": 0.2202,
      "objective": 0.21088647842407227,
      "ranking_simple": 0.8458333611488342,
      "regularize": 0.21088647842407227,
      "step": 450,
      "wo_beta": 2.1799509525299072
    },
    {
      "epoch": 1.2753897024090695,
      "eval_dpo_loss": 0.7433450222015381,
      "eval_logits": -1.9911303520202637,
      "eval_logps": -115.7781753540039,
      "eval_loss": 0.7462677955627441,
      "eval_objective": 0.7433450222015381,
      "eval_ranking_simple": 0.5512422323226929,
      "eval_regularize": 0.7433450222015381,
      "eval_runtime": 369.6423,
      "eval_samples_per_second": 15.664,
      "eval_steps_per_second": 1.307,
      "eval_wo_beta": 8.912315368652344,
      "step": 450
    },
    {
      "dpo_loss": 0.22404690086841583,
      "epoch": 1.2895606991025035,
      "grad_norm": 12.618712156759832,
      "learning_rate": 1.6780677352181781e-06,
      "logits": -1.821974515914917,
      "logps": -112.83589935302734,
      "loss": 0.2503,
      "objective": 0.22404690086841583,
      "ranking_simple": 0.8125,
      "regularize": 0.22404690086841583,
      "step": 455,
      "wo_beta": 2.789232015609741
    },
    {
      "dpo_loss": 0.2399408221244812,
      "epoch": 1.3037316957959377,
      "grad_norm": 10.297452895979795,
      "learning_rate": 1.6197377752635563e-06,
      "logits": -1.9601954221725464,
      "logps": -111.7652587890625,
      "loss": 0.2322,
      "objective": 0.2399408221244812,
      "ranking_simple": 0.8166666626930237,
      "regularize": 0.2399408221244812,
      "step": 460,
      "wo_beta": 3.1664645671844482
    },
    {
      "dpo_loss": 0.21428868174552917,
      "epoch": 1.3179026924893718,
      "grad_norm": 11.673896578608254,
      "learning_rate": 1.5619498437969302e-06,
      "logits": -1.9731502532958984,
      "logps": -109.419189453125,
      "loss": 0.2432,
      "objective": 0.21428868174552917,
      "ranking_simple": 0.824999988079071,
      "regularize": 0.21428868174552917,
      "step": 465,
      "wo_beta": 2.3578691482543945
    },
    {
      "dpo_loss": 0.2447548806667328,
      "epoch": 1.3320736891828058,
      "grad_norm": 11.318434823205884,
      "learning_rate": 1.5047395241981606e-06,
      "logits": -2.0559751987457275,
      "logps": -107.98088836669922,
      "loss": 0.2364,
      "objective": 0.2447548806667328,
      "ranking_simple": 0.8500000238418579,
      "regularize": 0.2447548806667328,
      "step": 470,
      "wo_beta": 2.524103879928589
    },
    {
      "dpo_loss": 0.2010059952735901,
      "epoch": 1.34624468587624,
      "grad_norm": 13.4415115238709,
      "learning_rate": 1.4481420441779862e-06,
      "logits": -1.910614013671875,
      "logps": -110.5365982055664,
      "loss": 0.2194,
      "objective": 0.2010059952735901,
      "ranking_simple": 0.8666666746139526,
      "regularize": 0.2010059952735901,
      "step": 475,
      "wo_beta": 2.4653513431549072
    },
    {
      "dpo_loss": 0.21145032346248627,
      "epoch": 1.360415682569674,
      "grad_norm": 15.045958431800248,
      "learning_rate": 1.3921922540862907e-06,
      "logits": -2.0142934322357178,
      "logps": -109.8845443725586,
      "loss": 0.2244,
      "objective": 0.21145032346248627,
      "ranking_simple": 0.8166666626930237,
      "regularize": 0.21145032346248627,
      "step": 480,
      "wo_beta": 3.300536632537842
    },
    {
      "dpo_loss": 0.23482932150363922,
      "epoch": 1.3745866792631083,
      "grad_norm": 14.203111358887098,
      "learning_rate": 1.3369246054527152e-06,
      "logits": -1.996147632598877,
      "logps": -108.94562530517578,
      "loss": 0.2473,
      "objective": 0.23482932150363922,
      "ranking_simple": 0.8291666507720947,
      "regularize": 0.23482932150363922,
      "step": 485,
      "wo_beta": 2.5876729488372803
    },
    {
      "dpo_loss": 0.23283059895038605,
      "epoch": 1.3887576759565423,
      "grad_norm": 11.652820113044354,
      "learning_rate": 1.2823731297728536e-06,
      "logits": -1.98202645778656,
      "logps": -112.48513793945312,
      "loss": 0.2222,
      "objective": 0.23283059895038605,
      "ranking_simple": 0.824999988079071,
      "regularize": 0.23283059895038605,
      "step": 490,
      "wo_beta": 2.047060251235962
    },
    {
      "dpo_loss": 0.19643358886241913,
      "epoch": 1.4029286726499763,
      "grad_norm": 16.098991414414286,
      "learning_rate": 1.2285714175530936e-06,
      "logits": -1.9894219636917114,
      "logps": -111.90442657470703,
      "loss": 0.2257,
      "objective": 0.19643358886241913,
      "ranking_simple": 0.8458333611488342,
      "regularize": 0.19643358886241913,
      "step": 495,
      "wo_beta": 2.041602611541748
    },
    {
      "dpo_loss": 0.22570651769638062,
      "epoch": 1.4170996693434104,
      "grad_norm": 14.942258982525255,
      "learning_rate": 1.1755525976269851e-06,
      "logits": -1.9338775873184204,
      "logps": -109.2611083984375,
      "loss": 0.2366,
      "objective": 0.22570651769638062,
      "ranking_simple": 0.8208333253860474,
      "regularize": 0.22570651769638062,
      "step": 500,
      "wo_beta": 1.8392161130905151
    },
    {
      "epoch": 1.4170996693434104,
      "eval_dpo_loss": 0.7387389540672302,
      "eval_logits": -2.046374797821045,
      "eval_logps": -114.77104187011719,
      "eval_loss": 0.7444195747375488,
      "eval_objective": 0.7387389540672302,
      "eval_ranking_simple": 0.5517598390579224,
      "eval_regularize": 0.7387389540672302,
      "eval_runtime": 367.6953,
      "eval_samples_per_second": 15.747,
      "eval_steps_per_second": 1.314,
      "eval_wo_beta": 8.863048553466797,
      "step": 500
    },
    {
      "dpo_loss": 0.18702387809753418,
      "epoch": 1.4312706660368446,
      "grad_norm": 11.645842357352148,
      "learning_rate": 1.1233493167559065e-06,
      "logits": -2.00331711769104,
      "logps": -110.28618621826172,
      "loss": 0.2087,
      "objective": 0.18702387809753418,
      "ranking_simple": 0.8458333611488342,
      "regularize": 0.18702387809753418,
      "step": 505,
      "wo_beta": 2.0265085697174072
    },
    {
      "dpo_loss": 0.23954346776008606,
      "epoch": 1.4454416627302786,
      "grad_norm": 11.232727785445284,
      "learning_rate": 1.0719937195265555e-06,
      "logits": -2.0127902030944824,
      "logps": -110.59080505371094,
      "loss": 0.2313,
      "objective": 0.23954346776008606,
      "ranking_simple": 0.8458333611488342,
      "regularize": 0.23954346776008606,
      "step": 510,
      "wo_beta": 2.1970200538635254
    },
    {
      "dpo_loss": 0.20086827874183655,
      "epoch": 1.4596126594237129,
      "grad_norm": 10.217720875279324,
      "learning_rate": 1.0215174285576615e-06,
      "logits": -2.087155818939209,
      "logps": -110.73674774169922,
      "loss": 0.2043,
      "objective": 0.20086827874183655,
      "ranking_simple": 0.8083333373069763,
      "regularize": 0.20086827874183655,
      "step": 515,
      "wo_beta": 2.5676777362823486
    },
    {
      "dpo_loss": 0.19097186625003815,
      "epoch": 1.473783656117147,
      "grad_norm": 11.650910390578003,
      "learning_rate": 9.719515250281122e-07,
      "logits": -1.9594320058822632,
      "logps": -110.79000091552734,
      "loss": 0.1898,
      "objective": 0.19097186625003815,
      "ranking_simple": 0.8833333253860474,
      "regularize": 0.19097186625003815,
      "step": 520,
      "wo_beta": 2.2379517555236816
    },
    {
      "dpo_loss": 0.2083693891763687,
      "epoch": 1.487954652810581,
      "grad_norm": 12.830417240600797,
      "learning_rate": 9.233265295384624e-07,
      "logits": -1.8601106405258179,
      "logps": -110.70569610595703,
      "loss": 0.2274,
      "objective": 0.2083693891763687,
      "ranking_simple": 0.8416666388511658,
      "regularize": 0.2083693891763687,
      "step": 525,
      "wo_beta": 2.14703369140625
    },
    {
      "dpo_loss": 0.19533474743366241,
      "epoch": 1.5021256495040152,
      "grad_norm": 14.108055837836243,
      "learning_rate": 8.756723833176376e-07,
      "logits": -2.0109665393829346,
      "logps": -114.77526092529297,
      "loss": 0.205,
      "objective": 0.19533474743366241,
      "ranking_simple": 0.8583333492279053,
      "regularize": 0.19533474743366241,
      "step": 530,
      "wo_beta": 1.9219799041748047
    },
    {
      "dpo_loss": 0.18096224963665009,
      "epoch": 1.5162966461974492,
      "grad_norm": 13.367132869446376,
      "learning_rate": 8.290184297863793e-07,
      "logits": -1.971710205078125,
      "logps": -113.77371978759766,
      "loss": 0.188,
      "objective": 0.18096224963665009,
      "ranking_simple": 0.8541666865348816,
      "regularize": 0.18096224963665009,
      "step": 535,
      "wo_beta": 2.045213222503662
    },
    {
      "dpo_loss": 0.26400619745254517,
      "epoch": 1.5304676428908834,
      "grad_norm": 15.135814096600672,
      "learning_rate": 7.833933964887985e-07,
      "logits": -1.9013224840164185,
      "logps": -113.3902816772461,
      "loss": 0.221,
      "objective": 0.26400619745254517,
      "ranking_simple": 0.8374999761581421,
      "regularize": 0.26400619745254517,
      "step": 540,
      "wo_beta": 2.3301329612731934
    },
    {
      "dpo_loss": 0.23929236829280853,
      "epoch": 1.5446386395843175,
      "grad_norm": 15.186538591312983,
      "learning_rate": 7.388253774031659e-07,
      "logits": -1.9914318323135376,
      "logps": -113.67050170898438,
      "loss": 0.2343,
      "objective": 0.23929236829280853,
      "ranking_simple": 0.8666666746139526,
      "regularize": 0.23929236829280853,
      "step": 545,
      "wo_beta": 1.8687002658843994
    },
    {
      "dpo_loss": 0.17025238275527954,
      "epoch": 1.5588096362777515,
      "grad_norm": 11.147125855718437,
      "learning_rate": 6.953418156428152e-07,
      "logits": -1.9760197401046753,
      "logps": -114.83491516113281,
      "loss": 0.1989,
      "objective": 0.17025238275527954,
      "ranking_simple": 0.8458333611488342,
      "regularize": 0.17025238275527954,
      "step": 550,
      "wo_beta": 2.0717434883117676
    },
    {
      "epoch": 1.5588096362777515,
      "eval_dpo_loss": 0.7519087195396423,
      "eval_logits": -2.0168354511260986,
      "eval_logps": -118.77753448486328,
      "eval_loss": 0.7552616596221924,
      "eval_objective": 0.7519087195396423,
      "eval_ranking_simple": 0.5595238208770752,
      "eval_regularize": 0.7519087195396423,
      "eval_runtime": 367.683,
      "eval_samples_per_second": 15.747,
      "eval_steps_per_second": 1.314,
      "eval_wo_beta": 8.984560012817383,
      "step": 550
    },
    {
      "dpo_loss": 0.2299811840057373,
      "epoch": 1.5729806329711855,
      "grad_norm": 14.207116899254263,
      "learning_rate": 6.529694865578318e-07,
      "logits": -1.7790377140045166,
      "logps": -116.34921264648438,
      "loss": 0.2215,
      "objective": 0.2299811840057373,
      "ranking_simple": 0.8166666626930237,
      "regularize": 0.2299811840057373,
      "step": 555,
      "wo_beta": 2.6884348392486572
    },
    {
      "dpo_loss": 0.2206832468509674,
      "epoch": 1.5871516296646198,
      "grad_norm": 14.824039668104726,
      "learning_rate": 6.117344812479154e-07,
      "logits": -1.8644143342971802,
      "logps": -111.79485321044922,
      "loss": 0.2146,
      "objective": 0.2206832468509674,
      "ranking_simple": 0.8416666388511658,
      "regularize": 0.2206832468509674,
      "step": 560,
      "wo_beta": 2.7559573650360107
    },
    {
      "dpo_loss": 0.18442773818969727,
      "epoch": 1.601322626358054,
      "grad_norm": 13.453462098796246,
      "learning_rate": 5.71662190496575e-07,
      "logits": -1.8391135931015015,
      "logps": -113.25553131103516,
      "loss": 0.2078,
      "objective": 0.18442773818969727,
      "ranking_simple": 0.8333333134651184,
      "regularize": 0.18442773818969727,
      "step": 565,
      "wo_beta": 2.2805912494659424
    },
    {
      "dpo_loss": 0.19617310166358948,
      "epoch": 1.615493623051488,
      "grad_norm": 12.497698816224,
      "learning_rate": 5.327772891365565e-07,
      "logits": -1.984673023223877,
      "logps": -115.45191192626953,
      "loss": 0.1869,
      "objective": 0.19617310166358948,
      "ranking_simple": 0.8374999761581421,
      "regularize": 0.19617310166358948,
      "step": 570,
      "wo_beta": 2.3473691940307617
    },
    {
      "dpo_loss": 0.2390568107366562,
      "epoch": 1.629664619744922,
      "grad_norm": 14.590840547972736,
      "learning_rate": 4.951037208561116e-07,
      "logits": -1.9447566270828247,
      "logps": -111.96437072753906,
      "loss": 0.2312,
      "objective": 0.2390568107366562,
      "ranking_simple": 0.8458333611488342,
      "regularize": 0.2390568107366562,
      "step": 575,
      "wo_beta": 2.296287775039673
    },
    {
      "dpo_loss": 0.20053791999816895,
      "epoch": 1.643835616438356,
      "grad_norm": 15.024890919749566,
      "learning_rate": 4.586646834554864e-07,
      "logits": -1.9810107946395874,
      "logps": -110.92058563232422,
      "loss": 0.2089,
      "objective": 0.20053791999816895,
      "ranking_simple": 0.8666666746139526,
      "regularize": 0.20053791999816895,
      "step": 580,
      "wo_beta": 1.4837419986724854
    },
    {
      "dpo_loss": 0.19388006627559662,
      "epoch": 1.6580066131317903,
      "grad_norm": 11.343666690979733,
      "learning_rate": 4.234826145626855e-07,
      "logits": -1.8341389894485474,
      "logps": -109.36263275146484,
      "loss": 0.2048,
      "objective": 0.19388006627559662,
      "ranking_simple": 0.8500000238418579,
      "regularize": 0.19388006627559662,
      "step": 585,
      "wo_beta": 2.263803243637085
    },
    {
      "dpo_loss": 0.2181146889925003,
      "epoch": 1.6721776098252243,
      "grad_norm": 13.28342611357644,
      "learning_rate": 3.8957917781732883e-07,
      "logits": -1.87205970287323,
      "logps": -113.15511322021484,
      "loss": 0.2156,
      "objective": 0.2181146889925003,
      "ranking_simple": 0.8333333134651184,
      "regularize": 0.2181146889925003,
      "step": 590,
      "wo_beta": 2.4472413063049316
    },
    {
      "dpo_loss": 0.1822492927312851,
      "epoch": 1.6863486065186586,
      "grad_norm": 10.596854767592859,
      "learning_rate": 3.569752495310877e-07,
      "logits": -1.8391311168670654,
      "logps": -113.05868530273438,
      "loss": 0.1882,
      "objective": 0.1822492927312851,
      "ranking_simple": 0.8333333134651184,
      "regularize": 0.1822492927312851,
      "step": 595,
      "wo_beta": 2.815018892288208
    },
    {
      "dpo_loss": 0.17215129733085632,
      "epoch": 1.7005196032120926,
      "grad_norm": 11.460383813341208,
      "learning_rate": 3.2569090583293356e-07,
      "logits": -1.8718314170837402,
      "logps": -113.5940933227539,
      "loss": 0.1952,
      "objective": 0.17215129733085632,
      "ranking_simple": 0.8583333492279053,
      "regularize": 0.17215129733085632,
      "step": 600,
      "wo_beta": 1.7114546298980713
    },
    {
      "epoch": 1.7005196032120926,
      "eval_dpo_loss": 0.7512882947921753,
      "eval_logits": -1.970719814300537,
      "eval_logps": -117.48804473876953,
      "eval_loss": 0.7544336318969727,
      "eval_objective": 0.7512882947921753,
      "eval_ranking_simple": 0.5595238208770752,
      "eval_regularize": 0.7512882947921753,
      "eval_runtime": 367.7185,
      "eval_samples_per_second": 15.746,
      "eval_steps_per_second": 1.314,
      "eval_wo_beta": 9.029719352722168,
      "step": 600
    },
    {
      "dpo_loss": 0.19925004243850708,
      "epoch": 1.7146905999055266,
      "grad_norm": 17.415629157530653,
      "learning_rate": 2.957454103070978e-07,
      "logits": -1.8373870849609375,
      "logps": -112.65380096435547,
      "loss": 0.2262,
      "objective": 0.19925004243850708,
      "ranking_simple": 0.8333333134651184,
      "regularize": 0.19925004243850708,
      "step": 605,
      "wo_beta": 2.9711499214172363
    },
    {
      "dpo_loss": 0.21553590893745422,
      "epoch": 1.7288615965989607,
      "grad_norm": 16.124254557780983,
      "learning_rate": 2.6715720213136955e-07,
      "logits": -1.8880244493484497,
      "logps": -112.18531799316406,
      "loss": 0.1988,
      "objective": 0.21553590893745422,
      "ranking_simple": 0.8291666507720947,
      "regularize": 0.21553590893745422,
      "step": 610,
      "wo_beta": 2.8244071006774902
    },
    {
      "dpo_loss": 0.21212069690227509,
      "epoch": 1.743032593292395,
      "grad_norm": 12.906958893668936,
      "learning_rate": 2.399438847230212e-07,
      "logits": -1.9108936786651611,
      "logps": -111.8115005493164,
      "loss": 0.2027,
      "objective": 0.21212069690227509,
      "ranking_simple": 0.8166666626930237,
      "regularize": 0.21212069690227509,
      "step": 615,
      "wo_beta": 2.961397886276245
    },
    {
      "dpo_loss": 0.20163790881633759,
      "epoch": 1.7572035899858292,
      "grad_norm": 15.219355254954188,
      "learning_rate": 2.1412221489936796e-07,
      "logits": -1.947303295135498,
      "logps": -111.9202880859375,
      "loss": 0.2188,
      "objective": 0.20163790881633759,
      "ranking_simple": 0.8583333492279053,
      "regularize": 0.20163790881633759,
      "step": 620,
      "wo_beta": 1.839400053024292
    },
    {
      "dpo_loss": 0.1938161551952362,
      "epoch": 1.7713745866792632,
      "grad_norm": 17.102171291997916,
      "learning_rate": 1.897080925596187e-07,
      "logits": -1.8294084072113037,
      "logps": -111.60057830810547,
      "loss": 0.2068,
      "objective": 0.1938161551952362,
      "ranking_simple": 0.8583333492279053,
      "regularize": 0.1938161551952362,
      "step": 625,
      "wo_beta": 2.0019137859344482
    },
    {
      "dpo_loss": 0.25330111384391785,
      "epoch": 1.7855455833726972,
      "grad_norm": 14.422081659451388,
      "learning_rate": 1.6671655089439186e-07,
      "logits": -1.8686004877090454,
      "logps": -112.4965591430664,
      "loss": 0.2312,
      "objective": 0.25330111384391785,
      "ranking_simple": 0.7791666388511658,
      "regularize": 0.25330111384391785,
      "step": 630,
      "wo_beta": 3.286885976791382
    },
    {
      "dpo_loss": 0.20808285474777222,
      "epoch": 1.7997165800661312,
      "grad_norm": 16.725368606459067,
      "learning_rate": 1.4516174712890406e-07,
      "logits": -1.9575639963150024,
      "logps": -114.01631164550781,
      "loss": 0.2184,
      "objective": 0.20808285474777222,
      "ranking_simple": 0.8291666507720947,
      "regularize": 0.20808285474777222,
      "step": 635,
      "wo_beta": 2.427999258041382
    },
    {
      "dpo_loss": 0.20768284797668457,
      "epoch": 1.8138875767595655,
      "grad_norm": 12.030792650400683,
      "learning_rate": 1.2505695380554712e-07,
      "logits": -1.8871350288391113,
      "logps": -113.53579711914062,
      "loss": 0.2015,
      "objective": 0.20768284797668457,
      "ranking_simple": 0.8291666507720947,
      "regularize": 0.20768284797668457,
      "step": 640,
      "wo_beta": 2.9419753551483154
    },
    {
      "dpo_loss": 0.21022367477416992,
      "epoch": 1.8280585734529995,
      "grad_norm": 12.06072694382626,
      "learning_rate": 1.0641455061121519e-07,
      "logits": -1.9376109838485718,
      "logps": -114.6182632446289,
      "loss": 0.2087,
      "objective": 0.21022367477416992,
      "ranking_simple": 0.875,
      "regularize": 0.21022367477416992,
      "step": 645,
      "wo_beta": 2.246778964996338
    },
    {
      "dpo_loss": 0.21791066229343414,
      "epoch": 1.8422295701464337,
      "grad_norm": 17.74332836561221,
      "learning_rate": 8.924601675441207e-08,
      "logits": -1.9945629835128784,
      "logps": -115.14165496826172,
      "loss": 0.2252,
      "objective": 0.21791066229343414,
      "ranking_simple": 0.8791666626930237,
      "regularize": 0.21791066229343414,
      "step": 650,
      "wo_beta": 1.7516138553619385
    },
    {
      "epoch": 1.8422295701464337,
      "eval_dpo_loss": 0.7528993487358093,
      "eval_logits": -1.9747523069381714,
      "eval_logps": -117.80084228515625,
      "eval_loss": 0.7559728622436523,
      "eval_objective": 0.7528993487358093,
      "eval_ranking_simple": 0.5584886074066162,
      "eval_regularize": 0.7528993487358093,
      "eval_runtime": 367.648,
      "eval_samples_per_second": 15.749,
      "eval_steps_per_second": 1.314,
      "eval_wo_beta": 9.092611312866211,
      "step": 650
    },
    {
      "dpo_loss": 0.21706603467464447,
      "epoch": 1.8564005668398678,
      "grad_norm": 13.027657065435218,
      "learning_rate": 7.356192389683825e-08,
      "logits": -1.844641923904419,
      "logps": -110.724609375,
      "loss": 0.2263,
      "objective": 0.21706603467464447,
      "ranking_simple": 0.8333333134651184,
      "regularize": 0.21706603467464447,
      "step": 655,
      "wo_beta": 2.2099640369415283
    },
    {
      "dpo_loss": 0.19901646673679352,
      "epoch": 1.8705715635333018,
      "grad_norm": 11.442935681801226,
      "learning_rate": 5.937192964380556e-08,
      "logits": -1.9180775880813599,
      "logps": -115.1898193359375,
      "loss": 0.1992,
      "objective": 0.19901646673679352,
      "ranking_simple": 0.8708333373069763,
      "regularize": 0.19901646673679352,
      "step": 660,
      "wo_beta": 1.9250274896621704
    },
    {
      "dpo_loss": 0.20594698190689087,
      "epoch": 1.8847425602267358,
      "grad_norm": 14.384890803281685,
      "learning_rate": 4.668477159748858e-08,
      "logits": -1.8975155353546143,
      "logps": -112.05779266357422,
      "loss": 0.22,
      "objective": 0.20594698190689087,
      "ranking_simple": 0.8458333611488342,
      "regularize": 0.20594698190689087,
      "step": 665,
      "wo_beta": 1.7807292938232422
    },
    {
      "dpo_loss": 0.23021073639392853,
      "epoch": 1.89891355692017,
      "grad_norm": 11.678760414899436,
      "learning_rate": 3.5508261976678894e-08,
      "logits": -1.8350870609283447,
      "logps": -111.71804809570312,
      "loss": 0.2054,
      "objective": 0.23021073639392853,
      "ranking_simple": 0.8208333253860474,
      "regularize": 0.23021073639392853,
      "step": 670,
      "wo_beta": 2.838752269744873
    },
    {
      "dpo_loss": 0.19814661145210266,
      "epoch": 1.9130845536136043,
      "grad_norm": 14.42884511344777,
      "learning_rate": 2.5849282806345855e-08,
      "logits": -1.8832274675369263,
      "logps": -112.52620697021484,
      "loss": 0.2208,
      "objective": 0.19814661145210266,
      "ranking_simple": 0.8791666626930237,
      "regularize": 0.19814661145210266,
      "step": 675,
      "wo_beta": 1.6049467325210571
    },
    {
      "dpo_loss": 0.2337852120399475,
      "epoch": 1.9272555503070383,
      "grad_norm": 12.72035969054902,
      "learning_rate": 1.771378167997745e-08,
      "logits": -1.9469962120056152,
      "logps": -114.031494140625,
      "loss": 0.2132,
      "objective": 0.2337852120399475,
      "ranking_simple": 0.8541666865348816,
      "regularize": 0.2337852120399475,
      "step": 680,
      "wo_beta": 2.8053526878356934
    },
    {
      "dpo_loss": 0.18736791610717773,
      "epoch": 1.9414265470004723,
      "grad_norm": 11.578463444163043,
      "learning_rate": 1.1106768097300657e-08,
      "logits": -1.8774739503860474,
      "logps": -113.84810638427734,
      "loss": 0.1807,
      "objective": 0.18736791610717773,
      "ranking_simple": 0.8458333611488342,
      "regularize": 0.18736791610717773,
      "step": 685,
      "wo_beta": 2.1088075637817383
    },
    {
      "dpo_loss": 0.19145923852920532,
      "epoch": 1.9555975436939064,
      "grad_norm": 11.501744444827747,
      "learning_rate": 6.032310379642803e-09,
      "logits": -1.9342644214630127,
      "logps": -112.51237487792969,
      "loss": 0.204,
      "objective": 0.19145923852920532,
      "ranking_simple": 0.8583333492279053,
      "regularize": 0.19145923852920532,
      "step": 690,
      "wo_beta": 1.7116554975509644
    },
    {
      "dpo_loss": 0.2177831530570984,
      "epoch": 1.9697685403873406,
      "grad_norm": 13.662395330088454,
      "learning_rate": 2.4935331648298644e-09,
      "logits": -1.7998664379119873,
      "logps": -113.02543640136719,
      "loss": 0.2048,
      "objective": 0.2177831530570984,
      "ranking_simple": 0.8166666626930237,
      "regularize": 0.2177831530570984,
      "step": 695,
      "wo_beta": 2.7438058853149414
    },
    {
      "dpo_loss": 0.19972722232341766,
      "epoch": 1.9839395370807746,
      "grad_norm": 13.232400433126339,
      "learning_rate": 4.926154831655372e-10,
      "logits": -1.788934350013733,
      "logps": -115.31278228759766,
      "loss": 0.199,
      "objective": 0.19972722232341766,
      "ranking_simple": 0.8583333492279053,
      "regularize": 0.19972722232341766,
      "step": 700,
      "wo_beta": 2.272697687149048
    },
    {
      "epoch": 1.9839395370807746,
      "eval_dpo_loss": 0.7537301182746887,
      "eval_logits": -1.9777543544769287,
      "eval_logps": -117.9869155883789,
      "eval_loss": 0.7566089034080505,
      "eval_objective": 0.7537301182746887,
      "eval_ranking_simple": 0.5595238208770752,
      "eval_regularize": 0.7537301182746887,
      "eval_runtime": 367.5678,
      "eval_samples_per_second": 15.752,
      "eval_steps_per_second": 1.314,
      "eval_wo_beta": 9.104193687438965,
      "step": 700
    },
    {
      "epoch": 1.995276334435522,
      "step": 704,
      "total_flos": 0.0,
      "train_loss": 0.3641128831289031,
      "train_runtime": 23012.8835,
      "train_samples_per_second": 4.415,
      "train_steps_per_second": 0.031
    }
  ],
  "logging_steps": 5,
  "max_steps": 704,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}