{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.9765925925925925,
  "eval_steps": 500,
  "global_step": 315,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.047407407407407405,
      "grad_norm": 899.4839477539062,
      "learning_rate": 7.8125e-06,
      "log_odds_chosen": -3.253441572189331,
      "log_odds_ratio": -11.290441513061523,
      "logits/chosen": 151.96023559570312,
      "logits/rejected": 125.16461181640625,
      "logps/chosen": -21.246450424194336,
      "logps/rejected": -17.994016647338867,
      "loss": 456.0568,
      "nll_loss": 9.107990264892578,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -10.623225212097168,
      "rewards/margins": -1.6262180805206299,
      "rewards/rejected": -8.997008323669434,
      "step": 5
    },
    {
      "epoch": 0.09481481481481481,
      "grad_norm": 665.6010131835938,
      "learning_rate": 1.5625e-05,
      "log_odds_chosen": -0.5072727203369141,
      "log_odds_ratio": -9.659940719604492,
      "logits/chosen": 151.84805297851562,
      "logits/rejected": 114.96217346191406,
      "logps/chosen": -19.77961540222168,
      "logps/rejected": -19.273290634155273,
      "loss": 394.0516,
      "nll_loss": 8.324145317077637,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -9.88980770111084,
      "rewards/margins": -0.25316303968429565,
      "rewards/rejected": -9.636645317077637,
      "step": 10
    },
    {
      "epoch": 0.14222222222222222,
      "grad_norm": 584.8187255859375,
      "learning_rate": 2.34375e-05,
      "log_odds_chosen": -3.0495519638061523,
      "log_odds_ratio": -12.564888000488281,
      "logits/chosen": 165.22348022460938,
      "logits/rejected": 135.7709197998047,
      "logps/chosen": -23.73749351501465,
      "logps/rejected": -20.685997009277344,
      "loss": 398.6285,
      "nll_loss": 8.013145446777344,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -11.868746757507324,
      "rewards/margins": -1.5257480144500732,
      "rewards/rejected": -10.342998504638672,
      "step": 15
    },
    {
      "epoch": 0.18962962962962962,
      "grad_norm": 2222.182861328125,
      "learning_rate": 3.125e-05,
      "log_odds_chosen": -1.6242220401763916,
      "log_odds_ratio": -6.402337551116943,
      "logits/chosen": 166.9097137451172,
      "logits/rejected": 148.36062622070312,
      "logps/chosen": -13.310510635375977,
      "logps/rejected": -11.688575744628906,
      "loss": 263.4254,
      "nll_loss": 5.480126857757568,
      "rewards/accuracies": 0.4312500059604645,
      "rewards/chosen": -6.655255317687988,
      "rewards/margins": -0.8109671473503113,
      "rewards/rejected": -5.844287872314453,
      "step": 20
    },
    {
      "epoch": 0.23703703703703705,
      "grad_norm": 239.38790893554688,
      "learning_rate": 3.90625e-05,
      "log_odds_chosen": -0.3463691771030426,
      "log_odds_ratio": -1.4706324338912964,
      "logits/chosen": 178.37814331054688,
      "logits/rejected": 188.9853515625,
      "logps/chosen": -2.9337403774261475,
      "logps/rejected": -2.58659291267395,
      "loss": 101.069,
      "nll_loss": 2.465604305267334,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -1.4668701887130737,
      "rewards/margins": -0.17357376217842102,
      "rewards/rejected": -1.293296456336975,
      "step": 25
    },
    {
      "epoch": 0.28444444444444444,
      "grad_norm": 168.3093719482422,
      "learning_rate": 4.6875e-05,
      "log_odds_chosen": -0.08818509429693222,
      "log_odds_ratio": -0.9202126264572144,
      "logits/chosen": 208.46914672851562,
      "logits/rejected": 211.0690155029297,
      "logps/chosen": -1.8011804819107056,
      "logps/rejected": -1.7284847497940063,
      "loss": 76.8201,
      "nll_loss": 1.9414145946502686,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.9005902409553528,
      "rewards/margins": -0.036347877234220505,
      "rewards/rejected": -0.8642423748970032,
      "step": 30
    },
    {
      "epoch": 0.33185185185185184,
      "grad_norm": 344.3992919921875,
      "learning_rate": 4.998613757348784e-05,
      "log_odds_chosen": 0.2355690896511078,
      "log_odds_ratio": -0.7713075876235962,
      "logits/chosen": 221.82272338867188,
      "logits/rejected": 217.92764282226562,
      "logps/chosen": -1.5275888442993164,
      "logps/rejected": -1.7404054403305054,
      "loss": 73.3205,
      "nll_loss": 1.7808992862701416,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7637944221496582,
      "rewards/margins": 0.1064082607626915,
      "rewards/rejected": -0.8702027201652527,
      "step": 35
    },
    {
      "epoch": 0.37925925925925924,
      "grad_norm": 205.59912109375,
      "learning_rate": 4.990147841143462e-05,
      "log_odds_chosen": 0.3138998746871948,
      "log_odds_ratio": -0.7420972585678101,
      "logits/chosen": 225.82821655273438,
      "logits/rejected": 226.96774291992188,
      "logps/chosen": -1.5016671419143677,
      "logps/rejected": -1.7676680088043213,
      "loss": 71.1531,
      "nll_loss": 1.7955719232559204,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.7508335709571838,
      "rewards/margins": 0.1330004334449768,
      "rewards/rejected": -0.8838340044021606,
      "step": 40
    },
    {
      "epoch": 0.4266666666666667,
      "grad_norm": 139.50970458984375,
      "learning_rate": 4.97401218720448e-05,
      "log_odds_chosen": 0.1954536736011505,
      "log_odds_ratio": -0.72107994556427,
      "logits/chosen": 208.2395782470703,
      "logits/rejected": 205.417236328125,
      "logps/chosen": -1.3215817213058472,
      "logps/rejected": -1.5023378133773804,
      "loss": 66.4276,
      "nll_loss": 1.647528886795044,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.6607908606529236,
      "rewards/margins": 0.0903780609369278,
      "rewards/rejected": -0.7511689066886902,
      "step": 45
    },
    {
      "epoch": 0.4740740740740741,
      "grad_norm": 77.09749603271484,
      "learning_rate": 4.9502564938797946e-05,
      "log_odds_chosen": 0.19976872205734253,
      "log_odds_ratio": -0.7074006199836731,
      "logits/chosen": 202.8539276123047,
      "logits/rejected": 195.70155334472656,
      "logps/chosen": -1.2350599765777588,
      "logps/rejected": -1.4153530597686768,
      "loss": 63.0281,
      "nll_loss": 1.5506360530853271,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.6175299882888794,
      "rewards/margins": 0.0901465192437172,
      "rewards/rejected": -0.7076765298843384,
      "step": 50
    },
    {
      "epoch": 0.5214814814814814,
      "grad_norm": 59.40414810180664,
      "learning_rate": 4.918953929490768e-05,
      "log_odds_chosen": 0.14897426962852478,
      "log_odds_ratio": -0.7380977869033813,
      "logits/chosen": 190.48110961914062,
      "logits/rejected": 195.70455932617188,
      "logps/chosen": -1.221083641052246,
      "logps/rejected": -1.3429195880889893,
      "loss": 62.9068,
      "nll_loss": 1.543068528175354,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.610541820526123,
      "rewards/margins": 0.0609179362654686,
      "rewards/rejected": -0.6714597940444946,
      "step": 55
    },
    {
      "epoch": 0.5688888888888889,
      "grad_norm": 126.07240295410156,
      "learning_rate": 4.88020090697132e-05,
      "log_odds_chosen": 0.15872475504875183,
      "log_odds_ratio": -0.7120335102081299,
      "logits/chosen": 186.23619079589844,
      "logits/rejected": 179.56393432617188,
      "logps/chosen": -1.2440166473388672,
      "logps/rejected": -1.3791967630386353,
      "loss": 63.2646,
      "nll_loss": 1.5736169815063477,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.6220083236694336,
      "rewards/margins": 0.06759000569581985,
      "rewards/rejected": -0.6895983815193176,
      "step": 60
    },
    {
      "epoch": 0.6162962962962963,
      "grad_norm": 139.16448974609375,
      "learning_rate": 4.834116786912897e-05,
      "log_odds_chosen": 0.18847045302391052,
      "log_odds_ratio": -0.7004148364067078,
      "logits/chosen": 180.8818359375,
      "logits/rejected": 185.00570678710938,
      "logps/chosen": -1.2013404369354248,
      "logps/rejected": -1.3533903360366821,
      "loss": 60.8734,
      "nll_loss": 1.485313057899475,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.6006702184677124,
      "rewards/margins": 0.07602496445178986,
      "rewards/rejected": -0.6766951680183411,
      "step": 65
    },
    {
      "epoch": 0.6637037037037037,
      "grad_norm": 251.40016174316406,
      "learning_rate": 4.7808435099299045e-05,
      "log_odds_chosen": 0.18260684609413147,
      "log_odds_ratio": -0.7129999995231628,
      "logits/chosen": 182.959716796875,
      "logits/rejected": 183.2157745361328,
      "logps/chosen": -1.1487746238708496,
      "logps/rejected": -1.2979029417037964,
      "loss": 61.4203,
      "nll_loss": 1.4665228128433228,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.5743873119354248,
      "rewards/margins": 0.07456417381763458,
      "rewards/rejected": -0.6489514708518982,
      "step": 70
    },
    {
      "epoch": 0.7111111111111111,
      "grad_norm": 124.29930877685547,
      "learning_rate": 4.720545159477922e-05,
      "log_odds_chosen": 0.17588524520397186,
      "log_odds_ratio": -0.7309717535972595,
      "logits/chosen": 175.79225158691406,
      "logits/rejected": 185.4251708984375,
      "logps/chosen": -1.171048641204834,
      "logps/rejected": -1.3039839267730713,
      "loss": 60.8133,
      "nll_loss": 1.5102540254592896,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.585524320602417,
      "rewards/margins": 0.06646756827831268,
      "rewards/rejected": -0.6519919633865356,
      "step": 75
    },
    {
      "epoch": 0.7585185185185185,
      "grad_norm": 358.4803771972656,
      "learning_rate": 4.653407456471222e-05,
      "log_odds_chosen": 0.1115400418639183,
      "log_odds_ratio": -0.7338749170303345,
      "logits/chosen": 191.45970153808594,
      "logits/rejected": 181.09698486328125,
      "logps/chosen": -1.1183122396469116,
      "logps/rejected": -1.2054791450500488,
      "loss": 57.9734,
      "nll_loss": 1.4206385612487793,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.5591561198234558,
      "rewards/margins": 0.04358343034982681,
      "rewards/rejected": -0.6027395725250244,
      "step": 80
    },
    {
      "epoch": 0.8059259259259259,
      "grad_norm": 136.4271697998047,
      "learning_rate": 4.579637187256222e-05,
      "log_odds_chosen": 0.04175865277647972,
      "log_odds_ratio": -0.7415879964828491,
      "logits/chosen": 192.60121154785156,
      "logits/rejected": 192.03916931152344,
      "logps/chosen": -1.0868916511535645,
      "logps/rejected": -1.1372497081756592,
      "loss": 59.0801,
      "nll_loss": 1.4550647735595703,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.5434458255767822,
      "rewards/margins": 0.025179097428917885,
      "rewards/rejected": -0.5686248540878296,
      "step": 85
    },
    {
      "epoch": 0.8533333333333334,
      "grad_norm": 56.09324645996094,
      "learning_rate": 4.499461566702685e-05,
      "log_odds_chosen": 0.12758071720600128,
      "log_odds_ratio": -0.7168221473693848,
      "logits/chosen": 182.20901489257812,
      "logits/rejected": 187.005126953125,
      "logps/chosen": -1.0412800312042236,
      "logps/rejected": -1.140512466430664,
      "loss": 56.929,
      "nll_loss": 1.3427871465682983,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.5206400156021118,
      "rewards/margins": 0.04961626976728439,
      "rewards/rejected": -0.570256233215332,
      "step": 90
    },
    {
      "epoch": 0.9007407407407407,
      "grad_norm": 101.89226531982422,
      "learning_rate": 4.413127538374411e-05,
      "log_odds_chosen": 0.03433248773217201,
      "log_odds_ratio": -0.7621655464172363,
      "logits/chosen": 183.79061889648438,
      "logits/rejected": 181.02096557617188,
      "logps/chosen": -1.1271755695343018,
      "logps/rejected": -1.155602216720581,
      "loss": 57.6873,
      "nll_loss": 1.4238755702972412,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.5635877847671509,
      "rewards/margins": 0.014213320799171925,
      "rewards/rejected": -0.5778011083602905,
      "step": 95
    },
    {
      "epoch": 0.9481481481481482,
      "grad_norm": 56.19839859008789,
      "learning_rate": 4.320901013934887e-05,
      "log_odds_chosen": 0.19043493270874023,
      "log_odds_ratio": -0.6898019313812256,
      "logits/chosen": 187.52651977539062,
      "logits/rejected": 188.4263458251953,
      "logps/chosen": -1.064992904663086,
      "logps/rejected": -1.2005788087844849,
      "loss": 56.784,
      "nll_loss": 1.3723514080047607,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.532496452331543,
      "rewards/margins": 0.06779297441244125,
      "rewards/rejected": -0.6002894043922424,
      "step": 100
    },
    {
      "epoch": 0.9955555555555555,
      "grad_norm": 134.25718688964844,
      "learning_rate": 4.223066054130568e-05,
      "log_odds_chosen": 0.19098126888275146,
      "log_odds_ratio": -0.708988606929779,
      "logits/chosen": 186.099365234375,
      "logits/rejected": 191.89564514160156,
      "logps/chosen": -1.0854551792144775,
      "logps/rejected": -1.2490203380584717,
      "loss": 59.6264,
      "nll_loss": 1.4756276607513428,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.5427275896072388,
      "rewards/margins": 0.0817825049161911,
      "rewards/rejected": -0.6245101690292358,
      "step": 105
    },
    {
      "epoch": 1.037925925925926,
      "grad_norm": 48.896156311035156,
      "learning_rate": 4.1199239938743797e-05,
      "log_odds_chosen": 0.09143175929784775,
      "log_odds_ratio": -0.7199470400810242,
      "logits/chosen": 192.02879333496094,
      "logits/rejected": 189.01097106933594,
      "logps/chosen": -1.0210695266723633,
      "logps/rejected": -1.0847587585449219,
      "loss": 46.1645,
      "nll_loss": 1.2595568895339966,
      "rewards/accuracies": 0.5384615659713745,
      "rewards/chosen": -0.5105347633361816,
      "rewards/margins": 0.031844645738601685,
      "rewards/rejected": -0.5423793792724609,
      "step": 110
    },
    {
      "epoch": 1.0853333333333333,
      "grad_norm": 44.063316345214844,
      "learning_rate": 4.0117925141242174e-05,
      "log_odds_chosen": 0.09873426705598831,
      "log_odds_ratio": -0.741472601890564,
      "logits/chosen": 185.77395629882812,
      "logits/rejected": 194.8834686279297,
      "logps/chosen": -0.9893190264701843,
      "logps/rejected": -1.0386199951171875,
      "loss": 50.7825,
      "nll_loss": 1.2114965915679932,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.49465951323509216,
      "rewards/margins": 0.024650435894727707,
      "rewards/rejected": -0.5193099975585938,
      "step": 115
    },
    {
      "epoch": 1.1327407407407408,
      "grad_norm": 42.30266571044922,
      "learning_rate": 3.899004663415084e-05,
      "log_odds_chosen": 0.03399781137704849,
      "log_odds_ratio": -0.7742182612419128,
      "logits/chosen": 182.58143615722656,
      "logits/rejected": 186.83212280273438,
      "logps/chosen": -0.9977661967277527,
      "logps/rejected": -1.0048539638519287,
      "loss": 51.3458,
      "nll_loss": 1.2454302310943604,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.49888309836387634,
      "rewards/margins": 0.0035438979975879192,
      "rewards/rejected": -0.5024269819259644,
      "step": 120
    },
    {
      "epoch": 1.1801481481481482,
      "grad_norm": 125.85904693603516,
      "learning_rate": 3.781907832058587e-05,
      "log_odds_chosen": 0.09150873124599457,
      "log_odds_ratio": -0.7233962416648865,
      "logits/chosen": 181.49447631835938,
      "logits/rejected": 180.0559844970703,
      "logps/chosen": -0.9917734861373901,
      "logps/rejected": -1.051242470741272,
      "loss": 49.8693,
      "nll_loss": 1.1629549264907837,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.49588674306869507,
      "rewards/margins": 0.029734421521425247,
      "rewards/rejected": -0.525621235370636,
      "step": 125
    },
    {
      "epoch": 1.2275555555555555,
      "grad_norm": 39.95571517944336,
      "learning_rate": 3.660862682169282e-05,
      "log_odds_chosen": 0.09172184765338898,
      "log_odds_ratio": -0.7451839447021484,
      "logits/chosen": 179.13302612304688,
      "logits/rejected": 183.4129638671875,
      "logps/chosen": -1.0290215015411377,
      "logps/rejected": -1.0855556726455688,
      "loss": 51.0956,
      "nll_loss": 1.1969176530838013,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.5145107507705688,
      "rewards/margins": 0.028267022222280502,
      "rewards/rejected": -0.5427778363227844,
      "step": 130
    },
    {
      "epoch": 1.274962962962963,
      "grad_norm": 115.57888793945312,
      "learning_rate": 3.5362420368134356e-05,
      "log_odds_chosen": 0.07470552623271942,
      "log_odds_ratio": -0.7265952825546265,
      "logits/chosen": 183.11488342285156,
      "logits/rejected": 177.27346801757812,
      "logps/chosen": -0.9192112684249878,
      "logps/rejected": -0.9482501745223999,
      "loss": 49.3732,
      "nll_loss": 1.1410008668899536,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.4596056342124939,
      "rewards/margins": 0.0145194623619318,
      "rewards/rejected": -0.47412508726119995,
      "step": 135
    },
    {
      "epoch": 1.3223703703703704,
      "grad_norm": 40.40840148925781,
      "learning_rate": 3.408429731701635e-05,
      "log_odds_chosen": 0.15658657252788544,
      "log_odds_ratio": -0.7292534112930298,
      "logits/chosen": 183.2898712158203,
      "logits/rejected": 183.63088989257812,
      "logps/chosen": -0.9870889782905579,
      "logps/rejected": -1.0798033475875854,
      "loss": 51.391,
      "nll_loss": 1.1972719430923462,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.49354448914527893,
      "rewards/margins": 0.046357229351997375,
      "rewards/rejected": -0.5399016737937927,
      "step": 140
    },
    {
      "epoch": 1.3697777777777778,
      "grad_norm": 43.5965461730957,
      "learning_rate": 3.2778194329621104e-05,
      "log_odds_chosen": 0.0713091716170311,
      "log_odds_ratio": -0.7587572932243347,
      "logits/chosen": 175.42311096191406,
      "logits/rejected": 179.28359985351562,
      "logps/chosen": -0.9829910397529602,
      "logps/rejected": -1.0261934995651245,
      "loss": 51.113,
      "nll_loss": 1.2084944248199463,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.4914955198764801,
      "rewards/margins": 0.021601270884275436,
      "rewards/rejected": -0.5130967497825623,
      "step": 145
    },
    {
      "epoch": 1.417185185185185,
      "grad_norm": 68.55988311767578,
      "learning_rate": 3.144813424636031e-05,
      "log_odds_chosen": 0.03821509703993797,
      "log_odds_ratio": -0.7714879512786865,
      "logits/chosen": 173.73973083496094,
      "logits/rejected": 177.93600463867188,
      "logps/chosen": -1.0191147327423096,
      "logps/rejected": -1.0414360761642456,
      "loss": 49.2398,
      "nll_loss": 1.162936806678772,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.5095573663711548,
      "rewards/margins": 0.011160662397742271,
      "rewards/rejected": -0.5207180380821228,
      "step": 150
    },
    {
      "epoch": 1.4645925925925927,
      "grad_norm": 87.03449249267578,
      "learning_rate": 3.0098213696293542e-05,
      "log_odds_chosen": 0.10836371034383774,
      "log_odds_ratio": -0.7314402461051941,
      "logits/chosen": 175.88619995117188,
      "logits/rejected": 182.52371215820312,
      "logps/chosen": -0.9865490794181824,
      "logps/rejected": -1.0499975681304932,
      "loss": 50.2605,
      "nll_loss": 1.1926219463348389,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.4932745397090912,
      "rewards/margins": 0.03172413632273674,
      "rewards/rejected": -0.5249987840652466,
      "step": 155
    },
    {
      "epoch": 1.512,
      "grad_norm": 58.0168342590332,
      "learning_rate": 2.8732590479375165e-05,
      "log_odds_chosen": 0.12078354507684708,
      "log_odds_ratio": -0.7301235795021057,
      "logits/chosen": 173.8167266845703,
      "logits/rejected": 181.9156494140625,
      "logps/chosen": -0.9404324293136597,
      "logps/rejected": -1.0260807275772095,
      "loss": 48.9856,
      "nll_loss": 1.1453100442886353,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.47021621465682983,
      "rewards/margins": 0.0428241565823555,
      "rewards/rejected": -0.5130403637886047,
      "step": 160
    },
    {
      "epoch": 1.5594074074074074,
      "grad_norm": 47.982913970947266,
      "learning_rate": 2.7355470760292956e-05,
      "log_odds_chosen": 0.0946766808629036,
      "log_odds_ratio": -0.7334403395652771,
      "logits/chosen": 166.52523803710938,
      "logits/rejected": 171.69485473632812,
      "logps/chosen": -0.9203466176986694,
      "logps/rejected": -0.9718505144119263,
      "loss": 50.1007,
      "nll_loss": 1.1756643056869507,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.4601733088493347,
      "rewards/margins": 0.025751952081918716,
      "rewards/rejected": -0.48592525720596313,
      "step": 165
    },
    {
      "epoch": 1.6068148148148147,
      "grad_norm": 50.041561126708984,
      "learning_rate": 2.597109611334169e-05,
      "log_odds_chosen": 0.17343485355377197,
      "log_odds_ratio": -0.6794081926345825,
      "logits/chosen": 171.59066772460938,
      "logits/rejected": 179.27670288085938,
      "logps/chosen": -0.936198353767395,
      "logps/rejected": -1.0650147199630737,
      "loss": 49.7789,
      "nll_loss": 1.186452031135559,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.4680991768836975,
      "rewards/margins": 0.06440822780132294,
      "rewards/rejected": -0.5325073599815369,
      "step": 170
    },
    {
      "epoch": 1.6542222222222223,
      "grad_norm": 47.45258331298828,
      "learning_rate": 2.458373045823404e-05,
      "log_odds_chosen": 0.07308103144168854,
      "log_odds_ratio": -0.7454826235771179,
      "logits/chosen": 172.97149658203125,
      "logits/rejected": 172.43182373046875,
      "logps/chosen": -0.9455814361572266,
      "logps/rejected": -1.0050225257873535,
      "loss": 49.1083,
      "nll_loss": 1.1290260553359985,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.4727907180786133,
      "rewards/margins": 0.029720569029450417,
      "rewards/rejected": -0.5025112628936768,
      "step": 175
    },
    {
      "epoch": 1.7016296296296296,
      "grad_norm": 58.00626754760742,
      "learning_rate": 2.3197646927086697e-05,
      "log_odds_chosen": 0.1577189862728119,
      "log_odds_ratio": -0.7023075819015503,
      "logits/chosen": 168.512451171875,
      "logits/rejected": 175.3289794921875,
      "logps/chosen": -0.9105680584907532,
      "logps/rejected": -1.0289887189865112,
      "loss": 49.2958,
      "nll_loss": 1.1708818674087524,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.4552840292453766,
      "rewards/margins": 0.059210360050201416,
      "rewards/rejected": -0.5144943594932556,
      "step": 180
    },
    {
      "epoch": 1.749037037037037,
      "grad_norm": 34.98508071899414,
      "learning_rate": 2.1817114703032176e-05,
      "log_odds_chosen": 0.12351224571466446,
      "log_odds_ratio": -0.7270097732543945,
      "logits/chosen": 173.38623046875,
      "logits/rejected": 165.6890869140625,
      "logps/chosen": -0.9919120669364929,
      "logps/rejected": -1.0763611793518066,
      "loss": 48.9183,
      "nll_loss": 1.1456924676895142,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.49595603346824646,
      "rewards/margins": 0.04222451522946358,
      "rewards/rejected": -0.5381805896759033,
      "step": 185
    },
    {
      "epoch": 1.7964444444444445,
      "grad_norm": 35.57364273071289,
      "learning_rate": 2.0446385870993467e-05,
      "log_odds_chosen": 0.046650804579257965,
      "log_odds_ratio": -0.7537095546722412,
      "logits/chosen": 161.9040069580078,
      "logits/rejected": 164.87686157226562,
      "logps/chosen": -0.9197586178779602,
      "logps/rejected": -0.9383406639099121,
      "loss": 49.163,
      "nll_loss": 1.1618720293045044,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.4598793089389801,
      "rewards/margins": 0.009291025809943676,
      "rewards/rejected": -0.46917033195495605,
      "step": 190
    },
    {
      "epoch": 1.8438518518518519,
      "grad_norm": 57.462890625,
      "learning_rate": 1.9089682321121834e-05,
      "log_odds_chosen": 0.10921354591846466,
      "log_odds_ratio": -0.7022596001625061,
      "logits/chosen": 164.3706817626953,
      "logits/rejected": 167.53907775878906,
      "logps/chosen": -0.9306834936141968,
      "logps/rejected": -1.0057896375656128,
      "loss": 48.8493,
      "nll_loss": 1.1373628377914429,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.4653417468070984,
      "rewards/margins": 0.03755306452512741,
      "rewards/rejected": -0.5028948187828064,
      "step": 195
    },
    {
      "epoch": 1.8912592592592592,
      "grad_norm": 51.58010482788086,
      "learning_rate": 1.775118274523545e-05,
      "log_odds_chosen": 0.0803312212228775,
      "log_odds_ratio": -0.7448180913925171,
      "logits/chosen": 177.42755126953125,
      "logits/rejected": 166.17420959472656,
      "logps/chosen": -0.971086323261261,
      "logps/rejected": -1.025727391242981,
      "loss": 50.0697,
      "nll_loss": 1.1380794048309326,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.4855431616306305,
      "rewards/margins": 0.027320479974150658,
      "rewards/rejected": -0.5128636956214905,
      "step": 200
    },
    {
      "epoch": 1.9386666666666668,
      "grad_norm": 40.688987731933594,
      "learning_rate": 1.643500976631037e-05,
      "log_odds_chosen": 0.10912259668111801,
      "log_odds_ratio": -0.7380350232124329,
      "logits/chosen": 173.36325073242188,
      "logits/rejected": 171.0248565673828,
      "logps/chosen": -0.9703338742256165,
      "logps/rejected": -1.0370721817016602,
      "loss": 48.6517,
      "nll_loss": 1.1243274211883545,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.4851669371128082,
      "rewards/margins": 0.03336922079324722,
      "rewards/rejected": -0.5185360908508301,
      "step": 205
    },
    {
      "epoch": 1.986074074074074,
      "grad_norm": 47.03554153442383,
      "learning_rate": 1.514521724066537e-05,
      "log_odds_chosen": 0.07311935722827911,
      "log_odds_ratio": -0.7410269975662231,
      "logits/chosen": 161.46792602539062,
      "logits/rejected": 176.0153045654297,
      "logps/chosen": -0.9304733276367188,
      "logps/rejected": -0.9752525091171265,
      "loss": 49.3022,
      "nll_loss": 1.163869857788086,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.4652366638183594,
      "rewards/margins": 0.02238963544368744,
      "rewards/rejected": -0.48762625455856323,
      "step": 210
    },
    {
      "epoch": 2.0284444444444443,
      "grad_norm": 46.73811721801758,
      "learning_rate": 1.3885777771950348e-05,
      "log_odds_chosen": 0.04928523674607277,
      "log_odds_ratio": -0.7592554688453674,
      "logits/chosen": 154.12142944335938,
      "logits/rejected": 166.54530334472656,
      "logps/chosen": -0.8669578433036804,
      "logps/rejected": -0.8768206238746643,
      "loss": 38.0486,
      "nll_loss": 0.969521701335907,
      "rewards/accuracies": 0.5314685106277466,
      "rewards/chosen": -0.4334789216518402,
      "rewards/margins": 0.004931411240249872,
      "rewards/rejected": -0.43841031193733215,
      "step": 215
    },
    {
      "epoch": 2.075851851851852,
      "grad_norm": 66.5871353149414,
      "learning_rate": 1.2660570475395683e-05,
      "log_odds_chosen": 0.07944320142269135,
      "log_odds_ratio": -0.7612829804420471,
      "logits/chosen": 158.1134033203125,
      "logits/rejected": 145.2221221923828,
      "logps/chosen": -0.8715459704399109,
      "logps/rejected": -0.8965581059455872,
      "loss": 42.355,
      "nll_loss": 0.9041290283203125,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.43577298521995544,
      "rewards/margins": 0.01250611525028944,
      "rewards/rejected": -0.4482790529727936,
      "step": 220
    },
    {
      "epoch": 2.1232592592592594,
      "grad_norm": 43.993412017822266,
      "learning_rate": 1.1473369030008974e-05,
      "log_odds_chosen": 0.07922948896884918,
      "log_odds_ratio": -0.748726487159729,
      "logits/chosen": 150.79869079589844,
      "logits/rejected": 159.51519775390625,
      "logps/chosen": -0.9018028974533081,
      "logps/rejected": -0.9198867082595825,
      "loss": 41.5811,
      "nll_loss": 0.9309173822402954,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.45090144872665405,
      "rewards/margins": 0.009041833691298962,
      "rewards/rejected": -0.45994335412979126,
      "step": 225
    },
    {
      "epoch": 2.1706666666666665,
      "grad_norm": 42.56783676147461,
      "learning_rate": 1.0327830055518842e-05,
      "log_odds_chosen": 0.1404309719800949,
      "log_odds_ratio": -0.7541959285736084,
      "logits/chosen": 149.96189880371094,
      "logits/rejected": 155.0712127685547,
      "logps/chosen": -0.8312407732009888,
      "logps/rejected": -0.9046605825424194,
      "loss": 41.5919,
      "nll_loss": 0.9145228266716003,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.4156203866004944,
      "rewards/margins": 0.03670995682477951,
      "rewards/rejected": -0.4523302912712097,
      "step": 230
    },
    {
      "epoch": 2.218074074074074,
      "grad_norm": 46.45758819580078,
      "learning_rate": 9.227481849865235e-06,
      "log_odds_chosen": -0.03481599688529968,
      "log_odds_ratio": -0.8210490942001343,
      "logits/chosen": 156.96890258789062,
      "logits/rejected": 156.92953491210938,
      "logps/chosen": -0.899451732635498,
      "logps/rejected": -0.8785243034362793,
      "loss": 42.667,
      "nll_loss": 0.988889217376709,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.449725866317749,
      "rewards/margins": -0.010463694110512733,
      "rewards/rejected": -0.43926215171813965,
      "step": 235
    },
    {
      "epoch": 2.2654814814814817,
      "grad_norm": 41.77062225341797,
      "learning_rate": 8.175713521924978e-06,
      "log_odds_chosen": 0.19800688326358795,
      "log_odds_ratio": -0.7289772033691406,
      "logits/chosen": 161.128173828125,
      "logits/rejected": 159.4349365234375,
      "logps/chosen": -0.8593847155570984,
      "logps/rejected": -0.9744852185249329,
      "loss": 41.5045,
      "nll_loss": 0.9035336375236511,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.4296923577785492,
      "rewards/margins": 0.05755022168159485,
      "rewards/rejected": -0.48724260926246643,
      "step": 240
    },
    {
      "epoch": 2.3128888888888888,
      "grad_norm": 41.01064682006836,
      "learning_rate": 7.1757645529443665e-06,
      "log_odds_chosen": -0.0014010012382641435,
      "log_odds_ratio": -0.7947742342948914,
      "logits/chosen": 155.06546020507812,
      "logits/rejected": 155.10775756835938,
      "logps/chosen": -0.878911018371582,
      "logps/rejected": -0.8687770962715149,
      "loss": 41.6816,
      "nll_loss": 0.9432564973831177,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.439455509185791,
      "rewards/margins": -0.0050669461488723755,
      "rewards/rejected": -0.43438854813575745,
      "step": 245
    },
    {
      "epoch": 2.3602962962962963,
      "grad_norm": 41.34367752075195,
      "learning_rate": 6.230714818829733e-06,
      "log_odds_chosen": 0.0013508498668670654,
      "log_odds_ratio": -0.7936294674873352,
      "logits/chosen": 153.34498596191406,
      "logits/rejected": 158.2041015625,
      "logps/chosen": -0.8574392199516296,
      "logps/rejected": -0.8317993879318237,
      "loss": 42.0429,
      "nll_loss": 0.932290256023407,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.4287196099758148,
      "rewards/margins": -0.0128199253231287,
      "rewards/rejected": -0.41589969396591187,
      "step": 250
    },
    {
      "epoch": 2.407703703703704,
      "grad_norm": 37.83774185180664,
      "learning_rate": 5.343475104027743e-06,
      "log_odds_chosen": 0.17812205851078033,
      "log_odds_ratio": -0.7299228310585022,
      "logits/chosen": 155.40672302246094,
      "logits/rejected": 160.87484741210938,
      "logps/chosen": -0.8623374104499817,
      "logps/rejected": -0.9512872695922852,
      "loss": 42.1891,
      "nll_loss": 0.9383618235588074,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.43116870522499084,
      "rewards/margins": 0.044474903494119644,
      "rewards/rejected": -0.4756436347961426,
      "step": 255
    },
    {
      "epoch": 2.455111111111111,
      "grad_norm": 34.86994934082031,
      "learning_rate": 4.516778136213037e-06,
      "log_odds_chosen": 0.06448222696781158,
      "log_odds_ratio": -0.7623555660247803,
      "logits/chosen": 159.42433166503906,
      "logits/rejected": 155.6562957763672,
      "logps/chosen": -0.853624701499939,
      "logps/rejected": -0.8752864003181458,
      "loss": 41.3101,
      "nll_loss": 0.8972175717353821,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.4268123507499695,
      "rewards/margins": 0.010830795392394066,
      "rewards/rejected": -0.4376432001590729,
      "step": 260
    },
    {
      "epoch": 2.5025185185185186,
      "grad_norm": 40.81254577636719,
      "learning_rate": 3.7531701693965554e-06,
      "log_odds_chosen": 0.0020365030504763126,
      "log_odds_ratio": -0.7836844325065613,
      "logits/chosen": 152.5310516357422,
      "logits/rejected": 162.28890991210938,
      "logps/chosen": -0.904189944267273,
      "logps/rejected": -0.8998059034347534,
      "loss": 41.5474,
      "nll_loss": 0.9223392605781555,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.4520949721336365,
      "rewards/margins": -0.002192039042711258,
      "rewards/rejected": -0.4499029517173767,
      "step": 265
    },
    {
      "epoch": 2.549925925925926,
      "grad_norm": 35.19020080566406,
      "learning_rate": 3.055003141378948e-06,
      "log_odds_chosen": 0.08025786280632019,
      "log_odds_ratio": -0.7740025520324707,
      "logits/chosen": 152.10177612304688,
      "logits/rejected": 160.26998901367188,
      "logps/chosen": -0.8461346626281738,
      "logps/rejected": -0.8950198888778687,
      "loss": 41.2177,
      "nll_loss": 0.8989561200141907,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.4230673313140869,
      "rewards/margins": 0.02444261871278286,
      "rewards/rejected": -0.4475099444389343,
      "step": 270
    },
    {
      "epoch": 2.5973333333333333,
      "grad_norm": 39.02735900878906,
      "learning_rate": 2.424427429704365e-06,
      "log_odds_chosen": 0.061826206743717194,
      "log_odds_ratio": -0.7568526268005371,
      "logits/chosen": 161.21261596679688,
      "logits/rejected": 161.19491577148438,
      "logps/chosen": -0.8993644714355469,
      "logps/rejected": -0.903190016746521,
      "loss": 41.1732,
      "nll_loss": 0.8910198211669922,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.44968223571777344,
      "rewards/margins": 0.0019127646228298545,
      "rewards/rejected": -0.4515950083732605,
      "step": 275
    },
    {
      "epoch": 2.644740740740741,
      "grad_norm": 39.4185676574707,
      "learning_rate": 1.8633852284264508e-06,
      "log_odds_chosen": 0.02886720933020115,
      "log_odds_ratio": -0.8128382563591003,
      "logits/chosen": 153.80528259277344,
      "logits/rejected": 150.25201416015625,
      "logps/chosen": -0.8430732488632202,
      "logps/rejected": -0.8419806361198425,
      "loss": 41.047,
      "nll_loss": 0.9150816202163696,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.4215366244316101,
      "rewards/margins": -0.0005463186535052955,
      "rewards/rejected": -0.42099031805992126,
      "step": 280
    },
    {
      "epoch": 2.6921481481481484,
      "grad_norm": 50.44974899291992,
      "learning_rate": 1.3736045660864034e-06,
      "log_odds_chosen": -0.17111532390117645,
      "log_odds_ratio": -0.9221655130386353,
      "logits/chosen": 152.35177612304688,
      "logits/rejected": 156.8704376220703,
      "logps/chosen": -1.0376865863800049,
      "logps/rejected": -0.9074798822402954,
      "loss": 42.497,
      "nll_loss": 0.9657168388366699,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.5188432931900024,
      "rewards/margins": -0.06510341912508011,
      "rewards/rejected": -0.4537399411201477,
      "step": 285
    },
    {
      "epoch": 2.7395555555555555,
      "grad_norm": 38.56003952026367,
      "learning_rate": 9.565939833279192e-07,
      "log_odds_chosen": -0.0652594044804573,
      "log_odds_ratio": -0.8579480051994324,
      "logits/chosen": 155.33377075195312,
      "logits/rejected": 156.094482421875,
      "logps/chosen": -0.9537650942802429,
      "logps/rejected": -0.8857321739196777,
      "loss": 41.9732,
      "nll_loss": 0.9463297724723816,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.47688254714012146,
      "rewards/margins": -0.034016452729701996,
      "rewards/rejected": -0.44286608695983887,
      "step": 290
    },
    {
      "epoch": 2.786962962962963,
      "grad_norm": 35.984092712402344,
      "learning_rate": 6.136378865420872e-07,
      "log_odds_chosen": -0.05488128587603569,
      "log_odds_ratio": -0.8374530076980591,
      "logits/chosen": 159.79251098632812,
      "logits/rejected": 153.42626953125,
      "logps/chosen": -0.8930980563163757,
      "logps/rejected": -0.848624587059021,
      "loss": 42.7359,
      "nll_loss": 0.9567394256591797,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.44654902815818787,
      "rewards/margins": -0.02223675511777401,
      "rewards/rejected": -0.4243122935295105,
      "step": 295
    },
    {
      "epoch": 2.83437037037037,
      "grad_norm": 40.65980911254883,
      "learning_rate": 3.45792591853214e-07,
      "log_odds_chosen": 0.03835665062069893,
      "log_odds_ratio": -0.7734814286231995,
      "logits/chosen": 157.85092163085938,
      "logits/rejected": 154.9134979248047,
      "logps/chosen": -0.8622520565986633,
      "logps/rejected": -0.8551470637321472,
      "loss": 41.1462,
      "nll_loss": 0.9184630513191223,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.43112602829933167,
      "rewards/margins": -0.0035525336861610413,
      "rewards/rejected": -0.4275735318660736,
      "step": 300
    },
    {
      "epoch": 2.8817777777777778,
      "grad_norm": 45.03302001953125,
      "learning_rate": 1.538830716302092e-07,
      "log_odds_chosen": 0.0824732631444931,
      "log_odds_ratio": -0.7868438959121704,
      "logits/chosen": 152.17709350585938,
      "logits/rejected": 160.5044403076172,
      "logps/chosen": -0.8794566988945007,
      "logps/rejected": -0.887550950050354,
      "loss": 40.8093,
      "nll_loss": 0.9352946281433105,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.43972834944725037,
      "rewards/margins": 0.004047115799039602,
      "rewards/rejected": -0.443775475025177,
      "step": 305
    },
    {
      "epoch": 2.9291851851851853,
      "grad_norm": 41.13324737548828,
      "learning_rate": 3.8500413544415025e-08,
      "log_odds_chosen": 0.036232512444257736,
      "log_odds_ratio": -0.7697147727012634,
      "logits/chosen": 153.0935821533203,
      "logits/rejected": 152.86441040039062,
      "logps/chosen": -0.8729937672615051,
      "logps/rejected": -0.8775345087051392,
      "loss": 41.2786,
      "nll_loss": 0.9117900133132935,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.43649688363075256,
      "rewards/margins": 0.002270397264510393,
      "rewards/rejected": -0.4387672543525696,
      "step": 310
    },
    {
      "epoch": 2.9765925925925925,
      "grad_norm": 116.33460235595703,
      "learning_rate": 0.0,
      "log_odds_chosen": 0.13965412974357605,
      "log_odds_ratio": -0.712945282459259,
      "logits/chosen": 145.65011596679688,
      "logits/rejected": 152.03152465820312,
      "logps/chosen": -0.739302933216095,
      "logps/rejected": -0.7891243696212769,
      "loss": 39.5762,
      "nll_loss": 0.8406442403793335,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.3696514666080475,
      "rewards/margins": 0.024910712614655495,
      "rewards/rejected": -0.3945621848106384,
      "step": 315
    },
    {
      "epoch": 2.9765925925925925,
      "step": 315,
      "total_flos": 0.0,
      "train_loss": 71.97097947862413,
      "train_runtime": 8116.2448,
      "train_samples_per_second": 2.495,
      "train_steps_per_second": 0.039
    }
  ],
  "logging_steps": 5,
  "max_steps": 315,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 100000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}