{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 553,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.018083182640144666,
      "grad_norm": 270.0,
      "learning_rate": 5.000000000000001e-07,
      "log_odds_chosen": 0.524066150188446,
      "log_odds_ratio": -0.5735815763473511,
      "logits/chosen": -3.028451442718506,
      "logits/rejected": -3.2061283588409424,
      "logps/chosen": -1.6356823444366455,
      "logps/rejected": -2.097788095474243,
      "loss": 5.9583,
      "nll_loss": 6.051669120788574,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.08178411424160004,
      "rewards/margins": 0.02310529164969921,
      "rewards/rejected": -0.1048894077539444,
      "step": 10
    },
    {
      "epoch": 0.03616636528028933,
      "grad_norm": 338.0,
      "learning_rate": 1.0000000000000002e-06,
      "log_odds_chosen": 0.5587388873100281,
      "log_odds_ratio": -0.5176494121551514,
      "logits/chosen": -2.638868808746338,
      "logits/rejected": -2.6223673820495605,
      "logps/chosen": -1.23783278465271,
      "logps/rejected": -1.646284818649292,
      "loss": 5.6814,
      "nll_loss": 3.5105323791503906,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.061891645193099976,
      "rewards/margins": 0.020422598347067833,
      "rewards/rejected": -0.08231424540281296,
      "step": 20
    },
    {
      "epoch": 0.054249547920433995,
      "grad_norm": 197.0,
      "learning_rate": 1.5e-06,
      "log_odds_chosen": -0.7765415906906128,
      "log_odds_ratio": -1.4850478172302246,
      "logits/chosen": -2.554340362548828,
      "logits/rejected": -3.0053932666778564,
      "logps/chosen": -2.7067830562591553,
      "logps/rejected": -1.897303819656372,
      "loss": 4.095,
      "nll_loss": 5.7828497886657715,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.1353391706943512,
      "rewards/margins": -0.04047396779060364,
      "rewards/rejected": -0.09486518800258636,
      "step": 30
    },
    {
      "epoch": 0.07233273056057866,
      "grad_norm": 64.0,
      "learning_rate": 2.0000000000000003e-06,
      "log_odds_chosen": 1.3007441759109497,
      "log_odds_ratio": -0.4158785343170166,
      "logits/chosen": -2.897387981414795,
      "logits/rejected": -3.0413200855255127,
      "logps/chosen": -1.3996171951293945,
      "logps/rejected": -2.517918109893799,
      "loss": 2.6986,
      "nll_loss": 2.556391954421997,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06998086720705032,
      "rewards/margins": 0.05591505020856857,
      "rewards/rejected": -0.1258959025144577,
      "step": 40
    },
    {
      "epoch": 0.09041591320072333,
      "grad_norm": 21.875,
      "learning_rate": 2.5e-06,
      "log_odds_chosen": 0.607527494430542,
      "log_odds_ratio": -0.464752733707428,
      "logits/chosen": -3.3353819847106934,
      "logits/rejected": -3.417086124420166,
      "logps/chosen": -1.5169802904129028,
      "logps/rejected": -2.032203197479248,
      "loss": 1.9263,
      "nll_loss": 1.4955819845199585,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0758490115404129,
      "rewards/margins": 0.025761136785149574,
      "rewards/rejected": -0.10161016136407852,
      "step": 50
    },
    {
      "epoch": 0.10849909584086799,
      "grad_norm": 15.25,
      "learning_rate": 3e-06,
      "log_odds_chosen": 0.3995138704776764,
      "log_odds_ratio": -0.6033788919448853,
      "logits/chosen": -3.0420987606048584,
      "logits/rejected": -3.2451672554016113,
      "logps/chosen": -1.6125237941741943,
      "logps/rejected": -1.9586118459701538,
      "loss": 1.7326,
      "nll_loss": 2.063459634780884,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.08062619715929031,
      "rewards/margins": 0.017304399982094765,
      "rewards/rejected": -0.09793059527873993,
      "step": 60
    },
    {
      "epoch": 0.12658227848101267,
      "grad_norm": 8.5625,
      "learning_rate": 3.5e-06,
      "log_odds_chosen": 0.3118763566017151,
      "log_odds_ratio": -0.5856896638870239,
      "logits/chosen": -3.4981963634490967,
      "logits/rejected": -3.5791831016540527,
      "logps/chosen": -1.4380217790603638,
      "logps/rejected": -1.686657190322876,
      "loss": 1.4005,
      "nll_loss": 1.4499210119247437,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07190109044313431,
      "rewards/margins": 0.012431769631803036,
      "rewards/rejected": -0.08433286100625992,
      "step": 70
    },
    {
      "epoch": 0.14466546112115733,
      "grad_norm": 7.28125,
      "learning_rate": 4.000000000000001e-06,
      "log_odds_chosen": 0.6805863976478577,
      "log_odds_ratio": -0.5004494190216064,
      "logits/chosen": -3.5435702800750732,
      "logits/rejected": -3.5884978771209717,
      "logps/chosen": -1.284294843673706,
      "logps/rejected": -1.8150221109390259,
      "loss": 1.3307,
      "nll_loss": 1.4621249437332153,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.06421474367380142,
      "rewards/margins": 0.026536360383033752,
      "rewards/rejected": -0.09075109660625458,
      "step": 80
    },
    {
      "epoch": 0.162748643761302,
      "grad_norm": 6.6875,
      "learning_rate": 4.5e-06,
      "log_odds_chosen": 0.7451115846633911,
      "log_odds_ratio": -0.7124490141868591,
      "logits/chosen": -3.6752638816833496,
      "logits/rejected": -3.662033796310425,
      "logps/chosen": -1.0480401515960693,
      "logps/rejected": -1.207778811454773,
      "loss": 1.2034,
      "nll_loss": 1.1693445444107056,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.05240201950073242,
      "rewards/margins": 0.007986925542354584,
      "rewards/rejected": -0.06038894131779671,
      "step": 90
    },
    {
      "epoch": 0.18083182640144665,
      "grad_norm": 5.625,
      "learning_rate": 5e-06,
      "log_odds_chosen": 1.4625422954559326,
      "log_odds_ratio": -0.2866816222667694,
      "logits/chosen": -3.5862960815429688,
      "logits/rejected": -3.631056547164917,
      "logps/chosen": -0.616925060749054,
      "logps/rejected": -1.2389600276947021,
      "loss": 1.0472,
      "nll_loss": 0.997981071472168,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.030846258625388145,
      "rewards/margins": 0.03110174462199211,
      "rewards/rejected": -0.06194800138473511,
      "step": 100
    },
    {
      "epoch": 0.19891500904159132,
      "grad_norm": 4.34375,
      "learning_rate": 4.767312946227961e-06,
      "log_odds_chosen": 0.32260584831237793,
      "log_odds_ratio": -0.7831681966781616,
      "logits/chosen": -3.3512356281280518,
      "logits/rejected": -3.317343235015869,
      "logps/chosen": -0.8152042627334595,
      "logps/rejected": -1.023037314414978,
      "loss": 1.021,
      "nll_loss": 0.8699595332145691,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.040760211646556854,
      "rewards/margins": 0.010391652584075928,
      "rewards/rejected": -0.05115186423063278,
      "step": 110
    },
    {
      "epoch": 0.21699819168173598,
      "grad_norm": 4.46875,
      "learning_rate": 4.564354645876385e-06,
      "log_odds_chosen": 0.06312495470046997,
      "log_odds_ratio": -0.7186229228973389,
      "logits/chosen": -3.7797393798828125,
      "logits/rejected": -3.610599994659424,
      "logps/chosen": -1.342027187347412,
      "logps/rejected": -1.3699142932891846,
      "loss": 1.2306,
      "nll_loss": 1.2616217136383057,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0671013593673706,
      "rewards/margins": 0.0013943525264039636,
      "rewards/rejected": -0.06849571317434311,
      "step": 120
    },
    {
      "epoch": 0.23508137432188064,
      "grad_norm": 3.40625,
      "learning_rate": 4.385290096535147e-06,
      "log_odds_chosen": 0.3295042812824249,
      "log_odds_ratio": -0.6021639704704285,
      "logits/chosen": -3.6531271934509277,
      "logits/rejected": -3.6605091094970703,
      "logps/chosen": -1.1059828996658325,
      "logps/rejected": -1.2798744440078735,
      "loss": 1.0672,
      "nll_loss": 1.0846788883209229,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.05529915168881416,
      "rewards/margins": 0.008694576099514961,
      "rewards/rejected": -0.06399371474981308,
      "step": 130
    },
    {
      "epoch": 0.25316455696202533,
      "grad_norm": 3.921875,
      "learning_rate": 4.2257712736425835e-06,
      "log_odds_chosen": 0.005866944789886475,
      "log_odds_ratio": -0.7822607755661011,
      "logits/chosen": -3.4610767364501953,
      "logits/rejected": -3.5588440895080566,
      "logps/chosen": -1.0730628967285156,
      "logps/rejected": -1.1153419017791748,
      "loss": 1.0629,
      "nll_loss": 0.9272462725639343,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.05365313962101936,
      "rewards/margins": 0.0021139546297490597,
      "rewards/rejected": -0.05576709657907486,
      "step": 140
    },
    {
      "epoch": 0.27124773960216997,
      "grad_norm": 4.5625,
      "learning_rate": 4.082482904638631e-06,
      "log_odds_chosen": 0.42503032088279724,
      "log_odds_ratio": -0.8444482684135437,
      "logits/chosen": -3.4901211261749268,
      "logits/rejected": -3.6267387866973877,
      "logps/chosen": -0.8993390798568726,
      "logps/rejected": -1.1696584224700928,
      "loss": 1.0394,
      "nll_loss": 0.797405481338501,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.044966958463191986,
      "rewards/margins": 0.013515966013073921,
      "rewards/rejected": -0.05848292261362076,
      "step": 150
    },
    {
      "epoch": 0.28933092224231466,
      "grad_norm": 4.75,
      "learning_rate": 3.952847075210474e-06,
      "log_odds_chosen": 0.5311036109924316,
      "log_odds_ratio": -0.5203253030776978,
      "logits/chosen": -3.2638163566589355,
      "logits/rejected": -3.4405035972595215,
      "logps/chosen": -1.0335638523101807,
      "logps/rejected": -1.3805434703826904,
      "loss": 1.1398,
      "nll_loss": 1.0082361698150635,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.051678191870450974,
      "rewards/margins": 0.017348987981677055,
      "rewards/rejected": -0.06902717798948288,
      "step": 160
    },
    {
      "epoch": 0.3074141048824593,
      "grad_norm": 4.125,
      "learning_rate": 3.834824944236852e-06,
      "log_odds_chosen": 0.7853885293006897,
      "log_odds_ratio": -0.5176795721054077,
      "logits/chosen": -3.390453815460205,
      "logits/rejected": -3.4858481884002686,
      "logps/chosen": -1.0219676494598389,
      "logps/rejected": -1.3882640600204468,
      "loss": 1.0934,
      "nll_loss": 0.9619676470756531,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.05109838768839836,
      "rewards/margins": 0.018314816057682037,
      "rewards/rejected": -0.0694132000207901,
      "step": 170
    },
    {
      "epoch": 0.325497287522604,
      "grad_norm": 3.3125,
      "learning_rate": 3.72677996249965e-06,
      "log_odds_chosen": 0.2512182593345642,
      "log_odds_ratio": -0.6799607276916504,
      "logits/chosen": -3.8153457641601562,
      "logits/rejected": -3.849152088165283,
      "logps/chosen": -1.1906697750091553,
      "logps/rejected": -1.386765718460083,
      "loss": 1.0226,
      "nll_loss": 1.331679105758667,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.059533488005399704,
      "rewards/margins": 0.009804805740714073,
      "rewards/rejected": -0.06933829933404922,
      "step": 180
    },
    {
      "epoch": 0.3435804701627486,
      "grad_norm": 6.5625,
      "learning_rate": 3.6273812505500587e-06,
      "log_odds_chosen": 0.3922955393791199,
      "log_odds_ratio": -0.5585602521896362,
      "logits/chosen": -3.6088531017303467,
      "logits/rejected": -3.6289172172546387,
      "logps/chosen": -1.0057111978530884,
      "logps/rejected": -1.2608778476715088,
      "loss": 1.1235,
      "nll_loss": 0.9645823240280151,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.05028555914759636,
      "rewards/margins": 0.012758332304656506,
      "rewards/rejected": -0.06304388493299484,
      "step": 190
    },
    {
      "epoch": 0.3616636528028933,
      "grad_norm": 4.71875,
      "learning_rate": 3.5355339059327378e-06,
      "log_odds_chosen": 0.0035676597617566586,
      "log_odds_ratio": -0.7242649793624878,
      "logits/chosen": -3.624096632003784,
      "logits/rejected": -3.6793160438537598,
      "logps/chosen": -1.1755887269973755,
      "logps/rejected": -1.200249195098877,
      "loss": 1.0192,
      "nll_loss": 0.9476919174194336,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.05877942964434624,
      "rewards/margins": 0.0012330322060734034,
      "rewards/rejected": -0.06001245975494385,
      "step": 200
    },
    {
      "epoch": 0.379746835443038,
      "grad_norm": 3.5625,
      "learning_rate": 3.450327796711771e-06,
      "log_odds_chosen": 0.7048985362052917,
      "log_odds_ratio": -0.6235366463661194,
      "logits/chosen": -3.4578773975372314,
      "logits/rejected": -3.6635658740997314,
      "logps/chosen": -0.9914987683296204,
      "logps/rejected": -1.1977266073226929,
      "loss": 1.0038,
      "nll_loss": 0.8347546458244324,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04957494139671326,
      "rewards/margins": 0.010311387479305267,
      "rewards/rejected": -0.059886325150728226,
      "step": 210
    },
    {
      "epoch": 0.39783001808318263,
      "grad_norm": 3.9375,
      "learning_rate": 3.3709993123162106e-06,
      "log_odds_chosen": -0.7274879217147827,
      "log_odds_ratio": -1.5150617361068726,
      "logits/chosen": -3.378265857696533,
      "logits/rejected": -3.5164248943328857,
      "logps/chosen": -1.3154796361923218,
      "logps/rejected": -1.1096127033233643,
      "loss": 1.0616,
      "nll_loss": 0.9280896186828613,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.06577398627996445,
      "rewards/margins": -0.010293351486325264,
      "rewards/rejected": -0.055480629205703735,
      "step": 220
    },
    {
      "epoch": 0.4159132007233273,
      "grad_norm": 4.96875,
      "learning_rate": 3.296902366978936e-06,
      "log_odds_chosen": -0.029178833588957787,
      "log_odds_ratio": -0.7706394195556641,
      "logits/chosen": -3.5950729846954346,
      "logits/rejected": -3.6548149585723877,
      "logps/chosen": -1.2986187934875488,
      "logps/rejected": -1.2398769855499268,
      "loss": 1.0526,
      "nll_loss": 1.0869041681289673,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.06493094563484192,
      "rewards/margins": -0.0029370952397584915,
      "rewards/rejected": -0.06199384853243828,
      "step": 230
    },
    {
      "epoch": 0.43399638336347196,
      "grad_norm": 4.21875,
      "learning_rate": 3.2274861218395142e-06,
      "log_odds_chosen": 0.16786038875579834,
      "log_odds_ratio": -0.6685726046562195,
      "logits/chosen": -3.688838243484497,
      "logits/rejected": -3.749654769897461,
      "logps/chosen": -1.374859094619751,
      "logps/rejected": -1.5094839334487915,
      "loss": 1.0482,
      "nll_loss": 1.1397324800491333,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06874295324087143,
      "rewards/margins": 0.0067312465980648994,
      "rewards/rejected": -0.07547420263290405,
      "step": 240
    },
    {
      "epoch": 0.45207956600361665,
      "grad_norm": 4.5,
      "learning_rate": 3.1622776601683796e-06,
      "log_odds_chosen": 0.8509995341300964,
      "log_odds_ratio": -0.5062636137008667,
      "logits/chosen": -3.566221237182617,
      "logits/rejected": -3.6127326488494873,
      "logps/chosen": -0.9691341519355774,
      "logps/rejected": -1.377387285232544,
      "loss": 1.0446,
      "nll_loss": 0.9793401956558228,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.04845671355724335,
      "rewards/margins": 0.02041265368461609,
      "rewards/rejected": -0.06886936724185944,
      "step": 250
    },
    {
      "epoch": 0.4701627486437613,
      "grad_norm": 2.640625,
      "learning_rate": 3.1008683647302113e-06,
      "log_odds_chosen": 0.08310860395431519,
      "log_odds_ratio": -0.720168948173523,
      "logits/chosen": -3.4250590801239014,
      "logits/rejected": -3.513597011566162,
      "logps/chosen": -1.2283875942230225,
      "logps/rejected": -1.3563426733016968,
      "loss": 1.045,
      "nll_loss": 1.0739576816558838,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06141937896609306,
      "rewards/margins": 0.006397762335836887,
      "rewards/rejected": -0.06781713664531708,
      "step": 260
    },
    {
      "epoch": 0.488245931283906,
      "grad_norm": 3.734375,
      "learning_rate": 3.0429030972509227e-06,
      "log_odds_chosen": 0.2674502432346344,
      "log_odds_ratio": -0.6830952763557434,
      "logits/chosen": -3.3503127098083496,
      "logits/rejected": -3.4934239387512207,
      "logps/chosen": -1.0327961444854736,
      "logps/rejected": -1.2308813333511353,
      "loss": 1.0153,
      "nll_loss": 1.1966408491134644,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.05163981392979622,
      "rewards/margins": 0.009904252365231514,
      "rewards/rejected": -0.061544060707092285,
      "step": 270
    },
    {
      "epoch": 0.5063291139240507,
      "grad_norm": 7.75,
      "learning_rate": 2.988071523335984e-06,
      "log_odds_chosen": 0.9675368070602417,
      "log_odds_ratio": -0.4681246876716614,
      "logits/chosen": -2.9768073558807373,
      "logits/rejected": -3.2737374305725098,
      "logps/chosen": -0.9432920217514038,
      "logps/rejected": -1.5204970836639404,
      "loss": 1.0435,
      "nll_loss": 0.8414273262023926,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.047164615243673325,
      "rewards/margins": 0.02886023558676243,
      "rewards/rejected": -0.07602483779191971,
      "step": 280
    },
    {
      "epoch": 0.5244122965641953,
      "grad_norm": 4.3125,
      "learning_rate": 2.9361010975735177e-06,
      "log_odds_chosen": 0.782937228679657,
      "log_odds_ratio": -0.5052694082260132,
      "logits/chosen": -3.4432454109191895,
      "logits/rejected": -3.513373851776123,
      "logps/chosen": -0.959903359413147,
      "logps/rejected": -1.3499042987823486,
      "loss": 1.034,
      "nll_loss": 0.9008768200874329,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.04799516871571541,
      "rewards/margins": 0.019500048831105232,
      "rewards/rejected": -0.06749521195888519,
      "step": 290
    },
    {
      "epoch": 0.5424954792043399,
      "grad_norm": 3.328125,
      "learning_rate": 2.8867513459481293e-06,
      "log_odds_chosen": -0.06844829022884369,
      "log_odds_ratio": -0.8229748606681824,
      "logits/chosen": -3.6102004051208496,
      "logits/rejected": -3.6592490673065186,
      "logps/chosen": -1.3216373920440674,
      "logps/rejected": -1.3158751726150513,
      "loss": 1.0616,
      "nll_loss": 1.050123929977417,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.06608187407255173,
      "rewards/margins": -0.00028811878291890025,
      "rewards/rejected": -0.06579376012086868,
      "step": 300
    },
    {
      "epoch": 0.5605786618444847,
      "grad_norm": 3.796875,
      "learning_rate": 2.839809171235324e-06,
      "log_odds_chosen": 0.5723578333854675,
      "log_odds_ratio": -0.5454049110412598,
      "logits/chosen": -3.4600486755371094,
      "logits/rejected": -3.52336049079895,
      "logps/chosen": -0.9334306716918945,
      "logps/rejected": -1.1987031698226929,
      "loss": 1.0222,
      "nll_loss": 0.9817255735397339,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.046671535819768906,
      "rewards/margins": 0.013263625092804432,
      "rewards/rejected": -0.05993515998125076,
      "step": 310
    },
    {
      "epoch": 0.5786618444846293,
      "grad_norm": 3.53125,
      "learning_rate": 2.7950849718747376e-06,
      "log_odds_chosen": 0.8614484667778015,
      "log_odds_ratio": -0.4398893713951111,
      "logits/chosen": -3.6512210369110107,
      "logits/rejected": -3.76987886428833,
      "logps/chosen": -0.744703471660614,
      "logps/rejected": -1.220558762550354,
      "loss": 1.0317,
      "nll_loss": 0.9681469202041626,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.03723517805337906,
      "rewards/margins": 0.02379276230931282,
      "rewards/rejected": -0.06102793663740158,
      "step": 320
    },
    {
      "epoch": 0.596745027124774,
      "grad_norm": 3.703125,
      "learning_rate": 2.752409412815902e-06,
      "log_odds_chosen": 0.5696262121200562,
      "log_odds_ratio": -0.8096323013305664,
      "logits/chosen": -3.248673915863037,
      "logits/rejected": -3.3223063945770264,
      "logps/chosen": -1.0554739236831665,
      "logps/rejected": -1.1397294998168945,
      "loss": 1.0938,
      "nll_loss": 1.0340250730514526,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.052773695439100266,
      "rewards/margins": 0.004212772473692894,
      "rewards/rejected": -0.05698646977543831,
      "step": 330
    },
    {
      "epoch": 0.6148282097649186,
      "grad_norm": 4.46875,
      "learning_rate": 2.711630722733202e-06,
      "log_odds_chosen": 0.45444560050964355,
      "log_odds_ratio": -0.556340754032135,
      "logits/chosen": -3.499669313430786,
      "logits/rejected": -3.5566048622131348,
      "logps/chosen": -1.0938056707382202,
      "logps/rejected": -1.4399141073226929,
      "loss": 1.0092,
      "nll_loss": 0.9630126953125,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.05469028279185295,
      "rewards/margins": 0.01730542816221714,
      "rewards/rejected": -0.07199571281671524,
      "step": 340
    },
    {
      "epoch": 0.6329113924050633,
      "grad_norm": 5.03125,
      "learning_rate": 2.6726124191242444e-06,
      "log_odds_chosen": -0.18927308917045593,
      "log_odds_ratio": -1.1298543214797974,
      "logits/chosen": -3.469010829925537,
      "logits/rejected": -3.5768990516662598,
      "logps/chosen": -1.1069543361663818,
      "logps/rejected": -1.25455904006958,
      "loss": 1.035,
      "nll_loss": 1.04062819480896,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.055347710847854614,
      "rewards/margins": 0.007380232214927673,
      "rewards/rejected": -0.06272794306278229,
      "step": 350
    },
    {
      "epoch": 0.650994575045208,
      "grad_norm": 4.125,
      "learning_rate": 2.6352313834736496e-06,
      "log_odds_chosen": 1.1780674457550049,
      "log_odds_ratio": -0.458904892206192,
      "logits/chosen": -3.475041151046753,
      "logits/rejected": -3.641082763671875,
      "logps/chosen": -0.7767281532287598,
      "logps/rejected": -1.3540589809417725,
      "loss": 1.0549,
      "nll_loss": 0.994460940361023,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.03883640468120575,
      "rewards/margins": 0.028866535052657127,
      "rewards/rejected": -0.06770294159650803,
      "step": 360
    },
    {
      "epoch": 0.6690777576853526,
      "grad_norm": 4.03125,
      "learning_rate": 2.599376224550182e-06,
      "log_odds_chosen": -0.05644133687019348,
      "log_odds_ratio": -0.7978056073188782,
      "logits/chosen": -3.524318218231201,
      "logits/rejected": -3.6522598266601562,
      "logps/chosen": -1.1413499116897583,
      "logps/rejected": -1.1321113109588623,
      "loss": 0.9712,
      "nll_loss": 0.9401092529296875,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.05706749111413956,
      "rewards/margins": -0.00046192388981580734,
      "rewards/rejected": -0.05660557001829147,
      "step": 370
    },
    {
      "epoch": 0.6871609403254972,
      "grad_norm": 5.4375,
      "learning_rate": 2.564945880212886e-06,
      "log_odds_chosen": -0.03757842257618904,
      "log_odds_ratio": -0.9225370287895203,
      "logits/chosen": -3.3350226879119873,
      "logits/rejected": -3.4671382904052734,
      "logps/chosen": -1.4029977321624756,
      "logps/rejected": -1.209831714630127,
      "loss": 1.0249,
      "nll_loss": 1.2321926355361938,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.07014988362789154,
      "rewards/margins": -0.009658301249146461,
      "rewards/rejected": -0.06049158424139023,
      "step": 380
    },
    {
      "epoch": 0.705244122965642,
      "grad_norm": 3.453125,
      "learning_rate": 2.5318484177091667e-06,
      "log_odds_chosen": 0.38905367255210876,
      "log_odds_ratio": -0.6244643330574036,
      "logits/chosen": -3.652184009552002,
      "logits/rejected": -3.6555867195129395,
      "logps/chosen": -1.316056728363037,
      "logps/rejected": -1.5403589010238647,
      "loss": 0.9824,
      "nll_loss": 1.1117520332336426,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06580284237861633,
      "rewards/margins": 0.01121511310338974,
      "rewards/rejected": -0.07701796293258667,
      "step": 390
    },
    {
      "epoch": 0.7233273056057866,
      "grad_norm": 3.0,
      "learning_rate": 2.5e-06,
      "log_odds_chosen": 0.23738713562488556,
      "log_odds_ratio": -0.6204290390014648,
      "logits/chosen": -3.3568973541259766,
      "logits/rejected": -3.591095447540283,
      "logps/chosen": -1.1707801818847656,
      "logps/rejected": -1.3350715637207031,
      "loss": 1.071,
      "nll_loss": 1.0477981567382812,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0585390105843544,
      "rewards/margins": 0.008214573375880718,
      "rewards/rejected": -0.0667535811662674,
      "step": 400
    },
    {
      "epoch": 0.7414104882459313,
      "grad_norm": 3.5625,
      "learning_rate": 2.4693239916239746e-06,
      "log_odds_chosen": 0.060622893273830414,
      "log_odds_ratio": -0.7360556721687317,
      "logits/chosen": -3.5519211292266846,
      "logits/rejected": -3.641356945037842,
      "logps/chosen": -1.2109748125076294,
      "logps/rejected": -1.2431539297103882,
      "loss": 1.0531,
      "nll_loss": 1.063460111618042,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.06054873391985893,
      "rewards/margins": 0.0016089621931314468,
      "rewards/rejected": -0.06215769797563553,
      "step": 410
    },
    {
      "epoch": 0.759493670886076,
      "grad_norm": 3.515625,
      "learning_rate": 2.4397501823713327e-06,
      "log_odds_chosen": 0.27192121744155884,
      "log_odds_ratio": -0.7382515668869019,
      "logits/chosen": -3.4203600883483887,
      "logits/rejected": -3.5196239948272705,
      "logps/chosen": -0.8597354888916016,
      "logps/rejected": -1.0601986646652222,
      "loss": 1.0227,
      "nll_loss": 0.8366381525993347,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04298677295446396,
      "rewards/margins": 0.010023154318332672,
      "rewards/rejected": -0.05300993472337723,
      "step": 420
    },
    {
      "epoch": 0.7775768535262206,
      "grad_norm": 4.6875,
      "learning_rate": 2.411214110852061e-06,
      "log_odds_chosen": 0.00424801092594862,
      "log_odds_ratio": -0.7706916332244873,
      "logits/chosen": -3.656468629837036,
      "logits/rejected": -3.656891345977783,
      "logps/chosen": -1.2077748775482178,
      "logps/rejected": -1.210986852645874,
      "loss": 0.9992,
      "nll_loss": 1.0321767330169678,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.06038873642683029,
      "rewards/margins": 0.00016060545749496669,
      "rewards/rejected": -0.06054934114217758,
      "step": 430
    },
    {
      "epoch": 0.7956600361663653,
      "grad_norm": 3.203125,
      "learning_rate": 2.3836564731139807e-06,
      "log_odds_chosen": 0.15838010609149933,
      "log_odds_ratio": -0.706937849521637,
      "logits/chosen": -3.6181061267852783,
      "logits/rejected": -3.649228572845459,
      "logps/chosen": -1.0764718055725098,
      "logps/rejected": -1.1391253471374512,
      "loss": 1.0074,
      "nll_loss": 1.0357991456985474,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.05382359027862549,
      "rewards/margins": 0.003132682293653488,
      "rewards/rejected": -0.05695626884698868,
      "step": 440
    },
    {
      "epoch": 0.8137432188065099,
      "grad_norm": 3.609375,
      "learning_rate": 2.357022603955159e-06,
      "log_odds_chosen": 0.6918424367904663,
      "log_odds_ratio": -0.5028156042098999,
      "logits/chosen": -3.3800606727600098,
      "logits/rejected": -3.511284351348877,
      "logps/chosen": -0.7713460326194763,
      "logps/rejected": -1.1150473356246948,
      "loss": 1.0192,
      "nll_loss": 0.9132420420646667,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.038567304611206055,
      "rewards/margins": 0.017185065895318985,
      "rewards/rejected": -0.05575236678123474,
      "step": 450
    },
    {
      "epoch": 0.8318264014466547,
      "grad_norm": 4.46875,
      "learning_rate": 2.3312620206007847e-06,
      "log_odds_chosen": 0.9962455630302429,
      "log_odds_ratio": -0.4283738136291504,
      "logits/chosen": -3.4526374340057373,
      "logits/rejected": -3.6031575202941895,
      "logps/chosen": -0.7248684167861938,
      "logps/rejected": -1.235801100730896,
      "loss": 0.9926,
      "nll_loss": 0.7733514308929443,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.03624342009425163,
      "rewards/margins": 0.025546634569764137,
      "rewards/rejected": -0.06179005652666092,
      "step": 460
    },
    {
      "epoch": 0.8499095840867993,
      "grad_norm": 4.75,
      "learning_rate": 2.3063280200722128e-06,
      "log_odds_chosen": 0.4537855088710785,
      "log_odds_ratio": -0.5337843894958496,
      "logits/chosen": -3.5173232555389404,
      "logits/rejected": -3.6324150562286377,
      "logps/chosen": -0.9342852830886841,
      "logps/rejected": -1.2018686532974243,
      "loss": 1.0222,
      "nll_loss": 0.8243517875671387,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.04671426862478256,
      "rewards/margins": 0.013379162177443504,
      "rewards/rejected": -0.06009342521429062,
      "step": 470
    },
    {
      "epoch": 0.8679927667269439,
      "grad_norm": 5.09375,
      "learning_rate": 2.2821773229381924e-06,
      "log_odds_chosen": 1.8265974521636963,
      "log_odds_ratio": -0.33439093828201294,
      "logits/chosen": -3.333242416381836,
      "logits/rejected": -3.549222469329834,
      "logps/chosen": -0.7172473073005676,
      "logps/rejected": -1.4388408660888672,
      "loss": 0.9362,
      "nll_loss": 0.9278656244277954,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.03586236760020256,
      "rewards/margins": 0.036079682409763336,
      "rewards/rejected": -0.0719420462846756,
      "step": 480
    },
    {
      "epoch": 0.8860759493670886,
      "grad_norm": 4.21875,
      "learning_rate": 2.2587697572631284e-06,
      "log_odds_chosen": 0.9842466115951538,
      "log_odds_ratio": -0.42221134901046753,
      "logits/chosen": -3.305194139480591,
      "logits/rejected": -3.4940364360809326,
      "logps/chosen": -0.8084889650344849,
      "logps/rejected": -1.2992254495620728,
      "loss": 1.0103,
      "nll_loss": 0.9417991638183594,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.040424447506666183,
      "rewards/margins": 0.024536821991205215,
      "rewards/rejected": -0.0649612694978714,
      "step": 490
    },
    {
      "epoch": 0.9041591320072333,
      "grad_norm": 3.40625,
      "learning_rate": 2.23606797749979e-06,
      "log_odds_chosen": 0.06368019431829453,
      "log_odds_ratio": -0.6845844388008118,
      "logits/chosen": -3.6345558166503906,
      "logits/rejected": -3.47111177444458,
      "logps/chosen": -1.1341079473495483,
      "logps/rejected": -1.1745169162750244,
      "loss": 1.062,
      "nll_loss": 1.0511561632156372,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.05670539662241936,
      "rewards/margins": 0.0020204458851367235,
      "rewards/rejected": -0.05872584506869316,
      "step": 500
    },
    {
      "epoch": 0.9222423146473779,
      "grad_norm": 4.15625,
      "learning_rate": 2.2140372138502386e-06,
      "log_odds_chosen": 1.0155892372131348,
      "log_odds_ratio": -0.4495448172092438,
      "logits/chosen": -3.242302656173706,
      "logits/rejected": -3.3566043376922607,
      "logps/chosen": -0.6464399099349976,
      "logps/rejected": -1.0815407037734985,
      "loss": 1.0125,
      "nll_loss": 0.6781537532806396,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.032321996986866,
      "rewards/margins": 0.02175503969192505,
      "rewards/rejected": -0.054077040404081345,
      "step": 510
    },
    {
      "epoch": 0.9403254972875226,
      "grad_norm": 3.5625,
      "learning_rate": 2.1926450482675734e-06,
      "log_odds_chosen": 0.5763598084449768,
      "log_odds_ratio": -0.659260630607605,
      "logits/chosen": -3.5601630210876465,
      "logits/rejected": -3.527803897857666,
      "logps/chosen": -1.2068111896514893,
      "logps/rejected": -1.3772714138031006,
      "loss": 1.0074,
      "nll_loss": 1.2287429571151733,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.06034056469798088,
      "rewards/margins": 0.008523015305399895,
      "rewards/rejected": -0.06886357814073563,
      "step": 520
    },
    {
      "epoch": 0.9584086799276673,
      "grad_norm": 3.828125,
      "learning_rate": 2.1718612138153473e-06,
      "log_odds_chosen": 0.2709532380104065,
      "log_odds_ratio": -0.6409106850624084,
      "logits/chosen": -3.741631269454956,
      "logits/rejected": -3.731243133544922,
      "logps/chosen": -1.1519293785095215,
      "logps/rejected": -1.3367632627487183,
      "loss": 1.0247,
      "nll_loss": 1.2208020687103271,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.05759647488594055,
      "rewards/margins": 0.009241694584488869,
      "rewards/rejected": -0.06683816015720367,
      "step": 530
    },
    {
      "epoch": 0.976491862567812,
      "grad_norm": 3.203125,
      "learning_rate": 2.151657414559676e-06,
      "log_odds_chosen": 0.8211873173713684,
      "log_odds_ratio": -0.5390130877494812,
      "logits/chosen": -3.598482608795166,
      "logits/rejected": -3.682385206222534,
      "logps/chosen": -1.2253609895706177,
      "logps/rejected": -1.6024954319000244,
      "loss": 1.0344,
      "nll_loss": 1.1462525129318237,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.061268050223588943,
      "rewards/margins": 0.018856722861528397,
      "rewards/rejected": -0.08012477308511734,
      "step": 540
    },
    {
      "epoch": 0.9945750452079566,
      "grad_norm": 7.03125,
      "learning_rate": 2.132007163556104e-06,
      "log_odds_chosen": 1.1987555027008057,
      "log_odds_ratio": -0.3731897473335266,
      "logits/chosen": -3.2686665058135986,
      "logits/rejected": -3.5461959838867188,
      "logps/chosen": -0.7877604961395264,
      "logps/rejected": -1.328150749206543,
      "loss": 1.017,
      "nll_loss": 0.808586597442627,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0393880195915699,
      "rewards/margins": 0.027019521221518517,
      "rewards/rejected": -0.06640754640102386,
      "step": 550
    },
    {
      "epoch": 1.0,
      "step": 553,
      "total_flos": 0.0,
      "train_loss": 1.340359150177747,
      "train_runtime": 13809.2478,
      "train_samples_per_second": 0.64,
      "train_steps_per_second": 0.04
    }
  ],
  "logging_steps": 10,
  "max_steps": 553,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}