{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.9966101694915253,
  "eval_steps": 500,
  "global_step": 663,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.05,
      "grad_norm": 10.625,
      "learning_rate": 5.000000000000001e-07,
      "log_odds_chosen": -0.21723361313343048,
      "log_odds_ratio": -0.8646289706230164,
      "logits/chosen": -2.376826286315918,
      "logits/rejected": -2.3396213054656982,
      "logps/chosen": -1.1265513896942139,
      "logps/rejected": -0.9904217720031738,
      "loss": 1.1515,
      "nll_loss": 1.1495189666748047,
      "rewards/accuracies": 0.3687500059604645,
      "rewards/chosen": -0.05632757022976875,
      "rewards/margins": -0.006806477904319763,
      "rewards/rejected": -0.04952109605073929,
      "step": 10
    },
    {
      "epoch": 0.09,
      "grad_norm": 8.125,
      "learning_rate": 1.0000000000000002e-06,
      "log_odds_chosen": -0.2775025963783264,
      "log_odds_ratio": -0.8951504826545715,
      "logits/chosen": -2.366422653198242,
      "logits/rejected": -2.338573694229126,
      "logps/chosen": -1.1216810941696167,
      "logps/rejected": -0.9544132351875305,
      "loss": 1.1414,
      "nll_loss": 1.1651629209518433,
      "rewards/accuracies": 0.32499998807907104,
      "rewards/chosen": -0.056084055453538895,
      "rewards/margins": -0.00836339220404625,
      "rewards/rejected": -0.047720663249492645,
      "step": 20
    },
    {
      "epoch": 0.14,
      "grad_norm": 7.03125,
      "learning_rate": 1.5e-06,
      "log_odds_chosen": -0.1686987578868866,
      "log_odds_ratio": -0.8350237011909485,
      "logits/chosen": -2.346161365509033,
      "logits/rejected": -2.3042690753936768,
      "logps/chosen": -1.0229403972625732,
      "logps/rejected": -0.916629433631897,
      "loss": 1.0927,
      "nll_loss": 1.063353419303894,
      "rewards/accuracies": 0.38749998807907104,
      "rewards/chosen": -0.051147013902664185,
      "rewards/margins": -0.005315541289746761,
      "rewards/rejected": -0.045831479132175446,
      "step": 30
    },
    {
      "epoch": 0.18,
      "grad_norm": 7.09375,
      "learning_rate": 2.0000000000000003e-06,
      "log_odds_chosen": -0.2760249376296997,
      "log_odds_ratio": -0.8817641139030457,
      "logits/chosen": -2.3606069087982178,
      "logits/rejected": -2.3503973484039307,
      "logps/chosen": -0.9669380187988281,
      "logps/rejected": -0.8209754228591919,
      "loss": 1.0772,
      "nll_loss": 0.9983038902282715,
      "rewards/accuracies": 0.33125001192092896,
      "rewards/chosen": -0.04834689944982529,
      "rewards/margins": -0.007298128213733435,
      "rewards/rejected": -0.041048772633075714,
      "step": 40
    },
    {
      "epoch": 0.23,
      "grad_norm": 7.1875,
      "learning_rate": 2.5e-06,
      "log_odds_chosen": -0.2603610157966614,
      "log_odds_ratio": -0.8878246545791626,
      "logits/chosen": -2.4249422550201416,
      "logits/rejected": -2.325108766555786,
      "logps/chosen": -1.055864930152893,
      "logps/rejected": -0.8944637179374695,
      "loss": 1.1001,
      "nll_loss": 1.1144287586212158,
      "rewards/accuracies": 0.33125001192092896,
      "rewards/chosen": -0.05279325321316719,
      "rewards/margins": -0.008070068433880806,
      "rewards/rejected": -0.044723186641931534,
      "step": 50
    },
    {
      "epoch": 0.27,
      "grad_norm": 6.84375,
      "learning_rate": 3e-06,
      "log_odds_chosen": -0.31346869468688965,
      "log_odds_ratio": -0.9280476570129395,
      "logits/chosen": -2.419933319091797,
      "logits/rejected": -2.3587355613708496,
      "logps/chosen": -1.0151221752166748,
      "logps/rejected": -0.8353691101074219,
      "loss": 1.0398,
      "nll_loss": 1.0006625652313232,
      "rewards/accuracies": 0.3187499940395355,
      "rewards/chosen": -0.05075611546635628,
      "rewards/margins": -0.00898765493184328,
      "rewards/rejected": -0.04176845774054527,
      "step": 60
    },
    {
      "epoch": 0.32,
      "grad_norm": 6.65625,
      "learning_rate": 3.5e-06,
      "log_odds_chosen": -0.21962186694145203,
      "log_odds_ratio": -0.8736317753791809,
      "logits/chosen": -2.3793551921844482,
      "logits/rejected": -2.3073837757110596,
      "logps/chosen": -0.9888677597045898,
      "logps/rejected": -0.8603803515434265,
      "loss": 1.0517,
      "nll_loss": 1.0219568014144897,
      "rewards/accuracies": 0.38749998807907104,
      "rewards/chosen": -0.04944339022040367,
      "rewards/margins": -0.006424369756132364,
      "rewards/rejected": -0.043019019067287445,
      "step": 70
    },
    {
      "epoch": 0.36,
      "grad_norm": 6.65625,
      "learning_rate": 4.000000000000001e-06,
      "log_odds_chosen": -0.28525999188423157,
      "log_odds_ratio": -0.9010836482048035,
      "logits/chosen": -2.3715453147888184,
      "logits/rejected": -2.297002077102661,
      "logps/chosen": -0.999737560749054,
      "logps/rejected": -0.8434433937072754,
      "loss": 1.074,
      "nll_loss": 1.0341795682907104,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.04998688027262688,
      "rewards/margins": -0.007814710959792137,
      "rewards/rejected": -0.04217217117547989,
      "step": 80
    },
    {
      "epoch": 0.41,
      "grad_norm": 6.3125,
      "learning_rate": 4.5e-06,
      "log_odds_chosen": -0.18699277937412262,
      "log_odds_ratio": -0.8439871668815613,
      "logits/chosen": -2.3840651512145996,
      "logits/rejected": -2.3138346672058105,
      "logps/chosen": -0.9523512721061707,
      "logps/rejected": -0.8433036804199219,
      "loss": 1.0374,
      "nll_loss": 0.9886404275894165,
      "rewards/accuracies": 0.39375001192092896,
      "rewards/chosen": -0.04761756584048271,
      "rewards/margins": -0.005452378187328577,
      "rewards/rejected": -0.04216518625617027,
      "step": 90
    },
    {
      "epoch": 0.45,
      "grad_norm": 7.3125,
      "learning_rate": 5e-06,
      "log_odds_chosen": -0.20422692596912384,
      "log_odds_ratio": -0.8546068072319031,
      "logits/chosen": -2.344386577606201,
      "logits/rejected": -2.282938003540039,
      "logps/chosen": -1.053534984588623,
      "logps/rejected": -0.9320154190063477,
      "loss": 1.0879,
      "nll_loss": 1.0812509059906006,
      "rewards/accuracies": 0.35624998807907104,
      "rewards/chosen": -0.05267674848437309,
      "rewards/margins": -0.006075979210436344,
      "rewards/rejected": -0.04660077020525932,
      "step": 100
    },
    {
      "epoch": 0.5,
      "grad_norm": 6.4375,
      "learning_rate": 4.767312946227961e-06,
      "log_odds_chosen": -0.24251386523246765,
      "log_odds_ratio": -0.8658794164657593,
      "logits/chosen": -2.3458285331726074,
      "logits/rejected": -2.298800230026245,
      "logps/chosen": -1.0196887254714966,
      "logps/rejected": -0.8735698461532593,
      "loss": 1.099,
      "nll_loss": 1.0551105737686157,
      "rewards/accuracies": 0.33125001192092896,
      "rewards/chosen": -0.05098443105816841,
      "rewards/margins": -0.007305940147489309,
      "rewards/rejected": -0.043678492307662964,
      "step": 110
    },
    {
      "epoch": 0.54,
      "grad_norm": 7.5,
      "learning_rate": 4.564354645876385e-06,
      "log_odds_chosen": -0.29140716791152954,
      "log_odds_ratio": -0.9143481254577637,
      "logits/chosen": -2.3598785400390625,
      "logits/rejected": -2.299387216567993,
      "logps/chosen": -1.0395349264144897,
      "logps/rejected": -0.8586183786392212,
      "loss": 1.0884,
      "nll_loss": 1.0841031074523926,
      "rewards/accuracies": 0.34375,
      "rewards/chosen": -0.05197674781084061,
      "rewards/margins": -0.009045831859111786,
      "rewards/rejected": -0.04293091595172882,
      "step": 120
    },
    {
      "epoch": 0.59,
      "grad_norm": 7.1875,
      "learning_rate": 4.385290096535147e-06,
      "log_odds_chosen": -0.19871631264686584,
      "log_odds_ratio": -0.8721719980239868,
      "logits/chosen": -2.4017045497894287,
      "logits/rejected": -2.3413734436035156,
      "logps/chosen": -0.9847584962844849,
      "logps/rejected": -0.8684871792793274,
      "loss": 1.0817,
      "nll_loss": 1.0258110761642456,
      "rewards/accuracies": 0.35624998807907104,
      "rewards/chosen": -0.0492379292845726,
      "rewards/margins": -0.005813568830490112,
      "rewards/rejected": -0.04342436045408249,
      "step": 130
    },
    {
      "epoch": 0.63,
      "grad_norm": 6.375,
      "learning_rate": 4.2257712736425835e-06,
      "log_odds_chosen": -0.2122315913438797,
      "log_odds_ratio": -0.8670593500137329,
      "logits/chosen": -2.3736579418182373,
      "logits/rejected": -2.2930188179016113,
      "logps/chosen": -0.9382177591323853,
      "logps/rejected": -0.821458637714386,
      "loss": 1.0268,
      "nll_loss": 0.9702949523925781,
      "rewards/accuracies": 0.35624998807907104,
      "rewards/chosen": -0.04691087827086449,
      "rewards/margins": -0.00583794666454196,
      "rewards/rejected": -0.04107293859124184,
      "step": 140
    },
    {
      "epoch": 0.68,
      "grad_norm": 7.25,
      "learning_rate": 4.082482904638631e-06,
      "log_odds_chosen": -0.22532574832439423,
      "log_odds_ratio": -0.8638848066329956,
      "logits/chosen": -2.442842960357666,
      "logits/rejected": -2.3551251888275146,
      "logps/chosen": -1.0039573907852173,
      "logps/rejected": -0.8670075535774231,
      "loss": 1.0895,
      "nll_loss": 1.044359564781189,
      "rewards/accuracies": 0.35624998807907104,
      "rewards/chosen": -0.05019787698984146,
      "rewards/margins": -0.006847499404102564,
      "rewards/rejected": -0.043350376188755035,
      "step": 150
    },
    {
      "epoch": 0.72,
      "grad_norm": 6.875,
      "learning_rate": 3.952847075210474e-06,
      "log_odds_chosen": -0.22583532333374023,
      "log_odds_ratio": -0.8749126195907593,
      "logits/chosen": -2.354644298553467,
      "logits/rejected": -2.2900655269622803,
      "logps/chosen": -0.9405444264411926,
      "logps/rejected": -0.80528724193573,
      "loss": 1.0844,
      "nll_loss": 1.0093518495559692,
      "rewards/accuracies": 0.36250001192092896,
      "rewards/chosen": -0.04702722281217575,
      "rewards/margins": -0.006762862205505371,
      "rewards/rejected": -0.04026436060667038,
      "step": 160
    },
    {
      "epoch": 0.77,
      "grad_norm": 6.0,
      "learning_rate": 3.834824944236852e-06,
      "log_odds_chosen": -0.30897724628448486,
      "log_odds_ratio": -0.9169195294380188,
      "logits/chosen": -2.4037585258483887,
      "logits/rejected": -2.382966995239258,
      "logps/chosen": -1.0549595355987549,
      "logps/rejected": -0.8608039617538452,
      "loss": 1.1012,
      "nll_loss": 1.069485068321228,
      "rewards/accuracies": 0.35624998807907104,
      "rewards/chosen": -0.052747976034879684,
      "rewards/margins": -0.009707780554890633,
      "rewards/rejected": -0.0430401973426342,
      "step": 170
    },
    {
      "epoch": 0.81,
      "grad_norm": 6.65625,
      "learning_rate": 3.72677996249965e-06,
      "log_odds_chosen": -0.151263028383255,
      "log_odds_ratio": -0.8355720639228821,
      "logits/chosen": -2.3262200355529785,
      "logits/rejected": -2.2469918727874756,
      "logps/chosen": -1.0002092123031616,
      "logps/rejected": -0.9038441777229309,
      "loss": 1.0736,
      "nll_loss": 1.0467875003814697,
      "rewards/accuracies": 0.3687500059604645,
      "rewards/chosen": -0.05001046508550644,
      "rewards/margins": -0.004818252753466368,
      "rewards/rejected": -0.045192211866378784,
      "step": 180
    },
    {
      "epoch": 0.86,
      "grad_norm": 6.1875,
      "learning_rate": 3.6273812505500587e-06,
      "log_odds_chosen": -0.21370911598205566,
      "log_odds_ratio": -0.8563556671142578,
      "logits/chosen": -2.3353800773620605,
      "logits/rejected": -2.302805185317993,
      "logps/chosen": -0.9616407155990601,
      "logps/rejected": -0.8339935541152954,
      "loss": 1.0925,
      "nll_loss": 1.0070997476577759,
      "rewards/accuracies": 0.34375,
      "rewards/chosen": -0.04808203876018524,
      "rewards/margins": -0.006382361985743046,
      "rewards/rejected": -0.04169967398047447,
      "step": 190
    },
    {
      "epoch": 0.9,
      "grad_norm": 6.875,
      "learning_rate": 3.5355339059327378e-06,
      "log_odds_chosen": -0.2662477195262909,
      "log_odds_ratio": -0.8836930394172668,
      "logits/chosen": -2.3342254161834717,
      "logits/rejected": -2.279543876647949,
      "logps/chosen": -0.9679354429244995,
      "logps/rejected": -0.812696635723114,
      "loss": 1.0688,
      "nll_loss": 0.9891126751899719,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.04839677736163139,
      "rewards/margins": -0.007761943154036999,
      "rewards/rejected": -0.04063483327627182,
      "step": 200
    },
    {
      "epoch": 0.95,
      "grad_norm": 6.5,
      "learning_rate": 3.450327796711771e-06,
      "log_odds_chosen": -0.352990984916687,
      "log_odds_ratio": -0.9537938833236694,
      "logits/chosen": -2.2331907749176025,
      "logits/rejected": -2.1765122413635254,
      "logps/chosen": -0.999859631061554,
      "logps/rejected": -0.8078708648681641,
      "loss": 1.0732,
      "nll_loss": 1.0370758771896362,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": -0.049992986023426056,
      "rewards/margins": -0.00959943700581789,
      "rewards/rejected": -0.04039354994893074,
      "step": 210
    },
    {
      "epoch": 0.99,
      "grad_norm": 6.6875,
      "learning_rate": 3.3709993123162106e-06,
      "log_odds_chosen": -0.24687866866588593,
      "log_odds_ratio": -0.885057270526886,
      "logits/chosen": -2.32244873046875,
      "logits/rejected": -2.2400941848754883,
      "logps/chosen": -0.9790719151496887,
      "logps/rejected": -0.8183375597000122,
      "loss": 1.0485,
      "nll_loss": 1.0295348167419434,
      "rewards/accuracies": 0.36250001192092896,
      "rewards/chosen": -0.0489535927772522,
      "rewards/margins": -0.008036715909838676,
      "rewards/rejected": -0.04091687500476837,
      "step": 220
    },
    {
      "epoch": 1.04,
      "grad_norm": 7.0625,
      "learning_rate": 3.296902366978936e-06,
      "log_odds_chosen": 0.004374316427856684,
      "log_odds_ratio": -0.7569075226783752,
      "logits/chosen": -2.33986234664917,
      "logits/rejected": -2.275083065032959,
      "logps/chosen": -0.8708177804946899,
      "logps/rejected": -0.8615075349807739,
      "loss": 0.9401,
      "nll_loss": 0.9019868969917297,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.043540894985198975,
      "rewards/margins": -0.00046551605919376016,
      "rewards/rejected": -0.043075378984212875,
      "step": 230
    },
    {
      "epoch": 1.08,
      "grad_norm": 6.9375,
      "learning_rate": 3.2274861218395142e-06,
      "log_odds_chosen": 0.136603444814682,
      "log_odds_ratio": -0.6914254426956177,
      "logits/chosen": -2.302295207977295,
      "logits/rejected": -2.263159990310669,
      "logps/chosen": -0.8053948283195496,
      "logps/rejected": -0.8713942766189575,
      "loss": 0.9052,
      "nll_loss": 0.8538738489151001,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.04026973992586136,
      "rewards/margins": 0.003299972740933299,
      "rewards/rejected": -0.04356971010565758,
      "step": 240
    },
    {
      "epoch": 1.13,
      "grad_norm": 7.59375,
      "learning_rate": 3.1622776601683796e-06,
      "log_odds_chosen": 0.1561969816684723,
      "log_odds_ratio": -0.7018811106681824,
      "logits/chosen": -2.317003011703491,
      "logits/rejected": -2.2483248710632324,
      "logps/chosen": -0.8204513788223267,
      "logps/rejected": -0.8764976263046265,
      "loss": 0.9152,
      "nll_loss": 0.865095317363739,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.04102257266640663,
      "rewards/margins": 0.0028023170307278633,
      "rewards/rejected": -0.04382488504052162,
      "step": 250
    },
    {
      "epoch": 1.18,
      "grad_norm": 8.5,
      "learning_rate": 3.1008683647302113e-06,
      "log_odds_chosen": 0.07478588074445724,
      "log_odds_ratio": -0.7376130223274231,
      "logits/chosen": -2.3320107460021973,
      "logits/rejected": -2.273357391357422,
      "logps/chosen": -0.8615882992744446,
      "logps/rejected": -0.8847838640213013,
      "loss": 0.9318,
      "nll_loss": 0.8836938142776489,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.04307941347360611,
      "rewards/margins": 0.0011597753036767244,
      "rewards/rejected": -0.04423918575048447,
      "step": 260
    },
    {
      "epoch": 1.22,
      "grad_norm": 7.125,
      "learning_rate": 3.0429030972509227e-06,
      "log_odds_chosen": 0.193772554397583,
      "log_odds_ratio": -0.6876689791679382,
      "logits/chosen": -2.303919553756714,
      "logits/rejected": -2.2494609355926514,
      "logps/chosen": -0.7990426421165466,
      "logps/rejected": -0.8900362253189087,
      "loss": 0.8974,
      "nll_loss": 0.848175048828125,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.03995212912559509,
      "rewards/margins": 0.0045496742241084576,
      "rewards/rejected": -0.044501807540655136,
      "step": 270
    },
    {
      "epoch": 1.27,
      "grad_norm": 7.0,
      "learning_rate": 2.988071523335984e-06,
      "log_odds_chosen": 0.1328398734331131,
      "log_odds_ratio": -0.6926364302635193,
      "logits/chosen": -2.3918557167053223,
      "logits/rejected": -2.3479630947113037,
      "logps/chosen": -0.7991486191749573,
      "logps/rejected": -0.8500739336013794,
      "loss": 0.9058,
      "nll_loss": 0.8436633348464966,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.039957426488399506,
      "rewards/margins": 0.0025462661869823933,
      "rewards/rejected": -0.04250369220972061,
      "step": 280
    },
    {
      "epoch": 1.31,
      "grad_norm": 7.625,
      "learning_rate": 2.9361010975735177e-06,
      "log_odds_chosen": 0.1771097183227539,
      "log_odds_ratio": -0.6848055124282837,
      "logits/chosen": -2.3858330249786377,
      "logits/rejected": -2.3222715854644775,
      "logps/chosen": -0.8123289346694946,
      "logps/rejected": -0.8936681747436523,
      "loss": 0.9117,
      "nll_loss": 0.8546589016914368,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.04061644524335861,
      "rewards/margins": 0.004066964611411095,
      "rewards/rejected": -0.044683415442705154,
      "step": 290
    },
    {
      "epoch": 1.36,
      "grad_norm": 6.84375,
      "learning_rate": 2.8867513459481293e-06,
      "log_odds_chosen": 0.154900923371315,
      "log_odds_ratio": -0.6884406805038452,
      "logits/chosen": -2.3433773517608643,
      "logits/rejected": -2.337531089782715,
      "logps/chosen": -0.8393154144287109,
      "logps/rejected": -0.8944731950759888,
      "loss": 0.9008,
      "nll_loss": 0.8844978213310242,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.04196577146649361,
      "rewards/margins": 0.0027578892186284065,
      "rewards/rejected": -0.04472366347908974,
      "step": 300
    },
    {
      "epoch": 1.4,
      "grad_norm": 7.21875,
      "learning_rate": 2.839809171235324e-06,
      "log_odds_chosen": 0.12805406749248505,
      "log_odds_ratio": -0.7049511671066284,
      "logits/chosen": -2.3576130867004395,
      "logits/rejected": -2.306063652038574,
      "logps/chosen": -0.842370867729187,
      "logps/rejected": -0.8757230639457703,
      "loss": 0.9307,
      "nll_loss": 0.8907486200332642,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.04211854189634323,
      "rewards/margins": 0.0016676082741469145,
      "rewards/rejected": -0.04378615692257881,
      "step": 310
    },
    {
      "epoch": 1.45,
      "grad_norm": 6.6875,
      "learning_rate": 2.7950849718747376e-06,
      "log_odds_chosen": 0.21001560986042023,
      "log_odds_ratio": -0.68244469165802,
      "logits/chosen": -2.40632700920105,
      "logits/rejected": -2.3724734783172607,
      "logps/chosen": -0.8295213580131531,
      "logps/rejected": -0.942533016204834,
      "loss": 0.921,
      "nll_loss": 0.8767648935317993,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.04147607088088989,
      "rewards/margins": 0.005650580860674381,
      "rewards/rejected": -0.0471266508102417,
      "step": 320
    },
    {
      "epoch": 1.49,
      "grad_norm": 7.53125,
      "learning_rate": 2.752409412815902e-06,
      "log_odds_chosen": 0.15271909534931183,
      "log_odds_ratio": -0.6864677667617798,
      "logits/chosen": -2.375275135040283,
      "logits/rejected": -2.3332605361938477,
      "logps/chosen": -0.8226965665817261,
      "logps/rejected": -0.8755828738212585,
      "loss": 0.9249,
      "nll_loss": 0.8956872820854187,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.04113483056426048,
      "rewards/margins": 0.0026443139649927616,
      "rewards/rejected": -0.04377914220094681,
      "step": 330
    },
    {
      "epoch": 1.54,
      "grad_norm": 7.75,
      "learning_rate": 2.711630722733202e-06,
      "log_odds_chosen": 0.11997294425964355,
      "log_odds_ratio": -0.6974139213562012,
      "logits/chosen": -2.3423027992248535,
      "logits/rejected": -2.3120710849761963,
      "logps/chosen": -0.8107906579971313,
      "logps/rejected": -0.8437153100967407,
      "loss": 0.9476,
      "nll_loss": 0.8603833913803101,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.040539536625146866,
      "rewards/margins": 0.0016462260391563177,
      "rewards/rejected": -0.04218576103448868,
      "step": 340
    },
    {
      "epoch": 1.58,
      "grad_norm": 7.4375,
      "learning_rate": 2.6726124191242444e-06,
      "log_odds_chosen": 0.12810608744621277,
      "log_odds_ratio": -0.7091251015663147,
      "logits/chosen": -2.353813886642456,
      "logits/rejected": -2.307131052017212,
      "logps/chosen": -0.7978275418281555,
      "logps/rejected": -0.8407192230224609,
      "loss": 0.922,
      "nll_loss": 0.8526071310043335,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.039891377091407776,
      "rewards/margins": 0.002144585130736232,
      "rewards/rejected": -0.04203595966100693,
      "step": 350
    },
    {
      "epoch": 1.63,
      "grad_norm": 6.46875,
      "learning_rate": 2.6352313834736496e-06,
      "log_odds_chosen": 0.14650364220142365,
      "log_odds_ratio": -0.6985893845558167,
      "logits/chosen": -2.434299945831299,
      "logits/rejected": -2.3850483894348145,
      "logps/chosen": -0.8027324676513672,
      "logps/rejected": -0.8706743121147156,
      "loss": 0.932,
      "nll_loss": 0.840488076210022,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.04013662412762642,
      "rewards/margins": 0.003397094551473856,
      "rewards/rejected": -0.04353372007608414,
      "step": 360
    },
    {
      "epoch": 1.67,
      "grad_norm": 7.0,
      "learning_rate": 2.599376224550182e-06,
      "log_odds_chosen": 0.07762489467859268,
      "log_odds_ratio": -0.7249036431312561,
      "logits/chosen": -2.3616278171539307,
      "logits/rejected": -2.2978177070617676,
      "logps/chosen": -0.811935544013977,
      "logps/rejected": -0.837403416633606,
      "loss": 0.9067,
      "nll_loss": 0.8698326945304871,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.04059677943587303,
      "rewards/margins": 0.0012733942130580544,
      "rewards/rejected": -0.04187016934156418,
      "step": 370
    },
    {
      "epoch": 1.72,
      "grad_norm": 7.5,
      "learning_rate": 2.564945880212886e-06,
      "log_odds_chosen": 0.10106128454208374,
      "log_odds_ratio": -0.6977318525314331,
      "logits/chosen": -2.368786334991455,
      "logits/rejected": -2.305479049682617,
      "logps/chosen": -0.8443781137466431,
      "logps/rejected": -0.8856738209724426,
      "loss": 0.902,
      "nll_loss": 0.886816143989563,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.04221891239285469,
      "rewards/margins": 0.0020647819619625807,
      "rewards/rejected": -0.04428369551897049,
      "step": 380
    },
    {
      "epoch": 1.76,
      "grad_norm": 7.0,
      "learning_rate": 2.5318484177091667e-06,
      "log_odds_chosen": 0.22741122543811798,
      "log_odds_ratio": -0.6608260869979858,
      "logits/chosen": -2.3913867473602295,
      "logits/rejected": -2.3317337036132812,
      "logps/chosen": -0.8110629916191101,
      "logps/rejected": -0.9145916104316711,
      "loss": 0.9266,
      "nll_loss": 0.8723472356796265,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.040553152561187744,
      "rewards/margins": 0.005176428239792585,
      "rewards/rejected": -0.04572957754135132,
      "step": 390
    },
    {
      "epoch": 1.81,
      "grad_norm": 7.125,
      "learning_rate": 2.5e-06,
      "log_odds_chosen": 0.09950557351112366,
      "log_odds_ratio": -0.7172703742980957,
      "logits/chosen": -2.3557465076446533,
      "logits/rejected": -2.3033339977264404,
      "logps/chosen": -0.9033384323120117,
      "logps/rejected": -0.936965823173523,
      "loss": 0.9442,
      "nll_loss": 0.9896990060806274,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.045166920870542526,
      "rewards/margins": 0.0016813663532957435,
      "rewards/rejected": -0.04684828966856003,
      "step": 400
    },
    {
      "epoch": 1.85,
      "grad_norm": 6.9375,
      "learning_rate": 2.4693239916239746e-06,
      "log_odds_chosen": 0.1489110141992569,
      "log_odds_ratio": -0.6765201687812805,
      "logits/chosen": -2.3149614334106445,
      "logits/rejected": -2.264227867126465,
      "logps/chosen": -0.774600625038147,
      "logps/rejected": -0.8234383463859558,
      "loss": 0.8682,
      "nll_loss": 0.8227639198303223,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.03873003274202347,
      "rewards/margins": 0.002441887278109789,
      "rewards/rejected": -0.04117191582918167,
      "step": 410
    },
    {
      "epoch": 1.9,
      "grad_norm": 7.21875,
      "learning_rate": 2.4397501823713327e-06,
      "log_odds_chosen": 0.26809313893318176,
      "log_odds_ratio": -0.6399468183517456,
      "logits/chosen": -2.3514695167541504,
      "logits/rejected": -2.292144298553467,
      "logps/chosen": -0.767677903175354,
      "logps/rejected": -0.8895092010498047,
      "loss": 0.8851,
      "nll_loss": 0.8227022886276245,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.03838389366865158,
      "rewards/margins": 0.006091567222028971,
      "rewards/rejected": -0.04447546228766441,
      "step": 420
    },
    {
      "epoch": 1.94,
      "grad_norm": 8.0,
      "learning_rate": 2.411214110852061e-06,
      "log_odds_chosen": 0.10217361152172089,
      "log_odds_ratio": -0.7120336294174194,
      "logits/chosen": -2.332167148590088,
      "logits/rejected": -2.262996196746826,
      "logps/chosen": -0.861347496509552,
      "logps/rejected": -0.9110593795776367,
      "loss": 0.9142,
      "nll_loss": 0.9066430330276489,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.04306737333536148,
      "rewards/margins": 0.0024855921510607004,
      "rewards/rejected": -0.04555296525359154,
      "step": 430
    },
    {
      "epoch": 1.99,
      "grad_norm": 7.09375,
      "learning_rate": 2.3836564731139807e-06,
      "log_odds_chosen": 0.06129971146583557,
      "log_odds_ratio": -0.7196843028068542,
      "logits/chosen": -2.345583915710449,
      "logits/rejected": -2.3343563079833984,
      "logps/chosen": -0.8849897384643555,
      "logps/rejected": -0.8971759080886841,
      "loss": 0.9506,
      "nll_loss": 0.9216864705085754,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.04424949362874031,
      "rewards/margins": 0.0006093080155551434,
      "rewards/rejected": -0.04485879838466644,
      "step": 440
    },
    {
      "epoch": 2.03,
      "grad_norm": 12.1875,
      "learning_rate": 2.357022603955159e-06,
      "log_odds_chosen": 0.3704330623149872,
      "log_odds_ratio": -0.6121096611022949,
      "logits/chosen": -2.294252634048462,
      "logits/rejected": -2.2701363563537598,
      "logps/chosen": -0.7903162837028503,
      "logps/rejected": -0.9567170143127441,
      "loss": 0.8449,
      "nll_loss": 0.8487945795059204,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.039515819400548935,
      "rewards/margins": 0.008320031687617302,
      "rewards/rejected": -0.04783584922552109,
      "step": 450
    },
    {
      "epoch": 2.08,
      "grad_norm": 8.8125,
      "learning_rate": 2.3312620206007847e-06,
      "log_odds_chosen": 0.571201741695404,
      "log_odds_ratio": -0.5243600606918335,
      "logits/chosen": -2.3143715858459473,
      "logits/rejected": -2.277233600616455,
      "logps/chosen": -0.7199083566665649,
      "logps/rejected": -0.9758491516113281,
      "loss": 0.8267,
      "nll_loss": 0.8100913763046265,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.03599541634321213,
      "rewards/margins": 0.012797033414244652,
      "rewards/rejected": -0.048792459070682526,
      "step": 460
    },
    {
      "epoch": 2.12,
      "grad_norm": 7.34375,
      "learning_rate": 2.3063280200722128e-06,
      "log_odds_chosen": 0.5478326082229614,
      "log_odds_ratio": -0.5462910532951355,
      "logits/chosen": -2.284961223602295,
      "logits/rejected": -2.2431790828704834,
      "logps/chosen": -0.7308935523033142,
      "logps/rejected": -0.952447235584259,
      "loss": 0.7979,
      "nll_loss": 0.8102613687515259,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.03654467687010765,
      "rewards/margins": 0.011077682487666607,
      "rewards/rejected": -0.04762236028909683,
      "step": 470
    },
    {
      "epoch": 2.17,
      "grad_norm": 8.3125,
      "learning_rate": 2.2821773229381924e-06,
      "log_odds_chosen": 0.5902162790298462,
      "log_odds_ratio": -0.5309671759605408,
      "logits/chosen": -2.303758144378662,
      "logits/rejected": -2.2506182193756104,
      "logps/chosen": -0.7160202264785767,
      "logps/rejected": -0.9725910425186157,
      "loss": 0.8048,
      "nll_loss": 0.7902317047119141,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.03580101206898689,
      "rewards/margins": 0.012828533537685871,
      "rewards/rejected": -0.04862954840064049,
      "step": 480
    },
    {
      "epoch": 2.21,
      "grad_norm": 7.78125,
      "learning_rate": 2.2587697572631284e-06,
      "log_odds_chosen": 0.4584142565727234,
      "log_odds_ratio": -0.578834593296051,
      "logits/chosen": -2.3179421424865723,
      "logits/rejected": -2.2865946292877197,
      "logps/chosen": -0.7205600738525391,
      "logps/rejected": -0.899226188659668,
      "loss": 0.7868,
      "nll_loss": 0.7924690246582031,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.03602800518274307,
      "rewards/margins": 0.008933312259614468,
      "rewards/rejected": -0.044961318373680115,
      "step": 490
    },
    {
      "epoch": 2.26,
      "grad_norm": 6.9375,
      "learning_rate": 2.23606797749979e-06,
      "log_odds_chosen": 0.5127681493759155,
      "log_odds_ratio": -0.5489556789398193,
      "logits/chosen": -2.3282065391540527,
      "logits/rejected": -2.263848066329956,
      "logps/chosen": -0.7146658897399902,
      "logps/rejected": -0.9588983654975891,
      "loss": 0.8044,
      "nll_loss": 0.7427123188972473,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.03573329374194145,
      "rewards/margins": 0.012211622670292854,
      "rewards/rejected": -0.04794491454958916,
      "step": 500
    },
    {
      "epoch": 2.31,
      "grad_norm": 7.59375,
      "learning_rate": 2.2140372138502386e-06,
      "log_odds_chosen": 0.5753137469291687,
      "log_odds_ratio": -0.5367614030838013,
      "logits/chosen": -2.2426199913024902,
      "logits/rejected": -2.2221579551696777,
      "logps/chosen": -0.68536376953125,
      "logps/rejected": -0.9230566024780273,
      "loss": 0.8109,
      "nll_loss": 0.7715431451797485,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.03426819294691086,
      "rewards/margins": 0.011884642764925957,
      "rewards/rejected": -0.046152833849191666,
      "step": 510
    },
    {
      "epoch": 2.35,
      "grad_norm": 7.96875,
      "learning_rate": 2.1926450482675734e-06,
      "log_odds_chosen": 0.3541458547115326,
      "log_odds_ratio": -0.6071102619171143,
      "logits/chosen": -2.31622576713562,
      "logits/rejected": -2.2613685131073,
      "logps/chosen": -0.7861618995666504,
      "logps/rejected": -0.941517174243927,
      "loss": 0.8433,
      "nll_loss": 0.8615577816963196,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.0393080934882164,
      "rewards/margins": 0.007767767645418644,
      "rewards/rejected": -0.04707586020231247,
      "step": 520
    },
    {
      "epoch": 2.4,
      "grad_norm": 8.125,
      "learning_rate": 2.1718612138153473e-06,
      "log_odds_chosen": 0.5630629062652588,
      "log_odds_ratio": -0.5376764535903931,
      "logits/chosen": -2.322035551071167,
      "logits/rejected": -2.2575039863586426,
      "logps/chosen": -0.728439450263977,
      "logps/rejected": -0.9899078607559204,
      "loss": 0.7861,
      "nll_loss": 0.7822965383529663,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.03642197325825691,
      "rewards/margins": 0.013073420152068138,
      "rewards/rejected": -0.0494953915476799,
      "step": 530
    },
    {
      "epoch": 2.44,
      "grad_norm": 7.78125,
      "learning_rate": 2.151657414559676e-06,
      "log_odds_chosen": 0.37775903940200806,
      "log_odds_ratio": -0.5940607190132141,
      "logits/chosen": -2.375549793243408,
      "logits/rejected": -2.321992874145508,
      "logps/chosen": -0.7584763169288635,
      "logps/rejected": -0.9299229383468628,
      "loss": 0.7987,
      "nll_loss": 0.8176478147506714,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.037923816591501236,
      "rewards/margins": 0.00857233814895153,
      "rewards/rejected": -0.04649615287780762,
      "step": 540
    },
    {
      "epoch": 2.49,
      "grad_norm": 7.59375,
      "learning_rate": 2.132007163556104e-06,
      "log_odds_chosen": 0.4968021512031555,
      "log_odds_ratio": -0.559329092502594,
      "logits/chosen": -2.27695894241333,
      "logits/rejected": -2.2415664196014404,
      "logps/chosen": -0.7272771596908569,
      "logps/rejected": -0.9639213681221008,
      "loss": 0.8073,
      "nll_loss": 0.7941919565200806,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.036363862454891205,
      "rewards/margins": 0.011832201853394508,
      "rewards/rejected": -0.04819606989622116,
      "step": 550
    },
    {
      "epoch": 2.53,
      "grad_norm": 8.4375,
      "learning_rate": 2.1128856368212917e-06,
      "log_odds_chosen": 0.5834923982620239,
      "log_odds_ratio": -0.544324517250061,
      "logits/chosen": -2.2980105876922607,
      "logits/rejected": -2.26395845413208,
      "logps/chosen": -0.7306932210922241,
      "logps/rejected": -0.9826458096504211,
      "loss": 0.8055,
      "nll_loss": 0.7919368743896484,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.036534663289785385,
      "rewards/margins": 0.012597626075148582,
      "rewards/rejected": -0.049132294952869415,
      "step": 560
    },
    {
      "epoch": 2.58,
      "grad_norm": 8.5625,
      "learning_rate": 2.0942695414584777e-06,
      "log_odds_chosen": 0.35887032747268677,
      "log_odds_ratio": -0.6235562562942505,
      "logits/chosen": -2.361316204071045,
      "logits/rejected": -2.294520854949951,
      "logps/chosen": -0.7684736251831055,
      "logps/rejected": -0.9107894897460938,
      "loss": 0.8402,
      "nll_loss": 0.8586353063583374,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.03842368349432945,
      "rewards/margins": 0.007115787826478481,
      "rewards/rejected": -0.04553947597742081,
      "step": 570
    },
    {
      "epoch": 2.62,
      "grad_norm": 7.90625,
      "learning_rate": 2.0761369963434992e-06,
      "log_odds_chosen": 0.6161745190620422,
      "log_odds_ratio": -0.5137969851493835,
      "logits/chosen": -2.321749448776245,
      "logits/rejected": -2.320282459259033,
      "logps/chosen": -0.686724841594696,
      "logps/rejected": -0.961820125579834,
      "loss": 0.7929,
      "nll_loss": 0.7309606671333313,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.03433624282479286,
      "rewards/margins": 0.013754759915173054,
      "rewards/rejected": -0.04809100180864334,
      "step": 580
    },
    {
      "epoch": 2.67,
      "grad_norm": 7.84375,
      "learning_rate": 2.058467423981546e-06,
      "log_odds_chosen": 0.5524671077728271,
      "log_odds_ratio": -0.5460283756256104,
      "logits/chosen": -2.3575046062469482,
      "logits/rejected": -2.301823616027832,
      "logps/chosen": -0.6897765398025513,
      "logps/rejected": -0.9405665397644043,
      "loss": 0.8173,
      "nll_loss": 0.7388589382171631,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.034488826990127563,
      "rewards/margins": 0.012539500370621681,
      "rewards/rejected": -0.047028329223394394,
      "step": 590
    },
    {
      "epoch": 2.71,
      "grad_norm": 8.0625,
      "learning_rate": 2.0412414523193154e-06,
      "log_odds_chosen": 0.4845600128173828,
      "log_odds_ratio": -0.5647393465042114,
      "logits/chosen": -2.317967176437378,
      "logits/rejected": -2.288545846939087,
      "logps/chosen": -0.7509777545928955,
      "logps/rejected": -0.971537709236145,
      "loss": 0.8197,
      "nll_loss": 0.8222864270210266,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.037548888474702835,
      "rewards/margins": 0.011028001084923744,
      "rewards/rejected": -0.04857688397169113,
      "step": 600
    },
    {
      "epoch": 2.76,
      "grad_norm": 7.3125,
      "learning_rate": 2.0244408254472904e-06,
      "log_odds_chosen": 0.4420229494571686,
      "log_odds_ratio": -0.5893286466598511,
      "logits/chosen": -2.3047995567321777,
      "logits/rejected": -2.318437099456787,
      "logps/chosen": -0.7420299053192139,
      "logps/rejected": -0.9326221346855164,
      "loss": 0.7964,
      "nll_loss": 0.7895435094833374,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.03710149601101875,
      "rewards/margins": 0.009529616683721542,
      "rewards/rejected": -0.04663110896945,
      "step": 610
    },
    {
      "epoch": 2.8,
      "grad_norm": 7.375,
      "learning_rate": 2.0080483222562476e-06,
      "log_odds_chosen": 0.44583791494369507,
      "log_odds_ratio": -0.5645539164543152,
      "logits/chosen": -2.3024137020111084,
      "logits/rejected": -2.244235038757324,
      "logps/chosen": -0.748982310295105,
      "logps/rejected": -0.9248701930046082,
      "loss": 0.8177,
      "nll_loss": 0.8224814534187317,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.03744911774992943,
      "rewards/margins": 0.008794394321739674,
      "rewards/rejected": -0.04624350741505623,
      "step": 620
    },
    {
      "epoch": 2.85,
      "grad_norm": 7.4375,
      "learning_rate": 1.9920476822239895e-06,
      "log_odds_chosen": 0.3953856825828552,
      "log_odds_ratio": -0.5933858156204224,
      "logits/chosen": -2.347012758255005,
      "logits/rejected": -2.2884857654571533,
      "logps/chosen": -0.7635630965232849,
      "logps/rejected": -0.9257116317749023,
      "loss": 0.8293,
      "nll_loss": 0.8256405591964722,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.03817816078662872,
      "rewards/margins": 0.008107425644993782,
      "rewards/rejected": -0.046285584568977356,
      "step": 630
    },
    {
      "epoch": 2.89,
      "grad_norm": 7.3125,
      "learning_rate": 1.976423537605237e-06,
      "log_odds_chosen": 0.5705650448799133,
      "log_odds_ratio": -0.547913134098053,
      "logits/chosen": -2.304575204849243,
      "logits/rejected": -2.2619354724884033,
      "logps/chosen": -0.6860629916191101,
      "logps/rejected": -0.9498222470283508,
      "loss": 0.8156,
      "nll_loss": 0.7438285946846008,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.034303151071071625,
      "rewards/margins": 0.01318796444684267,
      "rewards/rejected": -0.04749111086130142,
      "step": 640
    },
    {
      "epoch": 2.94,
      "grad_norm": 7.5625,
      "learning_rate": 1.961161351381841e-06,
      "log_odds_chosen": 0.5407294034957886,
      "log_odds_ratio": -0.5611921548843384,
      "logits/chosen": -2.366647243499756,
      "logits/rejected": -2.3279268741607666,
      "logps/chosen": -0.7047677636146545,
      "logps/rejected": -0.9417093396186829,
      "loss": 0.8069,
      "nll_loss": 0.7865225076675415,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.03523838892579079,
      "rewards/margins": 0.011847076937556267,
      "rewards/rejected": -0.0470854714512825,
      "step": 650
    },
    {
      "epoch": 2.98,
      "grad_norm": 7.8125,
      "learning_rate": 1.9462473604038077e-06,
      "log_odds_chosen": 0.4032713770866394,
      "log_odds_ratio": -0.5882443189620972,
      "logits/chosen": -2.343567371368408,
      "logits/rejected": -2.28145432472229,
      "logps/chosen": -0.7442666292190552,
      "logps/rejected": -0.9124624133110046,
      "loss": 0.7916,
      "nll_loss": 0.8193455934524536,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.03721333295106888,
      "rewards/margins": 0.008409791626036167,
      "rewards/rejected": -0.04562311992049217,
      "step": 660
    },
    {
      "epoch": 3.0,
      "step": 663,
      "total_flos": 0.0,
      "train_loss": 0.9363477402682757,
      "train_runtime": 28864.8871,
      "train_samples_per_second": 1.471,
      "train_steps_per_second": 0.023
    }
  ],
  "logging_steps": 10,
  "max_steps": 663,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}