{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 633,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.05,
      "grad_norm": 45.76888063592087,
      "learning_rate": 5.000000000000001e-07,
      "log_odds_chosen": 0.2575085163116455,
      "log_odds_ratio": -0.704267144203186,
      "logits/chosen": -2.8495709896087646,
      "logits/rejected": -2.8519797325134277,
      "logps/chosen": -1.2749807834625244,
      "logps/rejected": -1.5134317874908447,
      "loss": 1.2045,
      "nll_loss": 1.1489894390106201,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.127498060464859,
      "rewards/margins": 0.023845110088586807,
      "rewards/rejected": -0.1513431817293167,
      "step": 10
    },
    {
      "epoch": 0.09,
      "grad_norm": 17.550572571834728,
      "learning_rate": 1.0000000000000002e-06,
      "log_odds_chosen": 0.18295660614967346,
      "log_odds_ratio": -0.7083484530448914,
      "logits/chosen": -2.9011380672454834,
      "logits/rejected": -2.903749465942383,
      "logps/chosen": -0.9543957710266113,
      "logps/rejected": -1.0926854610443115,
      "loss": 1.0092,
      "nll_loss": 0.9102001190185547,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.09543957561254501,
      "rewards/margins": 0.013828963041305542,
      "rewards/rejected": -0.10926854610443115,
      "step": 20
    },
    {
      "epoch": 0.14,
      "grad_norm": 10.772595379798407,
      "learning_rate": 1.5e-06,
      "log_odds_chosen": 0.22791412472724915,
      "log_odds_ratio": -0.6785577535629272,
      "logits/chosen": -2.970428943634033,
      "logits/rejected": -2.9696388244628906,
      "logps/chosen": -0.9358822107315063,
      "logps/rejected": -1.0665825605392456,
      "loss": 0.968,
      "nll_loss": 0.8899785280227661,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0935882106423378,
      "rewards/margins": 0.013070052489638329,
      "rewards/rejected": -0.10665826499462128,
      "step": 30
    },
    {
      "epoch": 0.19,
      "grad_norm": 11.974913883612869,
      "learning_rate": 2.0000000000000003e-06,
      "log_odds_chosen": 0.12087659537792206,
      "log_odds_ratio": -0.7188054323196411,
      "logits/chosen": -2.883781671524048,
      "logits/rejected": -2.8842344284057617,
      "logps/chosen": -0.9183737635612488,
      "logps/rejected": -0.9950372576713562,
      "loss": 0.9799,
      "nll_loss": 0.8861400485038757,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.09183738380670547,
      "rewards/margins": 0.007666352204978466,
      "rewards/rejected": -0.09950373321771622,
      "step": 40
    },
    {
      "epoch": 0.24,
      "grad_norm": 13.063526867338696,
      "learning_rate": 2.5e-06,
      "log_odds_chosen": 0.4301052689552307,
      "log_odds_ratio": -0.6214904189109802,
      "logits/chosen": -2.832704782485962,
      "logits/rejected": -2.8430252075195312,
      "logps/chosen": -0.9002718925476074,
      "logps/rejected": -1.2055391073226929,
      "loss": 0.9584,
      "nll_loss": 0.9324554204940796,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.09002719074487686,
      "rewards/margins": 0.03052671253681183,
      "rewards/rejected": -0.12055391073226929,
      "step": 50
    },
    {
      "epoch": 0.28,
      "grad_norm": 10.235387913194325,
      "learning_rate": 3e-06,
      "log_odds_chosen": 0.3391026258468628,
      "log_odds_ratio": -0.6046633720397949,
      "logits/chosen": -2.597712993621826,
      "logits/rejected": -2.598874568939209,
      "logps/chosen": -0.7375704050064087,
      "logps/rejected": -0.9383118748664856,
      "loss": 0.8856,
      "nll_loss": 0.7253891229629517,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.07375704497098923,
      "rewards/margins": 0.020074138417840004,
      "rewards/rejected": -0.09383118897676468,
      "step": 60
    },
    {
      "epoch": 0.33,
      "grad_norm": 10.615024222922383,
      "learning_rate": 3.5e-06,
      "log_odds_chosen": 0.34223267436027527,
      "log_odds_ratio": -0.6422346830368042,
      "logits/chosen": -2.540403366088867,
      "logits/rejected": -2.5442099571228027,
      "logps/chosen": -0.8444018363952637,
      "logps/rejected": -1.083935022354126,
      "loss": 0.9384,
      "nll_loss": 0.8686600923538208,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.08444017916917801,
      "rewards/margins": 0.02395332232117653,
      "rewards/rejected": -0.10839351266622543,
      "step": 70
    },
    {
      "epoch": 0.38,
      "grad_norm": 10.120353883295946,
      "learning_rate": 4.000000000000001e-06,
      "log_odds_chosen": 0.026019399985671043,
      "log_odds_ratio": -0.7628353834152222,
      "logits/chosen": -2.6254281997680664,
      "logits/rejected": -2.6229329109191895,
      "logps/chosen": -0.8879353404045105,
      "logps/rejected": -0.9107568860054016,
      "loss": 0.921,
      "nll_loss": 0.9201810956001282,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.08879353106021881,
      "rewards/margins": 0.0022821640595793724,
      "rewards/rejected": -0.09107569605112076,
      "step": 80
    },
    {
      "epoch": 0.43,
      "grad_norm": 9.558778662613774,
      "learning_rate": 4.5e-06,
      "log_odds_chosen": 0.29862236976623535,
      "log_odds_ratio": -0.6326144933700562,
      "logits/chosen": -2.6428685188293457,
      "logits/rejected": -2.621931552886963,
      "logps/chosen": -0.8801389932632446,
      "logps/rejected": -1.0846771001815796,
      "loss": 0.9325,
      "nll_loss": 0.8730384707450867,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.0880139023065567,
      "rewards/margins": 0.020453810691833496,
      "rewards/rejected": -0.1084677204489708,
      "step": 90
    },
    {
      "epoch": 0.47,
      "grad_norm": 11.437968315222852,
      "learning_rate": 5e-06,
      "log_odds_chosen": 0.3877044916152954,
      "log_odds_ratio": -0.6339091658592224,
      "logits/chosen": -2.583739757537842,
      "logits/rejected": -2.6161398887634277,
      "logps/chosen": -0.8836472630500793,
      "logps/rejected": -1.1088001728057861,
      "loss": 0.9742,
      "nll_loss": 0.9505327939987183,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.08836471289396286,
      "rewards/margins": 0.02251528576016426,
      "rewards/rejected": -0.11088001728057861,
      "step": 100
    },
    {
      "epoch": 0.52,
      "grad_norm": 9.765547500332794,
      "learning_rate": 4.767312946227961e-06,
      "log_odds_chosen": 0.33769896626472473,
      "log_odds_ratio": -0.6257166266441345,
      "logits/chosen": -2.6281538009643555,
      "logits/rejected": -2.6198818683624268,
      "logps/chosen": -0.7726272344589233,
      "logps/rejected": -0.9787896871566772,
      "loss": 0.9403,
      "nll_loss": 0.8244824409484863,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07726272195577621,
      "rewards/margins": 0.020616240799427032,
      "rewards/rejected": -0.09787896275520325,
      "step": 110
    },
    {
      "epoch": 0.57,
      "grad_norm": 10.192809233857067,
      "learning_rate": 4.564354645876385e-06,
      "log_odds_chosen": 0.31468966603279114,
      "log_odds_ratio": -0.6638727784156799,
      "logits/chosen": -2.6107370853424072,
      "logits/rejected": -2.601783275604248,
      "logps/chosen": -0.8392475247383118,
      "logps/rejected": -1.036677598953247,
      "loss": 0.9841,
      "nll_loss": 0.8654748797416687,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.08392475545406342,
      "rewards/margins": 0.019743015989661217,
      "rewards/rejected": -0.10366777330636978,
      "step": 120
    },
    {
      "epoch": 0.62,
      "grad_norm": 11.038589594109835,
      "learning_rate": 4.385290096535147e-06,
      "log_odds_chosen": 0.2760984003543854,
      "log_odds_ratio": -0.6513851881027222,
      "logits/chosen": -2.566074848175049,
      "logits/rejected": -2.565674304962158,
      "logps/chosen": -0.833086371421814,
      "logps/rejected": -0.9857648611068726,
      "loss": 0.9998,
      "nll_loss": 0.9077309370040894,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0833086371421814,
      "rewards/margins": 0.015267851762473583,
      "rewards/rejected": -0.09857650101184845,
      "step": 130
    },
    {
      "epoch": 0.66,
      "grad_norm": 9.191362766920426,
      "learning_rate": 4.2257712736425835e-06,
      "log_odds_chosen": 0.37576600909233093,
      "log_odds_ratio": -0.6204187273979187,
      "logits/chosen": -2.577667474746704,
      "logits/rejected": -2.584357976913452,
      "logps/chosen": -0.8501531481742859,
      "logps/rejected": -1.1260815858840942,
      "loss": 0.9545,
      "nll_loss": 0.8476670980453491,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.08501531183719635,
      "rewards/margins": 0.027592848986387253,
      "rewards/rejected": -0.1126081570982933,
      "step": 140
    },
    {
      "epoch": 0.71,
      "grad_norm": 10.020928126499054,
      "learning_rate": 4.082482904638631e-06,
      "log_odds_chosen": 0.40555381774902344,
      "log_odds_ratio": -0.6359796524047852,
      "logits/chosen": -2.624826669692993,
      "logits/rejected": -2.6218724250793457,
      "logps/chosen": -0.8737274408340454,
      "logps/rejected": -1.1572520732879639,
      "loss": 0.9391,
      "nll_loss": 0.8726488947868347,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.08737273514270782,
      "rewards/margins": 0.028352469205856323,
      "rewards/rejected": -0.11572520434856415,
      "step": 150
    },
    {
      "epoch": 0.76,
      "grad_norm": 11.288206284049263,
      "learning_rate": 3.952847075210474e-06,
      "log_odds_chosen": 0.35032859444618225,
      "log_odds_ratio": -0.6347723007202148,
      "logits/chosen": -2.7082316875457764,
      "logits/rejected": -2.711594820022583,
      "logps/chosen": -0.8456757664680481,
      "logps/rejected": -1.0942612886428833,
      "loss": 0.9052,
      "nll_loss": 0.8740180730819702,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0845675840973854,
      "rewards/margins": 0.024858549237251282,
      "rewards/rejected": -0.10942612588405609,
      "step": 160
    },
    {
      "epoch": 0.81,
      "grad_norm": 9.222481367340817,
      "learning_rate": 3.834824944236852e-06,
      "log_odds_chosen": 0.14469774067401886,
      "log_odds_ratio": -0.6979460120201111,
      "logits/chosen": -2.575446844100952,
      "logits/rejected": -2.572465419769287,
      "logps/chosen": -0.8792887926101685,
      "logps/rejected": -0.9753986597061157,
      "loss": 0.9229,
      "nll_loss": 0.8886219263076782,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.0879288762807846,
      "rewards/margins": 0.009610971435904503,
      "rewards/rejected": -0.09753986448049545,
      "step": 170
    },
    {
      "epoch": 0.85,
      "grad_norm": 10.622839554119508,
      "learning_rate": 3.72677996249965e-06,
      "log_odds_chosen": 0.18417394161224365,
      "log_odds_ratio": -0.7312262058258057,
      "logits/chosen": -2.558830976486206,
      "logits/rejected": -2.5850634574890137,
      "logps/chosen": -0.9030648469924927,
      "logps/rejected": -1.0534106492996216,
      "loss": 0.9502,
      "nll_loss": 0.9085375070571899,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.09030647575855255,
      "rewards/margins": 0.015034586191177368,
      "rewards/rejected": -0.10534106194972992,
      "step": 180
    },
    {
      "epoch": 0.9,
      "grad_norm": 10.63166001842217,
      "learning_rate": 3.6273812505500587e-06,
      "log_odds_chosen": 0.26296746730804443,
      "log_odds_ratio": -0.6528322100639343,
      "logits/chosen": -2.5235934257507324,
      "logits/rejected": -2.509213924407959,
      "logps/chosen": -0.842459499835968,
      "logps/rejected": -1.0031650066375732,
      "loss": 0.935,
      "nll_loss": 0.840924084186554,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.0842459425330162,
      "rewards/margins": 0.016070572659373283,
      "rewards/rejected": -0.10031652450561523,
      "step": 190
    },
    {
      "epoch": 0.95,
      "grad_norm": 10.265720627085773,
      "learning_rate": 3.5355339059327378e-06,
      "log_odds_chosen": 0.20157551765441895,
      "log_odds_ratio": -0.6874672174453735,
      "logits/chosen": -2.5119268894195557,
      "logits/rejected": -2.516270399093628,
      "logps/chosen": -0.8817191123962402,
      "logps/rejected": -1.0264997482299805,
      "loss": 0.9827,
      "nll_loss": 0.9070207476615906,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.08817192167043686,
      "rewards/margins": 0.01447806041687727,
      "rewards/rejected": -0.10264997184276581,
      "step": 200
    },
    {
      "epoch": 1.0,
      "grad_norm": 10.139451950242082,
      "learning_rate": 3.450327796711771e-06,
      "log_odds_chosen": 0.22201482951641083,
      "log_odds_ratio": -0.6832079887390137,
      "logits/chosen": -2.438990831375122,
      "logits/rejected": -2.4239249229431152,
      "logps/chosen": -0.9554345011711121,
      "logps/rejected": -1.1055123805999756,
      "loss": 0.9961,
      "nll_loss": 0.9768595695495605,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.09554344415664673,
      "rewards/margins": 0.01500779576599598,
      "rewards/rejected": -0.11055125296115875,
      "step": 210
    },
    {
      "epoch": 1.0,
      "eval_log_odds_chosen": 0.3896147906780243,
      "eval_log_odds_ratio": -0.6294437050819397,
      "eval_logits/chosen": -2.359004259109497,
      "eval_logits/rejected": -2.3542282581329346,
      "eval_logps/chosen": -0.8311433792114258,
      "eval_logps/rejected": -1.0547062158584595,
      "eval_loss": 0.9198942184448242,
      "eval_nll_loss": 0.8774265646934509,
      "eval_rewards/accuracies": 0.6302083134651184,
      "eval_rewards/chosen": -0.08311434835195541,
      "eval_rewards/margins": 0.022356273606419563,
      "eval_rewards/rejected": -0.10547062009572983,
      "eval_runtime": 61.0411,
      "eval_samples_per_second": 12.287,
      "eval_steps_per_second": 0.393,
      "step": 211
    },
    {
      "epoch": 1.04,
      "grad_norm": 8.627109146931504,
      "learning_rate": 3.3709993123162106e-06,
      "log_odds_chosen": 1.4988970756530762,
      "log_odds_ratio": -0.30856823921203613,
      "logits/chosen": -2.3857474327087402,
      "logits/rejected": -2.3801207542419434,
      "logps/chosen": -0.45462799072265625,
      "logps/rejected": -1.2372629642486572,
      "loss": 0.564,
      "nll_loss": 0.48940524458885193,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.045462802052497864,
      "rewards/margins": 0.07826349139213562,
      "rewards/rejected": -0.12372628599405289,
      "step": 220
    },
    {
      "epoch": 1.09,
      "grad_norm": 10.837022751981742,
      "learning_rate": 3.296902366978936e-06,
      "log_odds_chosen": 1.769687294960022,
      "log_odds_ratio": -0.22798781096935272,
      "logits/chosen": -2.460228204727173,
      "logits/rejected": -2.4452919960021973,
      "logps/chosen": -0.48005446791648865,
      "logps/rejected": -1.4746625423431396,
      "loss": 0.5158,
      "nll_loss": 0.5252029895782471,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.04800545051693916,
      "rewards/margins": 0.09946081787347794,
      "rewards/rejected": -0.1474662721157074,
      "step": 230
    },
    {
      "epoch": 1.14,
      "grad_norm": 9.220899180149999,
      "learning_rate": 3.2274861218395142e-06,
      "log_odds_chosen": 1.664380431175232,
      "log_odds_ratio": -0.2567726671695709,
      "logits/chosen": -2.4479331970214844,
      "logits/rejected": -2.453798294067383,
      "logps/chosen": -0.43317586183547974,
      "logps/rejected": -1.2501039505004883,
      "loss": 0.4887,
      "nll_loss": 0.45028549432754517,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.04331757873296738,
      "rewards/margins": 0.08169281482696533,
      "rewards/rejected": -0.1250104010105133,
      "step": 240
    },
    {
      "epoch": 1.18,
      "grad_norm": 9.9378304724897,
      "learning_rate": 3.1622776601683796e-06,
      "log_odds_chosen": 1.5019989013671875,
      "log_odds_ratio": -0.298888623714447,
      "logits/chosen": -2.5074825286865234,
      "logits/rejected": -2.4912190437316895,
      "logps/chosen": -0.4875302314758301,
      "logps/rejected": -1.2663142681121826,
      "loss": 0.5504,
      "nll_loss": 0.5222305655479431,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.04875302314758301,
      "rewards/margins": 0.07787840068340302,
      "rewards/rejected": -0.12663142383098602,
      "step": 250
    },
    {
      "epoch": 1.23,
      "grad_norm": 10.42321736026305,
      "learning_rate": 3.1008683647302113e-06,
      "log_odds_chosen": 1.7215213775634766,
      "log_odds_ratio": -0.2548612058162689,
      "logits/chosen": -2.523898124694824,
      "logits/rejected": -2.516082525253296,
      "logps/chosen": -0.4069024920463562,
      "logps/rejected": -1.2453140020370483,
      "loss": 0.4878,
      "nll_loss": 0.4438396394252777,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.04069024696946144,
      "rewards/margins": 0.08384115993976593,
      "rewards/rejected": -0.12453138828277588,
      "step": 260
    },
    {
      "epoch": 1.28,
      "grad_norm": 8.517195541285803,
      "learning_rate": 3.0429030972509227e-06,
      "log_odds_chosen": 1.4622563123703003,
      "log_odds_ratio": -0.31491032242774963,
      "logits/chosen": -2.421508312225342,
      "logits/rejected": -2.4460442066192627,
      "logps/chosen": -0.44591742753982544,
      "logps/rejected": -1.127700924873352,
      "loss": 0.5104,
      "nll_loss": 0.48815250396728516,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.044591739773750305,
      "rewards/margins": 0.06817835569381714,
      "rewards/rejected": -0.11277009546756744,
      "step": 270
    },
    {
      "epoch": 1.33,
      "grad_norm": 8.676525480706056,
      "learning_rate": 2.988071523335984e-06,
      "log_odds_chosen": 1.713667631149292,
      "log_odds_ratio": -0.23984098434448242,
      "logits/chosen": -2.433767795562744,
      "logits/rejected": -2.4231998920440674,
      "logps/chosen": -0.42581120133399963,
      "logps/rejected": -1.254382848739624,
      "loss": 0.4905,
      "nll_loss": 0.4856826364994049,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.04258112236857414,
      "rewards/margins": 0.0828571617603302,
      "rewards/rejected": -0.12543828785419464,
      "step": 280
    },
    {
      "epoch": 1.37,
      "grad_norm": 10.179660663431909,
      "learning_rate": 2.9361010975735177e-06,
      "log_odds_chosen": 1.6406803131103516,
      "log_odds_ratio": -0.2432694435119629,
      "logits/chosen": -2.40620493888855,
      "logits/rejected": -2.462026357650757,
      "logps/chosen": -0.43379467725753784,
      "logps/rejected": -1.231093406677246,
      "loss": 0.4952,
      "nll_loss": 0.4767753481864929,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.043379463255405426,
      "rewards/margins": 0.07972987741231918,
      "rewards/rejected": -0.1231093555688858,
      "step": 290
    },
    {
      "epoch": 1.42,
      "grad_norm": 10.354476045311776,
      "learning_rate": 2.8867513459481293e-06,
      "log_odds_chosen": 1.8753684759140015,
      "log_odds_ratio": -0.24006888270378113,
      "logits/chosen": -2.4737935066223145,
      "logits/rejected": -2.483096122741699,
      "logps/chosen": -0.3926279544830322,
      "logps/rejected": -1.2253657579421997,
      "loss": 0.4883,
      "nll_loss": 0.4013836979866028,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.0392627976834774,
      "rewards/margins": 0.08327377587556839,
      "rewards/rejected": -0.12253657728433609,
      "step": 300
    },
    {
      "epoch": 1.47,
      "grad_norm": 9.015044311790971,
      "learning_rate": 2.839809171235324e-06,
      "log_odds_chosen": 1.8872253894805908,
      "log_odds_ratio": -0.22469110786914825,
      "logits/chosen": -2.400991678237915,
      "logits/rejected": -2.397636890411377,
      "logps/chosen": -0.40627264976501465,
      "logps/rejected": -1.4336292743682861,
      "loss": 0.5011,
      "nll_loss": 0.43345755338668823,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.040627263486385345,
      "rewards/margins": 0.10273567587137222,
      "rewards/rejected": -0.14336295425891876,
      "step": 310
    },
    {
      "epoch": 1.52,
      "grad_norm": 9.539457700499925,
      "learning_rate": 2.7950849718747376e-06,
      "log_odds_chosen": 1.8418588638305664,
      "log_odds_ratio": -0.20935162901878357,
      "logits/chosen": -2.3977291584014893,
      "logits/rejected": -2.3498239517211914,
      "logps/chosen": -0.4429972767829895,
      "logps/rejected": -1.4245294332504272,
      "loss": 0.4827,
      "nll_loss": 0.4764658510684967,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.04429972171783447,
      "rewards/margins": 0.09815323352813721,
      "rewards/rejected": -0.14245295524597168,
      "step": 320
    },
    {
      "epoch": 1.56,
      "grad_norm": 9.434503294939443,
      "learning_rate": 2.752409412815902e-06,
      "log_odds_chosen": 1.5989511013031006,
      "log_odds_ratio": -0.26907485723495483,
      "logits/chosen": -2.4358298778533936,
      "logits/rejected": -2.4641692638397217,
      "logps/chosen": -0.4505313038825989,
      "logps/rejected": -1.2999070882797241,
      "loss": 0.4685,
      "nll_loss": 0.49505728483200073,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.04505313187837601,
      "rewards/margins": 0.08493757992982864,
      "rewards/rejected": -0.12999072670936584,
      "step": 330
    },
    {
      "epoch": 1.61,
      "grad_norm": 9.785466529429728,
      "learning_rate": 2.711630722733202e-06,
      "log_odds_chosen": 1.757354736328125,
      "log_odds_ratio": -0.22594013810157776,
      "logits/chosen": -2.386993885040283,
      "logits/rejected": -2.3797829151153564,
      "logps/chosen": -0.3836737275123596,
      "logps/rejected": -1.226621389389038,
      "loss": 0.5139,
      "nll_loss": 0.4257877469062805,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.0383673757314682,
      "rewards/margins": 0.08429475873708725,
      "rewards/rejected": -0.12266211211681366,
      "step": 340
    },
    {
      "epoch": 1.66,
      "grad_norm": 9.783556860345605,
      "learning_rate": 2.6726124191242444e-06,
      "log_odds_chosen": 1.5875946283340454,
      "log_odds_ratio": -0.282478392124176,
      "logits/chosen": -2.4133667945861816,
      "logits/rejected": -2.4031758308410645,
      "logps/chosen": -0.47809138894081116,
      "logps/rejected": -1.3111228942871094,
      "loss": 0.4947,
      "nll_loss": 0.46197766065597534,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.04780913516879082,
      "rewards/margins": 0.08330314606428146,
      "rewards/rejected": -0.13111227750778198,
      "step": 350
    },
    {
      "epoch": 1.71,
      "grad_norm": 10.952497066310066,
      "learning_rate": 2.6352313834736496e-06,
      "log_odds_chosen": 1.444085717201233,
      "log_odds_ratio": -0.302844375371933,
      "logits/chosen": -2.35420560836792,
      "logits/rejected": -2.40644907951355,
      "logps/chosen": -0.4726253151893616,
      "logps/rejected": -1.181214451789856,
      "loss": 0.5344,
      "nll_loss": 0.5417958498001099,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.0472625270485878,
      "rewards/margins": 0.0708589106798172,
      "rewards/rejected": -0.1181214451789856,
      "step": 360
    },
    {
      "epoch": 1.75,
      "grad_norm": 9.337096555826308,
      "learning_rate": 2.599376224550182e-06,
      "log_odds_chosen": 1.5420761108398438,
      "log_odds_ratio": -0.2706839144229889,
      "logits/chosen": -2.393585681915283,
      "logits/rejected": -2.3775479793548584,
      "logps/chosen": -0.4294983744621277,
      "logps/rejected": -1.177330493927002,
      "loss": 0.489,
      "nll_loss": 0.4465916156768799,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.04294983297586441,
      "rewards/margins": 0.07478321343660355,
      "rewards/rejected": -0.11773304641246796,
      "step": 370
    },
    {
      "epoch": 1.8,
      "grad_norm": 9.897099582842259,
      "learning_rate": 2.564945880212886e-06,
      "log_odds_chosen": 1.6586507558822632,
      "log_odds_ratio": -0.2737709581851959,
      "logits/chosen": -2.505715847015381,
      "logits/rejected": -2.478332996368408,
      "logps/chosen": -0.47623515129089355,
      "logps/rejected": -1.3953008651733398,
      "loss": 0.5232,
      "nll_loss": 0.498232901096344,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.047623515129089355,
      "rewards/margins": 0.0919065773487091,
      "rewards/rejected": -0.13953009247779846,
      "step": 380
    },
    {
      "epoch": 1.85,
      "grad_norm": 8.744712266716895,
      "learning_rate": 2.5318484177091667e-06,
      "log_odds_chosen": 1.4230409860610962,
      "log_odds_ratio": -0.29352661967277527,
      "logits/chosen": -2.455392360687256,
      "logits/rejected": -2.4511191844940186,
      "logps/chosen": -0.5203135013580322,
      "logps/rejected": -1.2620891332626343,
      "loss": 0.5658,
      "nll_loss": 0.547887921333313,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.05203135684132576,
      "rewards/margins": 0.0741775631904602,
      "rewards/rejected": -0.12620893120765686,
      "step": 390
    },
    {
      "epoch": 1.9,
      "grad_norm": 9.30311341527825,
      "learning_rate": 2.5e-06,
      "log_odds_chosen": 1.7381293773651123,
      "log_odds_ratio": -0.25197070837020874,
      "logits/chosen": -2.512232780456543,
      "logits/rejected": -2.5167810916900635,
      "logps/chosen": -0.4353383183479309,
      "logps/rejected": -1.34853196144104,
      "loss": 0.5158,
      "nll_loss": 0.4797208309173584,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.04353383183479309,
      "rewards/margins": 0.09131935983896255,
      "rewards/rejected": -0.13485319912433624,
      "step": 400
    },
    {
      "epoch": 1.94,
      "grad_norm": 9.272816070528886,
      "learning_rate": 2.4693239916239746e-06,
      "log_odds_chosen": 1.5647149085998535,
      "log_odds_ratio": -0.29222574830055237,
      "logits/chosen": -2.4848999977111816,
      "logits/rejected": -2.4973931312561035,
      "logps/chosen": -0.4486003816127777,
      "logps/rejected": -1.2298743724822998,
      "loss": 0.4991,
      "nll_loss": 0.49500808119773865,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.04486004263162613,
      "rewards/margins": 0.07812739908695221,
      "rewards/rejected": -0.12298743426799774,
      "step": 410
    },
    {
      "epoch": 1.99,
      "grad_norm": 11.956370778331848,
      "learning_rate": 2.4397501823713327e-06,
      "log_odds_chosen": 1.8623790740966797,
      "log_odds_ratio": -0.23835237324237823,
      "logits/chosen": -2.491649866104126,
      "logits/rejected": -2.4964215755462646,
      "logps/chosen": -0.36661866307258606,
      "logps/rejected": -1.2916817665100098,
      "loss": 0.4999,
      "nll_loss": 0.4245728850364685,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.03666186332702637,
      "rewards/margins": 0.09250631183385849,
      "rewards/rejected": -0.12916818261146545,
      "step": 420
    },
    {
      "epoch": 2.0,
      "eval_log_odds_chosen": 0.4993869364261627,
      "eval_log_odds_ratio": -0.6011503338813782,
      "eval_logits/chosen": -2.51769757270813,
      "eval_logits/rejected": -2.518415689468384,
      "eval_logps/chosen": -0.8936966061592102,
      "eval_logps/rejected": -1.183638334274292,
      "eval_loss": 0.9812777638435364,
      "eval_nll_loss": 0.940436840057373,
      "eval_rewards/accuracies": 0.6614583134651184,
      "eval_rewards/chosen": -0.08936966210603714,
      "eval_rewards/margins": 0.028994165360927582,
      "eval_rewards/rejected": -0.11836383491754532,
      "eval_runtime": 60.9721,
      "eval_samples_per_second": 12.301,
      "eval_steps_per_second": 0.394,
      "step": 422
    },
    {
      "epoch": 2.04,
      "grad_norm": 11.137133583839702,
      "learning_rate": 2.411214110852061e-06,
      "log_odds_chosen": 2.8829193115234375,
      "log_odds_ratio": -0.11160006374120712,
      "logits/chosen": -2.481720209121704,
      "logits/rejected": -2.5044140815734863,
      "logps/chosen": -0.21169082820415497,
      "logps/rejected": -1.3710792064666748,
      "loss": 0.2521,
      "nll_loss": 0.2048369199037552,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.021169083192944527,
      "rewards/margins": 0.11593882739543915,
      "rewards/rejected": -0.13710792362689972,
      "step": 430
    },
    {
      "epoch": 2.09,
      "grad_norm": 9.190185309499267,
      "learning_rate": 2.3836564731139807e-06,
      "log_odds_chosen": 3.475831985473633,
      "log_odds_ratio": -0.0667722150683403,
      "logits/chosen": -2.468839645385742,
      "logits/rejected": -2.504293441772461,
      "logps/chosen": -0.13927796483039856,
      "logps/rejected": -1.5609562397003174,
      "loss": 0.1889,
      "nll_loss": 0.142722949385643,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.013927794992923737,
      "rewards/margins": 0.1421678364276886,
      "rewards/rejected": -0.15609565377235413,
      "step": 440
    },
    {
      "epoch": 2.13,
      "grad_norm": 9.564387172693797,
      "learning_rate": 2.357022603955159e-06,
      "log_odds_chosen": 3.917105197906494,
      "log_odds_ratio": -0.037336982786655426,
      "logits/chosen": -2.5166149139404297,
      "logits/rejected": -2.545985698699951,
      "logps/chosen": -0.13963064551353455,
      "logps/rejected": -1.9594411849975586,
      "loss": 0.181,
      "nll_loss": 0.15659931302070618,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.013963064178824425,
      "rewards/margins": 0.18198105692863464,
      "rewards/rejected": -0.19594410061836243,
      "step": 450
    },
    {
      "epoch": 2.18,
      "grad_norm": 9.849074304214747,
      "learning_rate": 2.3312620206007847e-06,
      "log_odds_chosen": 3.3254103660583496,
      "log_odds_ratio": -0.06604982912540436,
      "logits/chosen": -2.4885592460632324,
      "logits/rejected": -2.5000545978546143,
      "logps/chosen": -0.17423538863658905,
      "logps/rejected": -1.5009019374847412,
      "loss": 0.1952,
      "nll_loss": 0.21155783534049988,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.017423538491129875,
      "rewards/margins": 0.13266664743423462,
      "rewards/rejected": -0.15009018778800964,
      "step": 460
    },
    {
      "epoch": 2.23,
      "grad_norm": 9.877044500517252,
      "learning_rate": 2.3063280200722128e-06,
      "log_odds_chosen": 3.6100239753723145,
      "log_odds_ratio": -0.05039358139038086,
      "logits/chosen": -2.5267927646636963,
      "logits/rejected": -2.5161995887756348,
      "logps/chosen": -0.16191843152046204,
      "logps/rejected": -1.8569713830947876,
      "loss": 0.173,
      "nll_loss": 0.16810880601406097,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.016191843897104263,
      "rewards/margins": 0.1695052832365036,
      "rewards/rejected": -0.18569712340831757,
      "step": 470
    },
    {
      "epoch": 2.27,
      "grad_norm": 8.163874170175868,
      "learning_rate": 2.2821773229381924e-06,
      "log_odds_chosen": 3.612773895263672,
      "log_odds_ratio": -0.05763618275523186,
      "logits/chosen": -2.5359530448913574,
      "logits/rejected": -2.539139747619629,
      "logps/chosen": -0.17719587683677673,
      "logps/rejected": -1.8914750814437866,
      "loss": 0.1808,
      "nll_loss": 0.17015354335308075,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.017719587311148643,
      "rewards/margins": 0.171427920460701,
      "rewards/rejected": -0.18914751708507538,
      "step": 480
    },
    {
      "epoch": 2.32,
      "grad_norm": 9.330097236244276,
      "learning_rate": 2.2587697572631284e-06,
      "log_odds_chosen": 3.656383514404297,
      "log_odds_ratio": -0.05575472116470337,
      "logits/chosen": -2.450357437133789,
      "logits/rejected": -2.4894728660583496,
      "logps/chosen": -0.12832140922546387,
      "logps/rejected": -1.6764119863510132,
      "loss": 0.1743,
      "nll_loss": 0.1293896734714508,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.012832140550017357,
      "rewards/margins": 0.15480905771255493,
      "rewards/rejected": -0.16764120757579803,
      "step": 490
    },
    {
      "epoch": 2.37,
      "grad_norm": 8.638065176436028,
      "learning_rate": 2.23606797749979e-06,
      "log_odds_chosen": 3.5653939247131348,
      "log_odds_ratio": -0.04521235078573227,
      "logits/chosen": -2.4729416370391846,
      "logits/rejected": -2.4972083568573,
      "logps/chosen": -0.14347168803215027,
      "logps/rejected": -1.709639549255371,
      "loss": 0.1687,
      "nll_loss": 0.15304037928581238,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.014347168616950512,
      "rewards/margins": 0.15661677718162537,
      "rewards/rejected": -0.17096397280693054,
      "step": 500
    },
    {
      "epoch": 2.42,
      "grad_norm": 9.821968133030161,
      "learning_rate": 2.2140372138502386e-06,
      "log_odds_chosen": 3.384936809539795,
      "log_odds_ratio": -0.06072957068681717,
      "logits/chosen": -2.477996349334717,
      "logits/rejected": -2.482419729232788,
      "logps/chosen": -0.17441074550151825,
      "logps/rejected": -1.6888253688812256,
      "loss": 0.191,
      "nll_loss": 0.20637556910514832,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.017441075295209885,
      "rewards/margins": 0.15144145488739014,
      "rewards/rejected": -0.16888253390789032,
      "step": 510
    },
    {
      "epoch": 2.46,
      "grad_norm": 9.230367667532265,
      "learning_rate": 2.1926450482675734e-06,
      "log_odds_chosen": 3.698854446411133,
      "log_odds_ratio": -0.04557035490870476,
      "logits/chosen": -2.493439197540283,
      "logits/rejected": -2.507626533508301,
      "logps/chosen": -0.15794190764427185,
      "logps/rejected": -1.836679458618164,
      "loss": 0.1823,
      "nll_loss": 0.1679120659828186,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.015794191509485245,
      "rewards/margins": 0.16787376999855042,
      "rewards/rejected": -0.18366795778274536,
      "step": 520
    },
    {
      "epoch": 2.51,
      "grad_norm": 9.220963630868974,
      "learning_rate": 2.1718612138153473e-06,
      "log_odds_chosen": 3.2164981365203857,
      "log_odds_ratio": -0.08045772463083267,
      "logits/chosen": -2.4849820137023926,
      "logits/rejected": -2.513134717941284,
      "logps/chosen": -0.20204570889472961,
      "logps/rejected": -1.6124742031097412,
      "loss": 0.1955,
      "nll_loss": 0.1980821043252945,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.0202045738697052,
      "rewards/margins": 0.14104285836219788,
      "rewards/rejected": -0.16124743223190308,
      "step": 530
    },
    {
      "epoch": 2.56,
      "grad_norm": 10.33252022252946,
      "learning_rate": 2.151657414559676e-06,
      "log_odds_chosen": 3.355163097381592,
      "log_odds_ratio": -0.05846261978149414,
      "logits/chosen": -2.4703683853149414,
      "logits/rejected": -2.4815986156463623,
      "logps/chosen": -0.16165122389793396,
      "logps/rejected": -1.595573902130127,
      "loss": 0.1944,
      "nll_loss": 0.1843605637550354,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.016165124252438545,
      "rewards/margins": 0.14339227974414825,
      "rewards/rejected": -0.15955740213394165,
      "step": 540
    },
    {
      "epoch": 2.61,
      "grad_norm": 10.384732401123731,
      "learning_rate": 2.132007163556104e-06,
      "log_odds_chosen": 3.7364859580993652,
      "log_odds_ratio": -0.050393521785736084,
      "logits/chosen": -2.488259792327881,
      "logits/rejected": -2.49784517288208,
      "logps/chosen": -0.1458165943622589,
      "logps/rejected": -1.7980148792266846,
      "loss": 0.1656,
      "nll_loss": 0.16633237898349762,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.014581659808754921,
      "rewards/margins": 0.16521981358528137,
      "rewards/rejected": -0.17980147898197174,
      "step": 550
    },
    {
      "epoch": 2.65,
      "grad_norm": 8.88379907455603,
      "learning_rate": 2.1128856368212917e-06,
      "log_odds_chosen": 3.3721110820770264,
      "log_odds_ratio": -0.05570857599377632,
      "logits/chosen": -2.4560656547546387,
      "logits/rejected": -2.459944725036621,
      "logps/chosen": -0.14691922068595886,
      "logps/rejected": -1.4988293647766113,
      "loss": 0.1941,
      "nll_loss": 0.16041871905326843,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.014691921882331371,
      "rewards/margins": 0.13519100844860077,
      "rewards/rejected": -0.1498829424381256,
      "step": 560
    },
    {
      "epoch": 2.7,
      "grad_norm": 8.564539697443587,
      "learning_rate": 2.0942695414584777e-06,
      "log_odds_chosen": 3.540961503982544,
      "log_odds_ratio": -0.050423987209796906,
      "logits/chosen": -2.4706387519836426,
      "logits/rejected": -2.478842258453369,
      "logps/chosen": -0.15486404299736023,
      "logps/rejected": -1.6839189529418945,
      "loss": 0.1827,
      "nll_loss": 0.16179664433002472,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.015486404299736023,
      "rewards/margins": 0.15290549397468567,
      "rewards/rejected": -0.1683918982744217,
      "step": 570
    },
    {
      "epoch": 2.75,
      "grad_norm": 8.776123151000377,
      "learning_rate": 2.0761369963434992e-06,
      "log_odds_chosen": 3.2635269165039062,
      "log_odds_ratio": -0.07641835510730743,
      "logits/chosen": -2.49845552444458,
      "logits/rejected": -2.5065455436706543,
      "logps/chosen": -0.21721407771110535,
      "logps/rejected": -1.628244161605835,
      "loss": 0.2085,
      "nll_loss": 0.26145899295806885,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.021721405908465385,
      "rewards/margins": 0.14110301434993744,
      "rewards/rejected": -0.16282442212104797,
      "step": 580
    },
    {
      "epoch": 2.8,
      "grad_norm": 8.360109402740795,
      "learning_rate": 2.058467423981546e-06,
      "log_odds_chosen": 3.5911190509796143,
      "log_odds_ratio": -0.04746793955564499,
      "logits/chosen": -2.4481544494628906,
      "logits/rejected": -2.4686474800109863,
      "logps/chosen": -0.18100793659687042,
      "logps/rejected": -1.7877342700958252,
      "loss": 0.1973,
      "nll_loss": 0.19511719048023224,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.018100792542099953,
      "rewards/margins": 0.1606726348400116,
      "rewards/rejected": -0.178773432970047,
      "step": 590
    },
    {
      "epoch": 2.84,
      "grad_norm": 8.230484923418327,
      "learning_rate": 2.0412414523193154e-06,
      "log_odds_chosen": 3.52643084526062,
      "log_odds_ratio": -0.05582552030682564,
      "logits/chosen": -2.43485689163208,
      "logits/rejected": -2.452052116394043,
      "logps/chosen": -0.16544193029403687,
      "logps/rejected": -1.6863620281219482,
      "loss": 0.1863,
      "nll_loss": 0.17863860726356506,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.016544191166758537,
      "rewards/margins": 0.15209202468395233,
      "rewards/rejected": -0.16863620281219482,
      "step": 600
    },
    {
      "epoch": 2.89,
      "grad_norm": 9.657714694141937,
      "learning_rate": 2.0244408254472904e-06,
      "log_odds_chosen": 3.441495418548584,
      "log_odds_ratio": -0.050580836832523346,
      "logits/chosen": -2.3969621658325195,
      "logits/rejected": -2.4254117012023926,
      "logps/chosen": -0.1414327472448349,
      "logps/rejected": -1.5730435848236084,
      "loss": 0.1825,
      "nll_loss": 0.16234278678894043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.01414327509701252,
      "rewards/margins": 0.1431610882282257,
      "rewards/rejected": -0.15730436146259308,
      "step": 610
    },
    {
      "epoch": 2.94,
      "grad_norm": 9.418203083185244,
      "learning_rate": 2.0080483222562476e-06,
      "log_odds_chosen": 3.510298252105713,
      "log_odds_ratio": -0.04852527379989624,
      "logits/chosen": -2.365495443344116,
      "logits/rejected": -2.3816208839416504,
      "logps/chosen": -0.14439718425273895,
      "logps/rejected": -1.5629642009735107,
      "loss": 0.1816,
      "nll_loss": 0.1693393439054489,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.014439716935157776,
      "rewards/margins": 0.14185671508312225,
      "rewards/rejected": -0.15629643201828003,
      "step": 620
    },
    {
      "epoch": 2.99,
      "grad_norm": 9.905181462463698,
      "learning_rate": 1.9920476822239895e-06,
      "log_odds_chosen": 3.6201272010803223,
      "log_odds_ratio": -0.04453429579734802,
      "logits/chosen": -2.415280818939209,
      "logits/rejected": -2.403134822845459,
      "logps/chosen": -0.1919378936290741,
      "logps/rejected": -1.9056262969970703,
      "loss": 0.1899,
      "nll_loss": 0.1791400909423828,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.01919379085302353,
      "rewards/margins": 0.17136886715888977,
      "rewards/rejected": -0.1905626356601715,
      "step": 630
    },
    {
      "epoch": 3.0,
      "eval_log_odds_chosen": 0.539490818977356,
      "eval_log_odds_ratio": -0.6108235716819763,
      "eval_logits/chosen": -2.4178202152252197,
      "eval_logits/rejected": -2.420111894607544,
      "eval_logps/chosen": -1.1129909753799438,
      "eval_logps/rejected": -1.4606622457504272,
      "eval_loss": 1.2152478694915771,
      "eval_nll_loss": 1.1710882186889648,
      "eval_rewards/accuracies": 0.65625,
      "eval_rewards/chosen": -0.11129910498857498,
      "eval_rewards/margins": 0.03476712107658386,
      "eval_rewards/rejected": -0.14606623351573944,
      "eval_runtime": 60.9563,
      "eval_samples_per_second": 12.304,
      "eval_steps_per_second": 0.394,
      "step": 633
    },
    {
      "epoch": 3.0,
      "step": 633,
      "total_flos": 0.0,
      "train_loss": 0.5526923979816467,
      "train_runtime": 6310.4087,
      "train_samples_per_second": 3.209,
      "train_steps_per_second": 0.1
    }
  ],
  "logging_steps": 10,
  "max_steps": 633,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}