{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 1346,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "grad_norm": 19.997434679340365,
      "learning_rate": 3.7037037037037036e-09,
      "logits/chosen": -1.5453667640686035,
      "logits/rejected": -1.4501094818115234,
      "logps/chosen": -154.90496826171875,
      "logps/rejected": -163.46749877929688,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "grad_norm": 23.34805628576002,
      "learning_rate": 3.7037037037037036e-08,
      "logits/chosen": -1.726180076599121,
      "logits/rejected": -1.6590251922607422,
      "logps/chosen": -212.55889892578125,
      "logps/rejected": -207.66229248046875,
      "loss": 0.693,
      "rewards/accuracies": 0.4236111044883728,
      "rewards/chosen": -0.0009580536279827356,
      "rewards/margins": -0.0008072062046267092,
      "rewards/rejected": -0.00015084730694070458,
      "step": 10
    },
    {
      "epoch": 0.01,
      "grad_norm": 25.078221894482837,
      "learning_rate": 7.407407407407407e-08,
      "logits/chosen": -1.7819910049438477,
      "logits/rejected": -1.7246497869491577,
      "logps/chosen": -215.92410278320312,
      "logps/rejected": -204.28173828125,
      "loss": 0.6928,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.0006128093227744102,
      "rewards/margins": 0.0009098866721615195,
      "rewards/rejected": -0.00029707717476412654,
      "step": 20
    },
    {
      "epoch": 0.02,
      "grad_norm": 22.11850294931131,
      "learning_rate": 1.111111111111111e-07,
      "logits/chosen": -1.7216441631317139,
      "logits/rejected": -1.6432338953018188,
      "logps/chosen": -225.88699340820312,
      "logps/rejected": -185.396240234375,
      "loss": 0.6914,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": 0.004545359406620264,
      "rewards/margins": 0.0039664083160459995,
      "rewards/rejected": 0.0005789512651972473,
      "step": 30
    },
    {
      "epoch": 0.03,
      "grad_norm": 21.112061167479588,
      "learning_rate": 1.4814814814814815e-07,
      "logits/chosen": -1.8239158391952515,
      "logits/rejected": -1.7619836330413818,
      "logps/chosen": -256.3050231933594,
      "logps/rejected": -245.38198852539062,
      "loss": 0.6877,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.019687145948410034,
      "rewards/margins": 0.009989428333938122,
      "rewards/rejected": 0.009697715751826763,
      "step": 40
    },
    {
      "epoch": 0.04,
      "grad_norm": 20.90083454488397,
      "learning_rate": 1.8518518518518516e-07,
      "logits/chosen": -1.7489999532699585,
      "logits/rejected": -1.7239185571670532,
      "logps/chosen": -217.8907928466797,
      "logps/rejected": -211.642333984375,
      "loss": 0.68,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.03657875210046768,
      "rewards/margins": 0.0253153033554554,
      "rewards/rejected": 0.011263446882367134,
      "step": 50
    },
    {
      "epoch": 0.04,
      "grad_norm": 21.77197780821128,
      "learning_rate": 2.222222222222222e-07,
      "logits/chosen": -1.8024189472198486,
      "logits/rejected": -1.6662009954452515,
      "logps/chosen": -271.22235107421875,
      "logps/rejected": -233.28823852539062,
      "loss": 0.6713,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.06959772109985352,
      "rewards/margins": 0.04836159199476242,
      "rewards/rejected": 0.021236125379800797,
      "step": 60
    },
    {
      "epoch": 0.05,
      "grad_norm": 21.554118928262394,
      "learning_rate": 2.5925925925925923e-07,
      "logits/chosen": -1.7849029302597046,
      "logits/rejected": -1.718541145324707,
      "logps/chosen": -224.2913055419922,
      "logps/rejected": -203.49624633789062,
      "loss": 0.6608,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 0.05916479229927063,
      "rewards/margins": 0.06906653940677643,
      "rewards/rejected": -0.009901740588247776,
      "step": 70
    },
    {
      "epoch": 0.06,
      "grad_norm": 20.717805914821145,
      "learning_rate": 2.962962962962963e-07,
      "logits/chosen": -1.866686463356018,
      "logits/rejected": -1.7345689535140991,
      "logps/chosen": -252.84555053710938,
      "logps/rejected": -228.15359497070312,
      "loss": 0.6487,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.05975146219134331,
      "rewards/margins": 0.1380428969860077,
      "rewards/rejected": -0.07829144597053528,
      "step": 80
    },
    {
      "epoch": 0.07,
      "grad_norm": 24.15752327353423,
      "learning_rate": 3.333333333333333e-07,
      "logits/chosen": -1.7628377676010132,
      "logits/rejected": -1.696528673171997,
      "logps/chosen": -256.96954345703125,
      "logps/rejected": -260.56292724609375,
      "loss": 0.6244,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.009766384959220886,
      "rewards/margins": 0.20868687331676483,
      "rewards/rejected": -0.19892050325870514,
      "step": 90
    },
    {
      "epoch": 0.07,
      "grad_norm": 22.043263558957296,
      "learning_rate": 3.703703703703703e-07,
      "logits/chosen": -1.758519172668457,
      "logits/rejected": -1.6800458431243896,
      "logps/chosen": -239.4019775390625,
      "logps/rejected": -226.4539794921875,
      "loss": 0.6142,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.1279800534248352,
      "rewards/margins": 0.24427437782287598,
      "rewards/rejected": -0.3722544014453888,
      "step": 100
    },
    {
      "epoch": 0.07,
      "eval_logits/chosen": -1.7870845794677734,
      "eval_logits/rejected": -1.752089500427246,
      "eval_logps/chosen": -302.7545166015625,
      "eval_logps/rejected": -330.3116455078125,
      "eval_loss": 0.6372222304344177,
      "eval_rewards/accuracies": 0.69921875,
      "eval_rewards/chosen": -0.21120953559875488,
      "eval_rewards/margins": 0.2142910659313202,
      "eval_rewards/rejected": -0.42550066113471985,
      "eval_runtime": 98.1168,
      "eval_samples_per_second": 20.384,
      "eval_steps_per_second": 0.326,
      "step": 100
    },
    {
      "epoch": 0.08,
      "grad_norm": 27.76371677719842,
      "learning_rate": 4.0740740740740737e-07,
      "logits/chosen": -1.7010570764541626,
      "logits/rejected": -1.6076711416244507,
      "logps/chosen": -263.64080810546875,
      "logps/rejected": -244.4661407470703,
      "loss": 0.5944,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.26566341519355774,
      "rewards/margins": 0.27579429745674133,
      "rewards/rejected": -0.5414577722549438,
      "step": 110
    },
    {
      "epoch": 0.09,
      "grad_norm": 31.731787834651325,
      "learning_rate": 4.444444444444444e-07,
      "logits/chosen": -1.7044484615325928,
      "logits/rejected": -1.6412681341171265,
      "logps/chosen": -269.258544921875,
      "logps/rejected": -282.24713134765625,
      "loss": 0.5696,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.39111006259918213,
      "rewards/margins": 0.36286839842796326,
      "rewards/rejected": -0.7539784908294678,
      "step": 120
    },
    {
      "epoch": 0.1,
      "grad_norm": 37.69053452768137,
      "learning_rate": 4.814814814814814e-07,
      "logits/chosen": -1.860668420791626,
      "logits/rejected": -1.8003613948822021,
      "logps/chosen": -286.83123779296875,
      "logps/rejected": -299.9700927734375,
      "loss": 0.553,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.3989219665527344,
      "rewards/margins": 0.4949173927307129,
      "rewards/rejected": -0.8938392400741577,
      "step": 130
    },
    {
      "epoch": 0.1,
      "grad_norm": 45.43263624776604,
      "learning_rate": 4.999789692194508e-07,
      "logits/chosen": -1.7893892526626587,
      "logits/rejected": -1.7304092645645142,
      "logps/chosen": -324.6953430175781,
      "logps/rejected": -354.299072265625,
      "loss": 0.5254,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.7778688073158264,
      "rewards/margins": 0.5480495095252991,
      "rewards/rejected": -1.325918436050415,
      "step": 140
    },
    {
      "epoch": 0.11,
      "grad_norm": 38.76316746995152,
      "learning_rate": 4.998107442045616e-07,
      "logits/chosen": -1.7821582555770874,
      "logits/rejected": -1.7230908870697021,
      "logps/chosen": -304.8173828125,
      "logps/rejected": -349.8572082519531,
      "loss": 0.5181,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.702239990234375,
      "rewards/margins": 0.6602993011474609,
      "rewards/rejected": -1.362539291381836,
      "step": 150
    },
    {
      "epoch": 0.12,
      "grad_norm": 38.48791118895226,
      "learning_rate": 4.994744073829293e-07,
      "logits/chosen": -1.9425595998764038,
      "logits/rejected": -1.8510286808013916,
      "logps/chosen": -315.58038330078125,
      "logps/rejected": -354.3575744628906,
      "loss": 0.5104,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.928961455821991,
      "rewards/margins": 0.6893821954727173,
      "rewards/rejected": -1.618343710899353,
      "step": 160
    },
    {
      "epoch": 0.13,
      "grad_norm": 34.98469211427676,
      "learning_rate": 4.989701850946613e-07,
      "logits/chosen": -2.1142868995666504,
      "logits/rejected": -2.0348961353302,
      "logps/chosen": -336.83929443359375,
      "logps/rejected": -374.04046630859375,
      "loss": 0.5009,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.7460821866989136,
      "rewards/margins": 0.756350040435791,
      "rewards/rejected": -1.5024322271347046,
      "step": 170
    },
    {
      "epoch": 0.13,
      "grad_norm": 45.707896850254954,
      "learning_rate": 4.982984166595104e-07,
      "logits/chosen": -2.1524124145507812,
      "logits/rejected": -2.127004861831665,
      "logps/chosen": -349.15081787109375,
      "logps/rejected": -417.07452392578125,
      "loss": 0.4877,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.093063473701477,
      "rewards/margins": 0.8320043683052063,
      "rewards/rejected": -1.9250679016113281,
      "step": 180
    },
    {
      "epoch": 0.14,
      "grad_norm": 41.90753016315753,
      "learning_rate": 4.974595541485259e-07,
      "logits/chosen": -2.118150234222412,
      "logits/rejected": -2.0182414054870605,
      "logps/chosen": -376.4796447753906,
      "logps/rejected": -435.715087890625,
      "loss": 0.4957,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.2654850482940674,
      "rewards/margins": 0.9563212394714355,
      "rewards/rejected": -2.221806049346924,
      "step": 190
    },
    {
      "epoch": 0.15,
      "grad_norm": 43.92066412227129,
      "learning_rate": 4.964541620798307e-07,
      "logits/chosen": -2.0093464851379395,
      "logits/rejected": -1.843632698059082,
      "logps/chosen": -358.4123840332031,
      "logps/rejected": -408.1987609863281,
      "loss": 0.4726,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -1.1928468942642212,
      "rewards/margins": 0.9090213775634766,
      "rewards/rejected": -2.101868152618408,
      "step": 200
    },
    {
      "epoch": 0.15,
      "eval_logits/chosen": -2.047096014022827,
      "eval_logits/rejected": -2.0018393993377686,
      "eval_logps/chosen": -416.041015625,
      "eval_logps/rejected": -498.22076416015625,
      "eval_loss": 0.5515660047531128,
      "eval_rewards/accuracies": 0.75,
      "eval_rewards/chosen": -1.3440749645233154,
      "eval_rewards/margins": 0.7605167627334595,
      "eval_rewards/rejected": -2.1045916080474854,
      "eval_runtime": 98.1193,
      "eval_samples_per_second": 20.383,
      "eval_steps_per_second": 0.326,
      "step": 200
    },
    {
      "epoch": 0.16,
      "grad_norm": 38.95795396663457,
      "learning_rate": 4.952829170387241e-07,
      "logits/chosen": -1.9518171548843384,
      "logits/rejected": -1.8534870147705078,
      "logps/chosen": -342.5758361816406,
      "logps/rejected": -430.5548400878906,
      "loss": 0.4601,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.3162614107131958,
      "rewards/margins": 1.062984824180603,
      "rewards/rejected": -2.3792459964752197,
      "step": 210
    },
    {
      "epoch": 0.16,
      "grad_norm": 43.63837784340095,
      "learning_rate": 4.939466072223697e-07,
      "logits/chosen": -1.9940170049667358,
      "logits/rejected": -1.8776836395263672,
      "logps/chosen": -409.5476989746094,
      "logps/rejected": -429.2942810058594,
      "loss": 0.4812,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.562101125717163,
      "rewards/margins": 0.7047168612480164,
      "rewards/rejected": -2.266817808151245,
      "step": 220
    },
    {
      "epoch": 0.17,
      "grad_norm": 48.17484694517937,
      "learning_rate": 4.924461319093725e-07,
      "logits/chosen": -1.911233901977539,
      "logits/rejected": -1.8419215679168701,
      "logps/chosen": -366.72210693359375,
      "logps/rejected": -425.0821838378906,
      "loss": 0.4424,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.3060904741287231,
      "rewards/margins": 0.9145557284355164,
      "rewards/rejected": -2.220646381378174,
      "step": 230
    },
    {
      "epoch": 0.18,
      "grad_norm": 42.0731451177421,
      "learning_rate": 4.907825008546038e-07,
      "logits/chosen": -1.7983890771865845,
      "logits/rejected": -1.6776669025421143,
      "logps/chosen": -404.84942626953125,
      "logps/rejected": -461.9938049316406,
      "loss": 0.452,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.4010937213897705,
      "rewards/margins": 0.9703173637390137,
      "rewards/rejected": -2.371410846710205,
      "step": 240
    },
    {
      "epoch": 0.19,
      "grad_norm": 45.788391518578905,
      "learning_rate": 4.889568336096795e-07,
      "logits/chosen": -1.7662121057510376,
      "logits/rejected": -1.6878684759140015,
      "logps/chosen": -368.25567626953125,
      "logps/rejected": -454.2179260253906,
      "loss": 0.4514,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.3891403675079346,
      "rewards/margins": 1.1145490407943726,
      "rewards/rejected": -2.5036895275115967,
      "step": 250
    },
    {
      "epoch": 0.19,
      "grad_norm": 53.99920452912666,
      "learning_rate": 4.869703587695508e-07,
      "logits/chosen": -1.7349084615707397,
      "logits/rejected": -1.6830803155899048,
      "logps/chosen": -421.4908752441406,
      "logps/rejected": -481.27752685546875,
      "loss": 0.4519,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.3783433437347412,
      "rewards/margins": 1.064690351486206,
      "rewards/rejected": -2.4430336952209473,
      "step": 260
    },
    {
      "epoch": 0.2,
      "grad_norm": 48.505757522556536,
      "learning_rate": 4.848244131457127e-07,
      "logits/chosen": -1.601601243019104,
      "logits/rejected": -1.4739550352096558,
      "logps/chosen": -386.08428955078125,
      "logps/rejected": -498.78558349609375,
      "loss": 0.4367,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.4528039693832397,
      "rewards/margins": 1.1855452060699463,
      "rewards/rejected": -2.6383490562438965,
      "step": 270
    },
    {
      "epoch": 0.21,
      "grad_norm": 49.31169957835279,
      "learning_rate": 4.825204408665877e-07,
      "logits/chosen": -1.697139024734497,
      "logits/rejected": -1.5427892208099365,
      "logps/chosen": -399.53729248046875,
      "logps/rejected": -492.3568420410156,
      "loss": 0.4205,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.5937035083770752,
      "rewards/margins": 1.3382676839828491,
      "rewards/rejected": -2.931971311569214,
      "step": 280
    },
    {
      "epoch": 0.22,
      "grad_norm": 38.828455566499606,
      "learning_rate": 4.800599924056907e-07,
      "logits/chosen": -1.7314732074737549,
      "logits/rejected": -1.5896819829940796,
      "logps/chosen": -407.6255798339844,
      "logps/rejected": -502.23016357421875,
      "loss": 0.4189,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.5207576751708984,
      "rewards/margins": 1.350722074508667,
      "rewards/rejected": -2.8714797496795654,
      "step": 290
    },
    {
      "epoch": 0.22,
      "grad_norm": 44.43194576384378,
      "learning_rate": 4.774447235382259e-07,
      "logits/chosen": -1.7066457271575928,
      "logits/rejected": -1.5852010250091553,
      "logps/chosen": -425.8912658691406,
      "logps/rejected": -526.4098510742188,
      "loss": 0.4421,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.5712147951126099,
      "rewards/margins": 1.0973128080368042,
      "rewards/rejected": -2.668527841567993,
      "step": 300
    },
    {
      "epoch": 0.22,
      "eval_logits/chosen": -1.8325203657150269,
      "eval_logits/rejected": -1.7522265911102295,
      "eval_logps/chosen": -396.3378601074219,
      "eval_logps/rejected": -492.3901062011719,
      "eval_loss": 0.5335086584091187,
      "eval_rewards/accuracies": 0.75390625,
      "eval_rewards/chosen": -1.147042989730835,
      "eval_rewards/margins": 0.8992425799369812,
      "eval_rewards/rejected": -2.046285629272461,
      "eval_runtime": 98.1593,
      "eval_samples_per_second": 20.375,
      "eval_steps_per_second": 0.326,
      "step": 300
    },
    {
      "epoch": 0.23,
      "grad_norm": 44.68676000662293,
      "learning_rate": 4.7467639422682426e-07,
      "logits/chosen": -1.5684497356414795,
      "logits/rejected": -1.5880094766616821,
      "logps/chosen": -429.06072998046875,
      "logps/rejected": -549.975830078125,
      "loss": 0.431,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.6676822900772095,
      "rewards/margins": 1.3817517757415771,
      "rewards/rejected": -3.049434185028076,
      "step": 310
    },
    {
      "epoch": 0.24,
      "grad_norm": 49.129673207510045,
      "learning_rate": 4.7175686743716223e-07,
      "logits/chosen": -1.6045185327529907,
      "logits/rejected": -1.5781729221343994,
      "logps/chosen": -407.8392639160156,
      "logps/rejected": -560.4733276367188,
      "loss": 0.4164,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.750781774520874,
      "rewards/margins": 1.4461250305175781,
      "rewards/rejected": -3.196906566619873,
      "step": 320
    },
    {
      "epoch": 0.25,
      "grad_norm": 44.13714858626996,
      "learning_rate": 4.686881078842688e-07,
      "logits/chosen": -1.65378737449646,
      "logits/rejected": -1.5229465961456299,
      "logps/chosen": -369.09075927734375,
      "logps/rejected": -465.15411376953125,
      "loss": 0.4162,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.5404636859893799,
      "rewards/margins": 1.1851208209991455,
      "rewards/rejected": -2.7255845069885254,
      "step": 330
    },
    {
      "epoch": 0.25,
      "grad_norm": 67.75141126041814,
      "learning_rate": 4.654721807103558e-07,
      "logits/chosen": -1.668287992477417,
      "logits/rejected": -1.4913742542266846,
      "logps/chosen": -451.5079040527344,
      "logps/rejected": -548.6111450195312,
      "loss": 0.4124,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.996059775352478,
      "rewards/margins": 1.3178468942642212,
      "rewards/rejected": -3.3139069080352783,
      "step": 340
    },
    {
      "epoch": 0.26,
      "grad_norm": 51.63470007658604,
      "learning_rate": 4.621112500950678e-07,
      "logits/chosen": -1.4794889688491821,
      "logits/rejected": -1.3635644912719727,
      "logps/chosen": -392.33319091796875,
      "logps/rejected": -509.9889221191406,
      "loss": 0.3917,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.6687583923339844,
      "rewards/margins": 1.3096448183059692,
      "rewards/rejected": -2.9784035682678223,
      "step": 350
    },
    {
      "epoch": 0.27,
      "grad_norm": 49.136281330589576,
      "learning_rate": 4.5860757779908225e-07,
      "logits/chosen": -1.3447411060333252,
      "logits/rejected": -1.1584227085113525,
      "logps/chosen": -392.1809997558594,
      "logps/rejected": -517.8204345703125,
      "loss": 0.4184,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.6987504959106445,
      "rewards/margins": 1.4527620077133179,
      "rewards/rejected": -3.151512384414673,
      "step": 360
    },
    {
      "epoch": 0.27,
      "grad_norm": 54.14810763202607,
      "learning_rate": 4.5496352164204304e-07,
      "logits/chosen": -1.0756736993789673,
      "logits/rejected": -0.9471368789672852,
      "logps/chosen": -381.1976013183594,
      "logps/rejected": -491.38787841796875,
      "loss": 0.4015,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.6106783151626587,
      "rewards/margins": 1.3218671083450317,
      "rewards/rejected": -2.9325456619262695,
      "step": 370
    },
    {
      "epoch": 0.28,
      "grad_norm": 61.80294176300012,
      "learning_rate": 4.5118153391584966e-07,
      "logits/chosen": -0.9351271390914917,
      "logits/rejected": -0.686697244644165,
      "logps/chosen": -431.38116455078125,
      "logps/rejected": -523.0051879882812,
      "loss": 0.3966,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.933549165725708,
      "rewards/margins": 1.2298634052276611,
      "rewards/rejected": -3.1634128093719482,
      "step": 380
    },
    {
      "epoch": 0.29,
      "grad_norm": 49.95605603054853,
      "learning_rate": 4.472641597343713e-07,
      "logits/chosen": -0.7354714274406433,
      "logits/rejected": -0.6010663509368896,
      "logps/chosen": -441.1346740722656,
      "logps/rejected": -589.41064453125,
      "loss": 0.4152,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -2.0311450958251953,
      "rewards/margins": 1.5603392124176025,
      "rewards/rejected": -3.5914840698242188,
      "step": 390
    },
    {
      "epoch": 0.3,
      "grad_norm": 51.04567852493441,
      "learning_rate": 4.4321403532069523e-07,
      "logits/chosen": -0.6377015113830566,
      "logits/rejected": -0.3965984880924225,
      "logps/chosen": -387.339111328125,
      "logps/rejected": -515.1150512695312,
      "loss": 0.3828,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.7281858921051025,
      "rewards/margins": 1.4771182537078857,
      "rewards/rejected": -3.205303907394409,
      "step": 400
    },
    {
      "epoch": 0.3,
      "eval_logits/chosen": -1.1203975677490234,
      "eval_logits/rejected": -0.9529741406440735,
      "eval_logps/chosen": -448.1488037109375,
      "eval_logps/rejected": -563.927978515625,
      "eval_loss": 0.523780882358551,
      "eval_rewards/accuracies": 0.76953125,
      "eval_rewards/chosen": -1.6651523113250732,
      "eval_rewards/margins": 1.096511721611023,
      "eval_rewards/rejected": -2.7616641521453857,
      "eval_runtime": 98.0838,
      "eval_samples_per_second": 20.391,
      "eval_steps_per_second": 0.326,
      "step": 400
    },
    {
      "epoch": 0.3,
      "grad_norm": 48.086291234203195,
      "learning_rate": 4.390338862330631e-07,
      "logits/chosen": -0.7085340619087219,
      "logits/rejected": -0.5028501749038696,
      "logps/chosen": -433.546142578125,
      "logps/rejected": -573.782958984375,
      "loss": 0.3738,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.8388134241104126,
      "rewards/margins": 1.763929009437561,
      "rewards/rejected": -3.6027424335479736,
      "step": 410
    },
    {
      "epoch": 0.31,
      "grad_norm": 64.03484534800386,
      "learning_rate": 4.3472652553068835e-07,
      "logits/chosen": -0.47518905997276306,
      "logits/rejected": -0.23270010948181152,
      "logps/chosen": -484.5774841308594,
      "logps/rejected": -611.3958740234375,
      "loss": 0.3739,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -2.291532039642334,
      "rewards/margins": 1.4785693883895874,
      "rewards/rejected": -3.770102024078369,
      "step": 420
    },
    {
      "epoch": 0.32,
      "grad_norm": 52.3682956987419,
      "learning_rate": 4.3029485188068895e-07,
      "logits/chosen": -0.6198094487190247,
      "logits/rejected": -0.28696033358573914,
      "logps/chosen": -403.52703857421875,
      "logps/rejected": -581.9299926757812,
      "loss": 0.3654,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.8076034784317017,
      "rewards/margins": 1.710561990737915,
      "rewards/rejected": -3.5181660652160645,
      "step": 430
    },
    {
      "epoch": 0.33,
      "grad_norm": 52.85823705565832,
      "learning_rate": 4.257418476074103e-07,
      "logits/chosen": -0.6958103179931641,
      "logits/rejected": -0.40623918175697327,
      "logps/chosen": -442.66473388671875,
      "logps/rejected": -562.0119018554688,
      "loss": 0.3841,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.8445909023284912,
      "rewards/margins": 1.4965617656707764,
      "rewards/rejected": -3.3411526679992676,
      "step": 440
    },
    {
      "epoch": 0.33,
      "grad_norm": 58.890892905451516,
      "learning_rate": 4.210705766854504e-07,
      "logits/chosen": -0.8186171650886536,
      "logits/rejected": -0.4163144528865814,
      "logps/chosen": -412.9081115722656,
      "logps/rejected": -538.0160522460938,
      "loss": 0.3763,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.5352731943130493,
      "rewards/margins": 1.580110788345337,
      "rewards/rejected": -3.1153836250305176,
      "step": 450
    },
    {
      "epoch": 0.34,
      "grad_norm": 54.82348692428555,
      "learning_rate": 4.16284182677737e-07,
      "logits/chosen": -0.5104752779006958,
      "logits/rejected": -0.022202759981155396,
      "logps/chosen": -504.6559143066406,
      "logps/rejected": -638.7310180664062,
      "loss": 0.3747,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -2.382429599761963,
      "rewards/margins": 1.8315938711166382,
      "rewards/rejected": -4.214023590087891,
      "step": 460
    },
    {
      "epoch": 0.35,
      "grad_norm": 54.979690542314074,
      "learning_rate": 4.113858866200466e-07,
      "logits/chosen": -0.7018298506736755,
      "logits/rejected": -0.30557408928871155,
      "logps/chosen": -490.76947021484375,
      "logps/rejected": -648.2581787109375,
      "loss": 0.3667,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.0751655101776123,
      "rewards/margins": 1.8448684215545654,
      "rewards/rejected": -3.9200336933135986,
      "step": 470
    },
    {
      "epoch": 0.36,
      "grad_norm": 52.384860047470646,
      "learning_rate": 4.063789848533865e-07,
      "logits/chosen": -0.5765314698219299,
      "logits/rejected": -0.1146412268280983,
      "logps/chosen": -431.3514709472656,
      "logps/rejected": -595.962646484375,
      "loss": 0.364,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.9231284856796265,
      "rewards/margins": 1.8943427801132202,
      "rewards/rejected": -3.8174710273742676,
      "step": 480
    },
    {
      "epoch": 0.36,
      "grad_norm": 48.24790366086697,
      "learning_rate": 4.0126684680570074e-07,
      "logits/chosen": -0.8232128024101257,
      "logits/rejected": -0.3200731873512268,
      "logps/chosen": -487.1802673339844,
      "logps/rejected": -620.1703491210938,
      "loss": 0.3632,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.8669496774673462,
      "rewards/margins": 1.7825498580932617,
      "rewards/rejected": -3.6494994163513184,
      "step": 490
    },
    {
      "epoch": 0.37,
      "grad_norm": 53.47941403954316,
      "learning_rate": 3.960529127243902e-07,
      "logits/chosen": -0.6839994192123413,
      "logits/rejected": -0.3794109523296356,
      "logps/chosen": -454.06494140625,
      "logps/rejected": -560.31396484375,
      "loss": 0.3576,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.671570062637329,
      "rewards/margins": 1.5296937227249146,
      "rewards/rejected": -3.201263904571533,
      "step": 500
    },
    {
      "epoch": 0.37,
      "eval_logits/chosen": -1.1202046871185303,
      "eval_logits/rejected": -0.8921781182289124,
      "eval_logps/chosen": -444.0173034667969,
      "eval_logps/rejected": -560.5327758789062,
      "eval_loss": 0.5183658599853516,
      "eval_rewards/accuracies": 0.76953125,
      "eval_rewards/chosen": -1.6238377094268799,
      "eval_rewards/margins": 1.103874921798706,
      "eval_rewards/rejected": -2.727712631225586,
      "eval_runtime": 98.0942,
      "eval_samples_per_second": 20.389,
      "eval_steps_per_second": 0.326,
      "step": 500
    },
    {
      "epoch": 0.38,
      "grad_norm": 47.67862520127771,
      "learning_rate": 3.9074069136117594e-07,
      "logits/chosen": -0.6645992994308472,
      "logits/rejected": -0.23552604019641876,
      "logps/chosen": -437.04168701171875,
      "logps/rejected": -573.43017578125,
      "loss": 0.3616,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -2.357367753982544,
      "rewards/margins": 1.7008861303329468,
      "rewards/rejected": -4.058254241943359,
      "step": 510
    },
    {
      "epoch": 0.39,
      "grad_norm": 55.20432992821677,
      "learning_rate": 3.8533375761086094e-07,
      "logits/chosen": -0.6538324356079102,
      "logits/rejected": -0.28878337144851685,
      "logps/chosen": -424.3060607910156,
      "logps/rejected": -554.125732421875,
      "loss": 0.3651,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.6924982070922852,
      "rewards/margins": 1.4187054634094238,
      "rewards/rejected": -3.111203670501709,
      "step": 520
    },
    {
      "epoch": 0.39,
      "grad_norm": 48.99836031219826,
      "learning_rate": 3.79835750105581e-07,
      "logits/chosen": -0.5227429270744324,
      "logits/rejected": -0.26728391647338867,
      "logps/chosen": -419.548095703125,
      "logps/rejected": -586.7620849609375,
      "loss": 0.3642,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.778322458267212,
      "rewards/margins": 1.765242338180542,
      "rewards/rejected": -3.543564558029175,
      "step": 530
    },
    {
      "epoch": 0.4,
      "grad_norm": 58.08597953496209,
      "learning_rate": 3.742503687661627e-07,
      "logits/chosen": -0.20574072003364563,
      "logits/rejected": 0.2035825252532959,
      "logps/chosen": -465.63763427734375,
      "logps/rejected": -606.3990478515625,
      "loss": 0.3525,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.341270923614502,
      "rewards/margins": 1.657379388809204,
      "rewards/rejected": -3.998650312423706,
      "step": 540
    },
    {
      "epoch": 0.41,
      "grad_norm": 45.74418223365167,
      "learning_rate": 3.685813723122372e-07,
      "logits/chosen": -0.06602563709020615,
      "logits/rejected": 0.41205328702926636,
      "logps/chosen": -458.8877868652344,
      "logps/rejected": -588.2848510742188,
      "loss": 0.374,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -2.2084052562713623,
      "rewards/margins": 1.5406155586242676,
      "rewards/rejected": -3.74902081489563,
      "step": 550
    },
    {
      "epoch": 0.42,
      "grad_norm": 49.47898611306175,
      "learning_rate": 3.6283257573278466e-07,
      "logits/chosen": -0.08754973113536835,
      "logits/rejected": 0.23806925117969513,
      "logps/chosen": -423.28985595703125,
      "logps/rejected": -584.5783081054688,
      "loss": 0.3585,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.8439204692840576,
      "rewards/margins": 1.726400375366211,
      "rewards/rejected": -3.5703208446502686,
      "step": 560
    },
    {
      "epoch": 0.42,
      "grad_norm": 59.10150187997885,
      "learning_rate": 3.5700784771881224e-07,
      "logits/chosen": 0.15894003212451935,
      "logits/rejected": 0.4934872090816498,
      "logps/chosen": -443.24053955078125,
      "logps/rejected": -601.0978393554688,
      "loss": 0.3442,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.163815975189209,
      "rewards/margins": 1.6688892841339111,
      "rewards/rejected": -3.83270525932312,
      "step": 570
    },
    {
      "epoch": 0.43,
      "grad_norm": 54.44170466261096,
      "learning_rate": 3.511111080598925e-07,
      "logits/chosen": 0.004490786697715521,
      "logits/rejected": 0.34507110714912415,
      "logps/chosen": -441.2269592285156,
      "logps/rejected": -602.7559814453125,
      "loss": 0.3492,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.914200782775879,
      "rewards/margins": 1.9196977615356445,
      "rewards/rejected": -3.8338985443115234,
      "step": 580
    },
    {
      "epoch": 0.44,
      "grad_norm": 53.89900166712047,
      "learning_rate": 3.451463250063146e-07,
      "logits/chosen": -0.27837514877319336,
      "logits/rejected": 0.13625089824199677,
      "logps/chosen": -448.6805725097656,
      "logps/rejected": -645.89013671875,
      "loss": 0.324,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -1.93107008934021,
      "rewards/margins": 2.10721755027771,
      "rewards/rejected": -4.038287162780762,
      "step": 590
    },
    {
      "epoch": 0.45,
      "grad_norm": 51.7074718432086,
      "learning_rate": 3.3911751259862403e-07,
      "logits/chosen": -0.041231829673051834,
      "logits/rejected": 0.34056779742240906,
      "logps/chosen": -443.47412109375,
      "logps/rejected": -629.924560546875,
      "loss": 0.3328,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.894299864768982,
      "rewards/margins": 2.1391186714172363,
      "rewards/rejected": -4.033418655395508,
      "step": 600
    },
    {
      "epoch": 0.45,
      "eval_logits/chosen": -0.06941252946853638,
      "eval_logits/rejected": 0.24228118360042572,
      "eval_logps/chosen": -493.6551513671875,
      "eval_logps/rejected": -628.6859130859375,
      "eval_loss": 0.5151087641716003,
      "eval_rewards/accuracies": 0.765625,
      "eval_rewards/chosen": -2.1202151775360107,
      "eval_rewards/margins": 1.2890279293060303,
      "eval_rewards/rejected": -3.409243106842041,
      "eval_runtime": 98.1083,
      "eval_samples_per_second": 20.386,
      "eval_steps_per_second": 0.326,
      "step": 600
    },
    {
      "epoch": 0.45,
      "grad_norm": 55.66397888256945,
      "learning_rate": 3.3302872796634754e-07,
      "logits/chosen": 0.49535948038101196,
      "logits/rejected": 1.0743920803070068,
      "logps/chosen": -468.02978515625,
      "logps/rejected": -642.6104736328125,
      "loss": 0.3096,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.297302722930908,
      "rewards/margins": 2.0772812366485596,
      "rewards/rejected": -4.374583721160889,
      "step": 610
    },
    {
      "epoch": 0.46,
      "grad_norm": 51.7383697331199,
      "learning_rate": 3.2688406859772035e-07,
      "logits/chosen": 0.2597588300704956,
      "logits/rejected": 0.673038125038147,
      "logps/chosen": -444.8384704589844,
      "logps/rejected": -659.6806030273438,
      "loss": 0.3346,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.2546586990356445,
      "rewards/margins": 2.226531744003296,
      "rewards/rejected": -4.4811906814575195,
      "step": 620
    },
    {
      "epoch": 0.47,
      "grad_norm": 54.19919430150042,
      "learning_rate": 3.206876695822541e-07,
      "logits/chosen": -0.12093131244182587,
      "logits/rejected": 0.4730301797389984,
      "logps/chosen": -452.8343811035156,
      "logps/rejected": -602.112060546875,
      "loss": 0.3051,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -1.8909651041030884,
      "rewards/margins": 1.9853994846343994,
      "rewards/rejected": -3.876364231109619,
      "step": 630
    },
    {
      "epoch": 0.48,
      "grad_norm": 50.389338383353866,
      "learning_rate": 3.144437008280012e-07,
      "logits/chosen": -0.09394478052854538,
      "logits/rejected": 0.33072829246520996,
      "logps/chosen": -455.43731689453125,
      "logps/rejected": -609.87744140625,
      "loss": 0.3263,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.1607372760772705,
      "rewards/margins": 1.885525107383728,
      "rewards/rejected": -4.046262264251709,
      "step": 640
    },
    {
      "epoch": 0.48,
      "grad_norm": 61.479087062458596,
      "learning_rate": 3.0815636425538665e-07,
      "logits/chosen": 0.20981228351593018,
      "logits/rejected": 0.55927574634552,
      "logps/chosen": -474.67803955078125,
      "logps/rejected": -664.0802001953125,
      "loss": 0.3204,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.3460288047790527,
      "rewards/margins": 2.071723222732544,
      "rewards/rejected": -4.417751789093018,
      "step": 650
    },
    {
      "epoch": 0.49,
      "grad_norm": 69.05844503540243,
      "learning_rate": 3.018298909694986e-07,
      "logits/chosen": 0.25689777731895447,
      "logits/rejected": 0.7616270184516907,
      "logps/chosen": -470.198486328125,
      "logps/rejected": -662.539306640625,
      "loss": 0.3289,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.4795608520507812,
      "rewards/margins": 1.899802565574646,
      "rewards/rejected": -4.379363059997559,
      "step": 660
    },
    {
      "epoch": 0.5,
      "grad_norm": 45.7911918343519,
      "learning_rate": 2.954685384127371e-07,
      "logits/chosen": 0.062225330621004105,
      "logits/rejected": 0.41303902864456177,
      "logps/chosen": -428.16387939453125,
      "logps/rejected": -597.1376953125,
      "loss": 0.306,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.1429684162139893,
      "rewards/margins": 1.8161710500717163,
      "rewards/rejected": -3.959139347076416,
      "step": 670
    },
    {
      "epoch": 0.51,
      "grad_norm": 57.36445697043156,
      "learning_rate": 2.8907658749974054e-07,
      "logits/chosen": -0.09550069272518158,
      "logits/rejected": 0.35600176453590393,
      "logps/chosen": -455.721923828125,
      "logps/rejected": -670.6884765625,
      "loss": 0.2969,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.200000762939453,
      "rewards/margins": 2.278864622116089,
      "rewards/rejected": -4.478865146636963,
      "step": 680
    },
    {
      "epoch": 0.51,
      "grad_norm": 57.63859566762506,
      "learning_rate": 2.8265833973651503e-07,
      "logits/chosen": 0.4362770915031433,
      "logits/rejected": 1.019630789756775,
      "logps/chosen": -481.36944580078125,
      "logps/rejected": -682.4991455078125,
      "loss": 0.3037,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.728414297103882,
      "rewards/margins": 2.1998586654663086,
      "rewards/rejected": -4.928272724151611,
      "step": 690
    },
    {
      "epoch": 0.52,
      "grad_norm": 38.04349040823844,
      "learning_rate": 2.7621811432570736e-07,
      "logits/chosen": 0.0020432949531823397,
      "logits/rejected": 0.678579568862915,
      "logps/chosen": -452.239990234375,
      "logps/rejected": -615.9015502929688,
      "loss": 0.3131,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.9651463031768799,
      "rewards/margins": 2.032357931137085,
      "rewards/rejected": -3.997504472732544,
      "step": 700
    },
    {
      "epoch": 0.52,
      "eval_logits/chosen": -0.1655656397342682,
      "eval_logits/rejected": 0.172866553068161,
      "eval_logps/chosen": -451.9695739746094,
      "eval_logps/rejected": -578.1397705078125,
      "eval_loss": 0.5152586698532104,
      "eval_rewards/accuracies": 0.765625,
      "eval_rewards/chosen": -1.7033603191375732,
      "eval_rewards/margins": 1.2004213333129883,
      "eval_rewards/rejected": -2.9037814140319824,
      "eval_runtime": 98.1096,
      "eval_samples_per_second": 20.385,
      "eval_steps_per_second": 0.326,
      "step": 700
    },
    {
      "epoch": 0.53,
      "grad_norm": 60.9670305167988,
      "learning_rate": 2.6976024525996917e-07,
      "logits/chosen": 0.702204167842865,
      "logits/rejected": 0.9482040405273438,
      "logps/chosen": -400.1419372558594,
      "logps/rejected": -616.5157470703125,
      "loss": 0.2933,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.058786630630493,
      "rewards/margins": 2.1224417686462402,
      "rewards/rejected": -4.1812286376953125,
      "step": 710
    },
    {
      "epoch": 0.53,
      "grad_norm": 57.54299069526284,
      "learning_rate": 2.6328907840536706e-07,
      "logits/chosen": 0.6100508570671082,
      "logits/rejected": 0.9714565277099609,
      "logps/chosen": -446.7682189941406,
      "logps/rejected": -678.0450439453125,
      "loss": 0.312,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -2.1031315326690674,
      "rewards/margins": 2.523500680923462,
      "rewards/rejected": -4.626631736755371,
      "step": 720
    },
    {
      "epoch": 0.54,
      "grad_norm": 55.010003089665325,
      "learning_rate": 2.568089685768038e-07,
      "logits/chosen": 0.33207422494888306,
      "logits/rejected": 0.8213400840759277,
      "logps/chosen": -466.47967529296875,
      "logps/rejected": -616.5966796875,
      "loss": 0.3037,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.145937442779541,
      "rewards/margins": 1.8470547199249268,
      "rewards/rejected": -3.992992401123047,
      "step": 730
    },
    {
      "epoch": 0.55,
      "grad_norm": 56.034615355832116,
      "learning_rate": 2.503242766074156e-07,
      "logits/chosen": 1.0638476610183716,
      "logits/rejected": 1.7732141017913818,
      "logps/chosen": -467.27789306640625,
      "logps/rejected": -673.653564453125,
      "loss": 0.3135,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.2686350345611572,
      "rewards/margins": 2.4400157928466797,
      "rewards/rejected": -4.708651542663574,
      "step": 740
    },
    {
      "epoch": 0.56,
      "grad_norm": 55.33213405677105,
      "learning_rate": 2.4383936641392136e-07,
      "logits/chosen": 0.8964494466781616,
      "logits/rejected": 1.4589967727661133,
      "logps/chosen": -475.98529052734375,
      "logps/rejected": -655.1957397460938,
      "loss": 0.3094,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.501713514328003,
      "rewards/margins": 2.241748094558716,
      "rewards/rejected": -4.743461608886719,
      "step": 750
    },
    {
      "epoch": 0.56,
      "grad_norm": 56.701790611883894,
      "learning_rate": 2.3735860205989493e-07,
      "logits/chosen": 0.8617841005325317,
      "logits/rejected": 1.3072969913482666,
      "logps/chosen": -474.41583251953125,
      "logps/rejected": -685.8687744140625,
      "loss": 0.2749,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.408444881439209,
      "rewards/margins": 2.277350425720215,
      "rewards/rejected": -4.685795307159424,
      "step": 760
    },
    {
      "epoch": 0.57,
      "grad_norm": 53.32774327458172,
      "learning_rate": 2.308863448189402e-07,
      "logits/chosen": 0.5902019739151001,
      "logits/rejected": 0.8791207075119019,
      "logps/chosen": -452.8211364746094,
      "logps/rejected": -647.130126953125,
      "loss": 0.3028,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -2.1281752586364746,
      "rewards/margins": 2.103821277618408,
      "rewards/rejected": -4.231996536254883,
      "step": 770
    },
    {
      "epoch": 0.58,
      "grad_norm": 66.84990640273234,
      "learning_rate": 2.2442695023974246e-07,
      "logits/chosen": 0.7262977361679077,
      "logits/rejected": 1.3522447347640991,
      "logps/chosen": -426.29339599609375,
      "logps/rejected": -622.6487426757812,
      "loss": 0.289,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.120136022567749,
      "rewards/margins": 2.228008985519409,
      "rewards/rejected": -4.348145008087158,
      "step": 780
    },
    {
      "epoch": 0.59,
      "grad_norm": 58.09209476264956,
      "learning_rate": 2.179847652149729e-07,
      "logits/chosen": 0.5433401465415955,
      "logits/rejected": 1.0540757179260254,
      "logps/chosen": -516.3043212890625,
      "logps/rejected": -711.8958740234375,
      "loss": 0.2875,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.7564001083374023,
      "rewards/margins": 2.2523038387298584,
      "rewards/rejected": -5.00870418548584,
      "step": 790
    },
    {
      "epoch": 0.59,
      "grad_norm": 51.81489549587344,
      "learning_rate": 2.115641250560183e-07,
      "logits/chosen": 0.46662241220474243,
      "logits/rejected": 0.9530113339424133,
      "logps/chosen": -531.3187866210938,
      "logps/rejected": -718.2420043945312,
      "loss": 0.2547,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.6745731830596924,
      "rewards/margins": 2.1868960857391357,
      "rewards/rejected": -4.86146879196167,
      "step": 800
    },
    {
      "epoch": 0.59,
      "eval_logits/chosen": 0.12700016796588898,
      "eval_logits/rejected": 0.44755151867866516,
      "eval_logps/chosen": -535.29150390625,
      "eval_logps/rejected": -673.45654296875,
      "eval_loss": 0.5256171822547913,
      "eval_rewards/accuracies": 0.76171875,
      "eval_rewards/chosen": -2.5365793704986572,
      "eval_rewards/margins": 1.3203709125518799,
      "eval_rewards/rejected": -3.856950521469116,
      "eval_runtime": 98.0354,
      "eval_samples_per_second": 20.401,
      "eval_steps_per_second": 0.326,
      "step": 800
    },
    {
      "epoch": 0.6,
      "grad_norm": 54.66675021265227,
      "learning_rate": 2.051693505755042e-07,
      "logits/chosen": 0.33729100227355957,
      "logits/rejected": 0.9711275100708008,
      "logps/chosen": -483.88861083984375,
      "logps/rejected": -701.2562255859375,
      "loss": 0.2491,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.316730260848999,
      "rewards/margins": 2.6504831314086914,
      "rewards/rejected": -4.9672136306762695,
      "step": 810
    },
    {
      "epoch": 0.61,
      "grad_norm": 57.83387891787277,
      "learning_rate": 1.9880474517957542e-07,
      "logits/chosen": 0.38913315534591675,
      "logits/rejected": 0.8929936289787292,
      "logps/chosen": -511.2947692871094,
      "logps/rejected": -719.1864013671875,
      "loss": 0.2861,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.5449912548065186,
      "rewards/margins": 2.4298605918884277,
      "rewards/rejected": -4.974852085113525,
      "step": 820
    },
    {
      "epoch": 0.62,
      "grad_norm": 54.573496843465676,
      "learning_rate": 1.9247459197189e-07,
      "logits/chosen": 0.2409726083278656,
      "logits/rejected": 0.4742881655693054,
      "logps/chosen": -487.2377014160156,
      "logps/rejected": -707.2930908203125,
      "loss": 0.2709,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.423832416534424,
      "rewards/margins": 2.337522029876709,
      "rewards/rejected": -4.761354446411133,
      "step": 830
    },
    {
      "epoch": 0.62,
      "grad_norm": 57.59285615682952,
      "learning_rate": 1.8618315087127602e-07,
      "logits/chosen": -0.03551248461008072,
      "logits/rejected": 0.5289381146430969,
      "logps/chosen": -485.76348876953125,
      "logps/rejected": -648.2017822265625,
      "loss": 0.2908,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.2680952548980713,
      "rewards/margins": 2.0456957817077637,
      "rewards/rejected": -4.313790321350098,
      "step": 840
    },
    {
      "epoch": 0.63,
      "grad_norm": 73.30688641890157,
      "learning_rate": 1.7993465574499102e-07,
      "logits/chosen": -0.13685956597328186,
      "logits/rejected": 0.5006009340286255,
      "logps/chosen": -509.0750427246094,
      "logps/rejected": -739.2849731445312,
      "loss": 0.2572,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.391108512878418,
      "rewards/margins": 2.590216875076294,
      "rewards/rejected": -4.981324672698975,
      "step": 850
    },
    {
      "epoch": 0.64,
      "grad_norm": 60.15832089933168,
      "learning_rate": 1.7373331155951233e-07,
      "logits/chosen": 0.16482150554656982,
      "logits/rejected": 0.5440123677253723,
      "logps/chosen": -488.3111877441406,
      "logps/rejected": -694.9370727539062,
      "loss": 0.2517,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.627448558807373,
      "rewards/margins": 2.2580535411834717,
      "rewards/rejected": -4.885501861572266,
      "step": 860
    },
    {
      "epoch": 0.65,
      "grad_norm": 48.979507346317426,
      "learning_rate": 1.6758329155077743e-07,
      "logits/chosen": 0.2300490438938141,
      "logits/rejected": 0.6597995758056641,
      "logps/chosen": -471.4276428222656,
      "logps/rejected": -696.9656372070312,
      "loss": 0.2572,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -2.564178943634033,
      "rewards/margins": 2.4548990726470947,
      "rewards/rejected": -5.019078254699707,
      "step": 870
    },
    {
      "epoch": 0.65,
      "grad_norm": 54.56547566856159,
      "learning_rate": 1.6148873441577662e-07,
      "logits/chosen": 0.0745859146118164,
      "logits/rejected": 0.5892910361289978,
      "logps/chosen": -490.7748107910156,
      "logps/rejected": -704.1431884765625,
      "loss": 0.2334,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.500243663787842,
      "rewards/margins": 2.4989378452301025,
      "rewards/rejected": -4.999181270599365,
      "step": 880
    },
    {
      "epoch": 0.66,
      "grad_norm": 55.545059461777754,
      "learning_rate": 1.5545374152738934e-07,
      "logits/chosen": 0.016130054369568825,
      "logits/rejected": 0.48376768827438354,
      "logps/chosen": -458.7793884277344,
      "logps/rejected": -710.6851196289062,
      "loss": 0.25,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.172322988510132,
      "rewards/margins": 2.6098039150238037,
      "rewards/rejected": -4.782127380371094,
      "step": 890
    },
    {
      "epoch": 0.67,
      "grad_norm": 58.8472459822313,
      "learning_rate": 1.4948237417433775e-07,
      "logits/chosen": -0.05456262826919556,
      "logits/rejected": 0.4301723539829254,
      "logps/chosen": -505.43719482421875,
      "logps/rejected": -729.1197509765625,
      "loss": 0.2764,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.475316286087036,
      "rewards/margins": 2.53151535987854,
      "rewards/rejected": -5.006831169128418,
      "step": 900
    },
    {
      "epoch": 0.67,
      "eval_logits/chosen": -0.2431109994649887,
      "eval_logits/rejected": 0.051950518041849136,
      "eval_logps/chosen": -538.3812866210938,
      "eval_logps/rejected": -682.334228515625,
      "eval_loss": 0.5221121907234192,
      "eval_rewards/accuracies": 0.77734375,
      "eval_rewards/chosen": -2.567476749420166,
      "eval_rewards/margins": 1.3782495260238647,
      "eval_rewards/rejected": -3.9457266330718994,
      "eval_runtime": 98.1143,
      "eval_samples_per_second": 20.384,
      "eval_steps_per_second": 0.326,
      "step": 900
    },
    {
      "epoch": 0.68,
      "grad_norm": 76.04810070991684,
      "learning_rate": 1.435786508281158e-07,
      "logits/chosen": 0.3129107356071472,
      "logits/rejected": 0.6085219383239746,
      "logps/chosen": -462.0315856933594,
      "logps/rejected": -686.1160888671875,
      "loss": 0.2458,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.3812341690063477,
      "rewards/margins": 2.453599452972412,
      "rewards/rejected": -4.83483362197876,
      "step": 910
    },
    {
      "epoch": 0.68,
      "grad_norm": 58.24143150255672,
      "learning_rate": 1.3774654443873174e-07,
      "logits/chosen": -0.17105606198310852,
      "logits/rejected": 0.10635167360305786,
      "logps/chosen": -476.74884033203125,
      "logps/rejected": -704.75537109375,
      "loss": 0.2662,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.2945311069488525,
      "rewards/margins": 2.386672258377075,
      "rewards/rejected": -4.681203365325928,
      "step": 920
    },
    {
      "epoch": 0.69,
      "grad_norm": 61.580815446047275,
      "learning_rate": 1.31989979761085e-07,
      "logits/chosen": 0.19714362919330597,
      "logits/rejected": 0.6800551414489746,
      "logps/chosen": -470.64697265625,
      "logps/rejected": -731.5118408203125,
      "loss": 0.2524,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.3182313442230225,
      "rewards/margins": 2.8053975105285645,
      "rewards/rejected": -5.123629093170166,
      "step": 930
    },
    {
      "epoch": 0.7,
      "grad_norm": 66.23348721239962,
      "learning_rate": 1.2631283071377618e-07,
      "logits/chosen": 0.15574759244918823,
      "logits/rejected": 0.7999376058578491,
      "logps/chosen": -489.5287170410156,
      "logps/rejected": -739.4246215820312,
      "loss": 0.2535,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.4655532836914062,
      "rewards/margins": 2.7967965602874756,
      "rewards/rejected": -5.2623491287231445,
      "step": 940
    },
    {
      "epoch": 0.71,
      "grad_norm": 55.370463362822846,
      "learning_rate": 1.2071891777212744e-07,
      "logits/chosen": 0.4210018217563629,
      "logits/rejected": 1.0494170188903809,
      "logps/chosen": -442.0162658691406,
      "logps/rejected": -666.9036865234375,
      "loss": 0.238,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.3151867389678955,
      "rewards/margins": 2.4912314414978027,
      "rewards/rejected": -4.806417942047119,
      "step": 950
    },
    {
      "epoch": 0.71,
      "grad_norm": 54.37619099014209,
      "learning_rate": 1.1521200539716874e-07,
      "logits/chosen": 0.04333481192588806,
      "logits/rejected": 0.6315183639526367,
      "logps/chosen": -440.3460998535156,
      "logps/rejected": -668.5031127929688,
      "loss": 0.2538,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.369847297668457,
      "rewards/margins": 2.58512282371521,
      "rewards/rejected": -4.954970359802246,
      "step": 960
    },
    {
      "epoch": 0.72,
      "grad_norm": 59.50016505068154,
      "learning_rate": 1.0979579950231821e-07,
      "logits/chosen": -0.0721534788608551,
      "logits/rejected": 0.5839505195617676,
      "logps/chosen": -496.072021484375,
      "logps/rejected": -720.0166015625,
      "loss": 0.2331,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.456573963165283,
      "rewards/margins": 2.6809329986572266,
      "rewards/rejected": -5.13750696182251,
      "step": 970
    },
    {
      "epoch": 0.73,
      "grad_norm": 63.83570606232123,
      "learning_rate": 1.0447394495946291e-07,
      "logits/chosen": 0.007242382504045963,
      "logits/rejected": 0.5322221517562866,
      "logps/chosen": -520.4622192382812,
      "logps/rejected": -716.791015625,
      "loss": 0.2726,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.6428182125091553,
      "rewards/margins": 2.505241870880127,
      "rewards/rejected": -5.148059844970703,
      "step": 980
    },
    {
      "epoch": 0.74,
      "grad_norm": 76.60392151494439,
      "learning_rate": 9.925002314611841e-08,
      "logits/chosen": -0.2113700807094574,
      "logits/rejected": 0.4387185573577881,
      "logps/chosen": -535.4547119140625,
      "logps/rejected": -757.9600830078125,
      "loss": 0.2356,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.6833412647247314,
      "rewards/margins": 2.5519299507141113,
      "rewards/rejected": -5.235270977020264,
      "step": 990
    },
    {
      "epoch": 0.74,
      "grad_norm": 55.208058181749024,
      "learning_rate": 9.412754953531663e-08,
      "logits/chosen": 0.000503048300743103,
      "logits/rejected": 0.7104528546333313,
      "logps/chosen": -494.1576232910156,
      "logps/rejected": -719.8685913085938,
      "loss": 0.2261,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.6024506092071533,
      "rewards/margins": 2.5579113960266113,
      "rewards/rejected": -5.160361289978027,
      "step": 1000
    },
    {
      "epoch": 0.74,
      "eval_logits/chosen": -0.1104351058602333,
      "eval_logits/rejected": 0.20232482254505157,
      "eval_logps/chosen": -558.2006225585938,
      "eval_logps/rejected": -712.748291015625,
      "eval_loss": 0.5298039317131042,
      "eval_rewards/accuracies": 0.76953125,
      "eval_rewards/chosen": -2.7656705379486084,
      "eval_rewards/margins": 1.4841958284378052,
      "eval_rewards/rejected": -4.249866485595703,
      "eval_runtime": 98.1214,
      "eval_samples_per_second": 20.383,
      "eval_steps_per_second": 0.326,
      "step": 1000
    },
    {
      "epoch": 0.75,
      "grad_norm": 57.330660352565694,
      "learning_rate": 8.910997132984479e-08,
      "logits/chosen": 0.19682852923870087,
      "logits/rejected": 0.5228155255317688,
      "logps/chosen": -529.1737060546875,
      "logps/rejected": -791.169921875,
      "loss": 0.2475,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.5559616088867188,
      "rewards/margins": 2.631944179534912,
      "rewards/rejected": -5.187905311584473,
      "step": 1010
    },
    {
      "epoch": 0.76,
      "grad_norm": 55.457376945821615,
      "learning_rate": 8.42006651424274e-08,
      "logits/chosen": -0.11488042026758194,
      "logits/rejected": 0.43977147340774536,
      "logps/chosen": -529.4292602539062,
      "logps/rejected": -779.668212890625,
      "loss": 0.2315,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.6139321327209473,
      "rewards/margins": 2.7150697708129883,
      "rewards/rejected": -5.3290019035339355,
      "step": 1020
    },
    {
      "epoch": 0.77,
      "grad_norm": 57.42354594975122,
      "learning_rate": 7.940293472341217e-08,
      "logits/chosen": 0.25594162940979004,
      "logits/rejected": 0.661308228969574,
      "logps/chosen": -465.62347412109375,
      "logps/rejected": -748.9591064453125,
      "loss": 0.2417,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.5969550609588623,
      "rewards/margins": 2.9456839561462402,
      "rewards/rejected": -5.542638778686523,
      "step": 1030
    },
    {
      "epoch": 0.77,
      "grad_norm": 44.59552618611705,
      "learning_rate": 7.472000873748918e-08,
      "logits/chosen": 0.21030166745185852,
      "logits/rejected": 0.7939587831497192,
      "logps/chosen": -488.8642578125,
      "logps/rejected": -709.2621459960938,
      "loss": 0.2323,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -2.7148165702819824,
      "rewards/margins": 2.5755038261413574,
      "rewards/rejected": -5.29032039642334,
      "step": 1040
    },
    {
      "epoch": 0.78,
      "grad_norm": 64.60445708831573,
      "learning_rate": 7.015503859093927e-08,
      "logits/chosen": 0.2117297202348709,
      "logits/rejected": 0.6777232885360718,
      "logps/chosen": -483.72283935546875,
      "logps/rejected": -733.4895629882812,
      "loss": 0.2268,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.7278270721435547,
      "rewards/margins": 2.6605074405670166,
      "rewards/rejected": -5.38833475112915,
      "step": 1050
    },
    {
      "epoch": 0.79,
      "grad_norm": 55.01169146748481,
      "learning_rate": 6.571109631087451e-08,
      "logits/chosen": 0.0924178883433342,
      "logits/rejected": 0.8385933041572571,
      "logps/chosen": -520.66064453125,
      "logps/rejected": -789.1815185546875,
      "loss": 0.2073,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.671236515045166,
      "rewards/margins": 3.076190710067749,
      "rewards/rejected": -5.747427463531494,
      "step": 1060
    },
    {
      "epoch": 0.79,
      "grad_norm": 57.63704573643993,
      "learning_rate": 6.139117247789687e-08,
      "logits/chosen": 0.09710516035556793,
      "logits/rejected": 0.5915436148643494,
      "logps/chosen": -536.4342651367188,
      "logps/rejected": -775.2525634765625,
      "loss": 0.209,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.6840100288391113,
      "rewards/margins": 2.6324105262756348,
      "rewards/rejected": -5.316420555114746,
      "step": 1070
    },
    {
      "epoch": 0.8,
      "grad_norm": 65.7023707481576,
      "learning_rate": 5.719817421356685e-08,
      "logits/chosen": -0.050966888666152954,
      "logits/rejected": 0.3987392485141754,
      "logps/chosen": -558.064453125,
      "logps/rejected": -772.4810791015625,
      "loss": 0.229,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.901360034942627,
      "rewards/margins": 2.482283115386963,
      "rewards/rejected": -5.383642673492432,
      "step": 1080
    },
    {
      "epoch": 0.81,
      "grad_norm": 67.44035324151042,
      "learning_rate": 5.313492322403701e-08,
      "logits/chosen": 0.33903616666793823,
      "logits/rejected": 0.7725549936294556,
      "logps/chosen": -484.7267150878906,
      "logps/rejected": -763.6196899414062,
      "loss": 0.2302,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.789487838745117,
      "rewards/margins": 2.8590240478515625,
      "rewards/rejected": -5.6485114097595215,
      "step": 1090
    },
    {
      "epoch": 0.82,
      "grad_norm": 49.24699961134825,
      "learning_rate": 4.9204153901165805e-08,
      "logits/chosen": 0.14409589767456055,
      "logits/rejected": 0.7108417749404907,
      "logps/chosen": -508.53668212890625,
      "logps/rejected": -749.4099731445312,
      "loss": 0.2219,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.825862407684326,
      "rewards/margins": 2.6767187118530273,
      "rewards/rejected": -5.502581596374512,
      "step": 1100
    },
    {
      "epoch": 0.82,
      "eval_logits/chosen": -0.006662141531705856,
      "eval_logits/rejected": 0.307841420173645,
      "eval_logps/chosen": -591.4903564453125,
      "eval_logps/rejected": -754.2210693359375,
      "eval_loss": 0.5380497574806213,
      "eval_rewards/accuracies": 0.76953125,
      "eval_rewards/chosen": -3.0985679626464844,
      "eval_rewards/margins": 1.566027045249939,
      "eval_rewards/rejected": -4.664595127105713,
      "eval_runtime": 98.1364,
      "eval_samples_per_second": 20.38,
      "eval_steps_per_second": 0.326,
      "step": 1100
    },
    {
      "epoch": 0.82,
      "grad_norm": 53.77081112472508,
      "learning_rate": 4.540851148239036e-08,
      "logits/chosen": 0.05285036563873291,
      "logits/rejected": 0.5418864488601685,
      "logps/chosen": -480.589599609375,
      "logps/rejected": -721.0247192382812,
      "loss": 0.2259,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.6466457843780518,
      "rewards/margins": 2.60426664352417,
      "rewards/rejected": -5.250911712646484,
      "step": 1110
    },
    {
      "epoch": 0.83,
      "grad_norm": 61.18183448166774,
      "learning_rate": 4.1750550270596206e-08,
      "logits/chosen": 0.007505857851356268,
      "logits/rejected": 0.7265870571136475,
      "logps/chosen": -525.0111694335938,
      "logps/rejected": -770.3839111328125,
      "loss": 0.2062,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.770293712615967,
      "rewards/margins": 2.8563060760498047,
      "rewards/rejected": -5.6265997886657715,
      "step": 1120
    },
    {
      "epoch": 0.84,
      "grad_norm": 68.7383025915832,
      "learning_rate": 3.823273191518234e-08,
      "logits/chosen": 0.1847713440656662,
      "logits/rejected": 0.7002714276313782,
      "logps/chosen": -485.5477600097656,
      "logps/rejected": -744.1683959960938,
      "loss": 0.2208,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.557469606399536,
      "rewards/margins": 2.7433865070343018,
      "rewards/rejected": -5.300856113433838,
      "step": 1130
    },
    {
      "epoch": 0.85,
      "grad_norm": 52.91705405093606,
      "learning_rate": 3.485742375547745e-08,
      "logits/chosen": 0.2934524416923523,
      "logits/rejected": 0.7874400019645691,
      "logps/chosen": -540.8467407226562,
      "logps/rejected": -773.0992431640625,
      "loss": 0.2157,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.8220906257629395,
      "rewards/margins": 2.7147064208984375,
      "rewards/rejected": -5.536796569824219,
      "step": 1140
    },
    {
      "epoch": 0.85,
      "grad_norm": 71.88387361950765,
      "learning_rate": 3.162689722762365e-08,
      "logits/chosen": -0.06768418103456497,
      "logits/rejected": 0.4680994153022766,
      "logps/chosen": -544.25439453125,
      "logps/rejected": -792.7861328125,
      "loss": 0.219,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.854830026626587,
      "rewards/margins": 2.878235340118408,
      "rewards/rejected": -5.733065128326416,
      "step": 1150
    },
    {
      "epoch": 0.86,
      "grad_norm": 55.10045985903261,
      "learning_rate": 2.8543326335997904e-08,
      "logits/chosen": 0.22700032591819763,
      "logits/rejected": 0.5510324239730835,
      "logps/chosen": -525.5604248046875,
      "logps/rejected": -764.0299072265625,
      "loss": 0.2111,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.728689193725586,
      "rewards/margins": 2.7138609886169434,
      "rewards/rejected": -5.442550182342529,
      "step": 1160
    },
    {
      "epoch": 0.87,
      "grad_norm": 69.28988155945117,
      "learning_rate": 2.560878619020157e-08,
      "logits/chosen": 0.41892099380493164,
      "logits/rejected": 0.961225152015686,
      "logps/chosen": -475.5420837402344,
      "logps/rejected": -717.2962036132812,
      "loss": 0.2244,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.7839531898498535,
      "rewards/margins": 2.69148588180542,
      "rewards/rejected": -5.475439071655273,
      "step": 1170
    },
    {
      "epoch": 0.88,
      "grad_norm": 64.73986275241352,
      "learning_rate": 2.2825251608601466e-08,
      "logits/chosen": 0.1623622179031372,
      "logits/rejected": 0.6435664892196655,
      "logps/chosen": -536.0408935546875,
      "logps/rejected": -744.7503662109375,
      "loss": 0.2261,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.80417799949646,
      "rewards/margins": 2.5097708702087402,
      "rewards/rejected": -5.313948631286621,
      "step": 1180
    },
    {
      "epoch": 0.88,
      "grad_norm": 46.21048147117716,
      "learning_rate": 2.0194595789362474e-08,
      "logits/chosen": -0.05237337946891785,
      "logits/rejected": 0.4507738947868347,
      "logps/chosen": -519.0675048828125,
      "logps/rejected": -777.24072265625,
      "loss": 0.2245,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.453911542892456,
      "rewards/margins": 3.05096435546875,
      "rewards/rejected": -5.504876136779785,
      "step": 1190
    },
    {
      "epoch": 0.89,
      "grad_norm": 55.74231137792324,
      "learning_rate": 1.7718589049866728e-08,
      "logits/chosen": 0.32885435223579407,
      "logits/rejected": 0.45243850350379944,
      "logps/chosen": -502.6609802246094,
      "logps/rejected": -755.8111572265625,
      "loss": 0.2165,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.85229754447937,
      "rewards/margins": 2.4965226650238037,
      "rewards/rejected": -5.348820209503174,
      "step": 1200
    },
    {
      "epoch": 0.89,
      "eval_logits/chosen": -0.09797076135873795,
      "eval_logits/rejected": 0.20152458548545837,
      "eval_logps/chosen": -580.1854858398438,
      "eval_logps/rejected": -738.0179443359375,
      "eval_loss": 0.5336408615112305,
      "eval_rewards/accuracies": 0.76171875,
      "eval_rewards/chosen": -2.9855194091796875,
      "eval_rewards/margins": 1.5170438289642334,
      "eval_rewards/rejected": -4.5025634765625,
      "eval_runtime": 98.1382,
      "eval_samples_per_second": 20.379,
      "eval_steps_per_second": 0.326,
      "step": 1200
    },
    {
      "epoch": 0.9,
      "grad_norm": 59.87247578419851,
      "learning_rate": 1.539889763536645e-08,
      "logits/chosen": 0.1656707227230072,
      "logits/rejected": 0.5841933488845825,
      "logps/chosen": -501.6282653808594,
      "logps/rejected": -756.2093505859375,
      "loss": 0.2316,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.5563437938690186,
      "rewards/margins": 2.784249782562256,
      "rewards/rejected": -5.3405938148498535,
      "step": 1210
    },
    {
      "epoch": 0.91,
      "grad_norm": 60.03113730852447,
      "learning_rate": 1.3237082597673172e-08,
      "logits/chosen": 0.1520698368549347,
      "logits/rejected": 0.7296702861785889,
      "logps/chosen": -479.1258850097656,
      "logps/rejected": -756.1190185546875,
      "loss": 0.1913,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.538914680480957,
      "rewards/margins": 2.866367816925049,
      "rewards/rejected": -5.405282974243164,
      "step": 1220
    },
    {
      "epoch": 0.91,
      "grad_norm": 50.74907680854386,
      "learning_rate": 1.1234598744637502e-08,
      "logits/chosen": 0.06600452959537506,
      "logits/rejected": 0.34231507778167725,
      "logps/chosen": -536.6253051757812,
      "logps/rejected": -802.2919311523438,
      "loss": 0.2021,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.8277297019958496,
      "rewards/margins": 2.8031983375549316,
      "rewards/rejected": -5.630928039550781,
      "step": 1230
    },
    {
      "epoch": 0.92,
      "grad_norm": 53.04165278516392,
      "learning_rate": 9.392793661126414e-09,
      "logits/chosen": -0.1451832503080368,
      "logits/rejected": 0.4121910631656647,
      "logps/chosen": -569.254150390625,
      "logps/rejected": -809.6979370117188,
      "loss": 0.196,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.7587039470672607,
      "rewards/margins": 2.8314685821533203,
      "rewards/rejected": -5.590172290802002,
      "step": 1240
    },
    {
      "epoch": 0.93,
      "grad_norm": 64.9259915789077,
      "learning_rate": 7.71290680215711e-09,
      "logits/chosen": 0.08149626106023788,
      "logits/rejected": 0.6067185401916504,
      "logps/chosen": -518.8594970703125,
      "logps/rejected": -728.6957397460938,
      "loss": 0.227,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.8665976524353027,
      "rewards/margins": 2.4750990867614746,
      "rewards/rejected": -5.3416972160339355,
      "step": 1250
    },
    {
      "epoch": 0.94,
      "grad_norm": 56.48468871383734,
      "learning_rate": 6.196068658797543e-09,
      "logits/chosen": 0.31367558240890503,
      "logits/rejected": 0.856947124004364,
      "logps/chosen": -480.42205810546875,
      "logps/rejected": -714.3417358398438,
      "loss": 0.2094,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.750648021697998,
      "rewards/margins": 2.5990233421325684,
      "rewards/rejected": -5.349670886993408,
      "step": 1260
    },
    {
      "epoch": 0.94,
      "grad_norm": 55.81763854062536,
      "learning_rate": 4.843299997394717e-09,
      "logits/chosen": 0.29153138399124146,
      "logits/rejected": 0.6525137424468994,
      "logps/chosen": -505.7523498535156,
      "logps/rejected": -773.7969970703125,
      "loss": 0.199,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.656862735748291,
      "rewards/margins": 2.929708957672119,
      "rewards/rejected": -5.58657169342041,
      "step": 1270
    },
    {
      "epoch": 0.95,
      "grad_norm": 45.256008770336216,
      "learning_rate": 3.655511172643372e-09,
      "logits/chosen": -0.10818688571453094,
      "logits/rejected": 0.48161396384239197,
      "logps/chosen": -536.8485107421875,
      "logps/rejected": -788.7921142578125,
      "loss": 0.1927,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.5987658500671387,
      "rewards/margins": 2.9826765060424805,
      "rewards/rejected": -5.581442832946777,
      "step": 1280
    },
    {
      "epoch": 0.96,
      "grad_norm": 75.93254873129665,
      "learning_rate": 2.633501514956532e-09,
      "logits/chosen": -0.07302987575531006,
      "logits/rejected": 0.7443720102310181,
      "logps/chosen": -507.47894287109375,
      "logps/rejected": -765.6873168945312,
      "loss": 0.2005,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.673506259918213,
      "rewards/margins": 2.9605891704559326,
      "rewards/rejected": -5.634096145629883,
      "step": 1290
    },
    {
      "epoch": 0.97,
      "grad_norm": 55.65736054179639,
      "learning_rate": 1.777958792550993e-09,
      "logits/chosen": 0.05213532596826553,
      "logits/rejected": 0.4943299889564514,
      "logps/chosen": -537.5960693359375,
      "logps/rejected": -807.1434326171875,
      "loss": 0.1728,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.9276278018951416,
      "rewards/margins": 2.9666807651519775,
      "rewards/rejected": -5.894307613372803,
      "step": 1300
    },
    {
      "epoch": 0.97,
      "eval_logits/chosen": -0.06342385709285736,
      "eval_logits/rejected": 0.23894435167312622,
      "eval_logps/chosen": -598.8973999023438,
      "eval_logps/rejected": -761.6608276367188,
      "eval_loss": 0.5417826175689697,
      "eval_rewards/accuracies": 0.75390625,
      "eval_rewards/chosen": -3.172638177871704,
      "eval_rewards/margins": 1.5663537979125977,
      "eval_rewards/rejected": -4.738992214202881,
      "eval_runtime": 98.1336,
      "eval_samples_per_second": 20.38,
      "eval_steps_per_second": 0.326,
      "step": 1300
    },
    {
      "epoch": 0.97,
      "grad_norm": 50.04724298737362,
      "learning_rate": 1.0894587486089125e-09,
      "logits/chosen": 0.30416375398635864,
      "logits/rejected": 0.5567177534103394,
      "logps/chosen": -561.5665893554688,
      "logps/rejected": -784.9238891601562,
      "loss": 0.2226,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -3.119717836380005,
      "rewards/margins": 2.4494900703430176,
      "rewards/rejected": -5.569207668304443,
      "step": 1310
    },
    {
      "epoch": 0.98,
      "grad_norm": 58.64068052496794,
      "learning_rate": 5.684647138277098e-10,
      "logits/chosen": 0.06530937552452087,
      "logits/rejected": 0.6911398768424988,
      "logps/chosen": -524.456298828125,
      "logps/rejected": -789.5941162109375,
      "loss": 0.2211,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.7515296936035156,
      "rewards/margins": 3.075373411178589,
      "rewards/rejected": -5.826903343200684,
      "step": 1320
    },
    {
      "epoch": 0.99,
      "grad_norm": 51.7145291726634,
      "learning_rate": 2.153272946184559e-10,
      "logits/chosen": 0.033466748893260956,
      "logits/rejected": 0.613431453704834,
      "logps/chosen": -560.579833984375,
      "logps/rejected": -819.1729736328125,
      "loss": 0.2092,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.932191848754883,
      "rewards/margins": 2.935133695602417,
      "rewards/rejected": -5.867325782775879,
      "step": 1330
    },
    {
      "epoch": 1.0,
      "grad_norm": 45.056558756064035,
      "learning_rate": 3.0284137163189004e-11,
      "logits/chosen": -0.001438182545825839,
      "logits/rejected": 0.446831613779068,
      "logps/chosen": -533.5986938476562,
      "logps/rejected": -776.7227783203125,
      "loss": 0.1891,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.894265651702881,
      "rewards/margins": 2.6473288536071777,
      "rewards/rejected": -5.541594505310059,
      "step": 1340
    },
    {
      "epoch": 1.0,
      "step": 1346,
      "total_flos": 0.0,
      "train_loss": 0.3438705863959722,
      "train_runtime": 21850.8794,
      "train_samples_per_second": 7.884,
      "train_steps_per_second": 0.062
    }
  ],
  "logging_steps": 10,
  "max_steps": 1346,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}