{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9994767137624281,
  "eval_steps": 100,
  "global_step": 955,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "grad_norm": 71.7172276228935,
      "learning_rate": 5.208333333333333e-09,
      "logits/chosen": -1.4981693029403687,
      "logits/rejected": -1.5654948949813843,
      "logps/chosen": -257.4665222167969,
      "logps/rejected": -99.09321594238281,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "grad_norm": 72.96425106358538,
      "learning_rate": 5.208333333333333e-08,
      "logits/chosen": -1.875927209854126,
      "logits/rejected": -1.8456478118896484,
      "logps/chosen": -326.4024658203125,
      "logps/rejected": -141.96922302246094,
      "loss": 0.6926,
      "rewards/accuracies": 0.5138888955116272,
      "rewards/chosen": 4.941297447658144e-05,
      "rewards/margins": 0.001581083401106298,
      "rewards/rejected": -0.0015316703356802464,
      "step": 10
    },
    {
      "epoch": 0.02,
      "grad_norm": 67.94798943175728,
      "learning_rate": 1.0416666666666667e-07,
      "logits/chosen": -1.8996143341064453,
      "logits/rejected": -1.8235642910003662,
      "logps/chosen": -323.2792663574219,
      "logps/rejected": -135.71456909179688,
      "loss": 0.6788,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.013530316762626171,
      "rewards/margins": 0.030556190758943558,
      "rewards/rejected": -0.01702587492763996,
      "step": 20
    },
    {
      "epoch": 0.03,
      "grad_norm": 50.35196678274095,
      "learning_rate": 1.5624999999999999e-07,
      "logits/chosen": -1.9016050100326538,
      "logits/rejected": -1.8865772485733032,
      "logps/chosen": -306.1601257324219,
      "logps/rejected": -138.81069946289062,
      "loss": 0.6075,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 0.08184734731912613,
      "rewards/margins": 0.19270142912864685,
      "rewards/rejected": -0.11085411161184311,
      "step": 30
    },
    {
      "epoch": 0.04,
      "grad_norm": 24.96243157624135,
      "learning_rate": 2.0833333333333333e-07,
      "logits/chosen": -1.9789104461669922,
      "logits/rejected": -2.000354766845703,
      "logps/chosen": -269.0250549316406,
      "logps/rejected": -152.91973876953125,
      "loss": 0.5009,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.12138372659683228,
      "rewards/margins": 0.45498305559158325,
      "rewards/rejected": -0.3335992991924286,
      "step": 40
    },
    {
      "epoch": 0.05,
      "grad_norm": 14.96717932132009,
      "learning_rate": 2.604166666666667e-07,
      "logits/chosen": -2.158332109451294,
      "logits/rejected": -2.16444993019104,
      "logps/chosen": -280.4437255859375,
      "logps/rejected": -197.60487365722656,
      "loss": 0.4282,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 0.15608879923820496,
      "rewards/margins": 0.867284893989563,
      "rewards/rejected": -0.7111960053443909,
      "step": 50
    },
    {
      "epoch": 0.06,
      "grad_norm": 10.138930051409568,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": -2.3020527362823486,
      "logits/rejected": -2.289559841156006,
      "logps/chosen": -255.1080780029297,
      "logps/rejected": -206.7790069580078,
      "loss": 0.3883,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.01837761141359806,
      "rewards/margins": 0.9224237203598022,
      "rewards/rejected": -0.9408015012741089,
      "step": 60
    },
    {
      "epoch": 0.07,
      "grad_norm": 9.627361782454214,
      "learning_rate": 3.645833333333333e-07,
      "logits/chosen": -2.3991897106170654,
      "logits/rejected": -2.4038243293762207,
      "logps/chosen": -295.4974670410156,
      "logps/rejected": -273.0234375,
      "loss": 0.3499,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.15266235172748566,
      "rewards/margins": 1.2677773237228394,
      "rewards/rejected": -1.4204397201538086,
      "step": 70
    },
    {
      "epoch": 0.08,
      "grad_norm": 14.313296662619264,
      "learning_rate": 4.1666666666666667e-07,
      "logits/chosen": -2.5073745250701904,
      "logits/rejected": -2.553334951400757,
      "logps/chosen": -332.49945068359375,
      "logps/rejected": -319.8727722167969,
      "loss": 0.3368,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.5055555105209351,
      "rewards/margins": 1.468584418296814,
      "rewards/rejected": -1.9741401672363281,
      "step": 80
    },
    {
      "epoch": 0.09,
      "grad_norm": 23.431306250567218,
      "learning_rate": 4.6874999999999996e-07,
      "logits/chosen": -2.611389636993408,
      "logits/rejected": -2.643505334854126,
      "logps/chosen": -355.7128601074219,
      "logps/rejected": -361.992431640625,
      "loss": 0.309,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.61676025390625,
      "rewards/margins": 1.7021185159683228,
      "rewards/rejected": -2.318878412246704,
      "step": 90
    },
    {
      "epoch": 0.1,
      "grad_norm": 12.356735796295311,
      "learning_rate": 4.999732492681437e-07,
      "logits/chosen": -2.621342420578003,
      "logits/rejected": -2.6476845741271973,
      "logps/chosen": -319.81988525390625,
      "logps/rejected": -322.2472229003906,
      "loss": 0.3047,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.6059430241584778,
      "rewards/margins": 1.4836266040802002,
      "rewards/rejected": -2.089569568634033,
      "step": 100
    },
    {
      "epoch": 0.1,
      "eval_logits/chosen": -2.44050931930542,
      "eval_logits/rejected": -2.3863425254821777,
      "eval_logps/chosen": -361.08013916015625,
      "eval_logps/rejected": -337.77484130859375,
      "eval_loss": 0.8551393747329712,
      "eval_rewards/accuracies": 0.3203125,
      "eval_rewards/chosen": -0.49303656816482544,
      "eval_rewards/margins": -0.29053980112075806,
      "eval_rewards/rejected": -0.20249679684638977,
      "eval_runtime": 97.3683,
      "eval_samples_per_second": 20.541,
      "eval_steps_per_second": 0.329,
      "step": 100
    },
    {
      "epoch": 0.12,
      "grad_norm": 12.540254444715838,
      "learning_rate": 4.996723692767926e-07,
      "logits/chosen": -2.7024824619293213,
      "logits/rejected": -2.732452154159546,
      "logps/chosen": -409.3314208984375,
      "logps/rejected": -413.891357421875,
      "loss": 0.2958,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.829090416431427,
      "rewards/margins": 2.046325206756592,
      "rewards/rejected": -2.875415325164795,
      "step": 110
    },
    {
      "epoch": 0.13,
      "grad_norm": 15.24853315289221,
      "learning_rate": 4.990375746213598e-07,
      "logits/chosen": -2.661564350128174,
      "logits/rejected": -2.7124838829040527,
      "logps/chosen": -378.96533203125,
      "logps/rejected": -433.1641540527344,
      "loss": 0.2884,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.9071281552314758,
      "rewards/margins": 2.1523900032043457,
      "rewards/rejected": -3.059518337249756,
      "step": 120
    },
    {
      "epoch": 0.14,
      "grad_norm": 20.41295055769396,
      "learning_rate": 4.980697142834314e-07,
      "logits/chosen": -2.693155527114868,
      "logits/rejected": -2.738525390625,
      "logps/chosen": -401.2513122558594,
      "logps/rejected": -464.12353515625,
      "loss": 0.2528,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.0387437343597412,
      "rewards/margins": 2.259091854095459,
      "rewards/rejected": -3.2978355884552,
      "step": 130
    },
    {
      "epoch": 0.15,
      "grad_norm": 27.736891370306594,
      "learning_rate": 4.967700826904229e-07,
      "logits/chosen": -2.6829416751861572,
      "logits/rejected": -2.743159770965576,
      "logps/chosen": -405.8708801269531,
      "logps/rejected": -488.6322326660156,
      "loss": 0.2529,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.1785838603973389,
      "rewards/margins": 2.3818531036376953,
      "rewards/rejected": -3.560436964035034,
      "step": 140
    },
    {
      "epoch": 0.16,
      "grad_norm": 31.193716735224292,
      "learning_rate": 4.951404179843962e-07,
      "logits/chosen": -2.4764091968536377,
      "logits/rejected": -2.481703281402588,
      "logps/chosen": -504.5975036621094,
      "logps/rejected": -567.6092529296875,
      "loss": 0.2358,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.8161567449569702,
      "rewards/margins": 2.5039477348327637,
      "rewards/rejected": -4.320104598999023,
      "step": 150
    },
    {
      "epoch": 0.17,
      "grad_norm": 16.84967305265633,
      "learning_rate": 4.931828996974498e-07,
      "logits/chosen": -2.4284980297088623,
      "logits/rejected": -2.4476141929626465,
      "logps/chosen": -411.03546142578125,
      "logps/rejected": -520.0714721679688,
      "loss": 0.2377,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.3872876167297363,
      "rewards/margins": 2.5429630279541016,
      "rewards/rejected": -3.930250883102417,
      "step": 160
    },
    {
      "epoch": 0.18,
      "grad_norm": 25.1927910072169,
      "learning_rate": 4.909001458367866e-07,
      "logits/chosen": -2.2736942768096924,
      "logits/rejected": -2.292642831802368,
      "logps/chosen": -452.697998046875,
      "logps/rejected": -563.0865478515625,
      "loss": 0.2128,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.6114835739135742,
      "rewards/margins": 2.781120538711548,
      "rewards/rejected": -4.392604351043701,
      "step": 170
    },
    {
      "epoch": 0.19,
      "grad_norm": 18.107740978814483,
      "learning_rate": 4.882952093833627e-07,
      "logits/chosen": -2.1593220233917236,
      "logits/rejected": -2.1532785892486572,
      "logps/chosen": -509.1954650878906,
      "logps/rejected": -644.2286376953125,
      "loss": 0.2037,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.7663100957870483,
      "rewards/margins": 3.2817070484161377,
      "rewards/rejected": -5.0480170249938965,
      "step": 180
    },
    {
      "epoch": 0.2,
      "grad_norm": 22.004607228534194,
      "learning_rate": 4.853715742087946e-07,
      "logits/chosen": -2.1366734504699707,
      "logits/rejected": -2.1492080688476562,
      "logps/chosen": -469.94976806640625,
      "logps/rejected": -587.4523315429688,
      "loss": 0.2059,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.6838188171386719,
      "rewards/margins": 2.9172627925872803,
      "rewards/rejected": -4.601081848144531,
      "step": 190
    },
    {
      "epoch": 0.21,
      "grad_norm": 23.839892866837005,
      "learning_rate": 4.821331504159906e-07,
      "logits/chosen": -2.0906856060028076,
      "logits/rejected": -2.1124696731567383,
      "logps/chosen": -466.5874938964844,
      "logps/rejected": -608.2426147460938,
      "loss": 0.1861,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.8450790643692017,
      "rewards/margins": 3.033769130706787,
      "rewards/rejected": -4.878848552703857,
      "step": 200
    },
    {
      "epoch": 0.21,
      "eval_logits/chosen": -1.5417730808258057,
      "eval_logits/rejected": -1.5107452869415283,
      "eval_logps/chosen": -450.2716369628906,
      "eval_logps/rejected": -421.0933532714844,
      "eval_loss": 1.0494639873504639,
      "eval_rewards/accuracies": 0.38671875,
      "eval_rewards/chosen": -1.3849513530731201,
      "eval_rewards/margins": -0.3492693305015564,
      "eval_rewards/rejected": -1.0356820821762085,
      "eval_runtime": 97.2226,
      "eval_samples_per_second": 20.571,
      "eval_steps_per_second": 0.329,
      "step": 200
    },
    {
      "epoch": 0.22,
      "grad_norm": 21.766108201350445,
      "learning_rate": 4.785842691097342e-07,
      "logits/chosen": -1.914615273475647,
      "logits/rejected": -1.9393634796142578,
      "logps/chosen": -541.3153076171875,
      "logps/rejected": -766.5374145507812,
      "loss": 0.1892,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.552523374557495,
      "rewards/margins": 3.7902703285217285,
      "rewards/rejected": -6.342793941497803,
      "step": 210
    },
    {
      "epoch": 0.23,
      "grad_norm": 37.7465298870174,
      "learning_rate": 4.7472967660421603e-07,
      "logits/chosen": -2.004246234893799,
      "logits/rejected": -2.0251355171203613,
      "logps/chosen": -528.6552734375,
      "logps/rejected": -743.7687377929688,
      "loss": 0.1798,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.033097743988037,
      "rewards/margins": 4.051178932189941,
      "rewards/rejected": -6.0842766761779785,
      "step": 220
    },
    {
      "epoch": 0.24,
      "grad_norm": 37.93991066707489,
      "learning_rate": 4.705745280752585e-07,
      "logits/chosen": -1.9027751684188843,
      "logits/rejected": -1.8593933582305908,
      "logps/chosen": -495.2493591308594,
      "logps/rejected": -688.589599609375,
      "loss": 0.1765,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.9664674997329712,
      "rewards/margins": 3.703148603439331,
      "rewards/rejected": -5.66961669921875,
      "step": 230
    },
    {
      "epoch": 0.25,
      "grad_norm": 19.738306178674453,
      "learning_rate": 4.6612438066572555e-07,
      "logits/chosen": -1.7326488494873047,
      "logits/rejected": -1.585479497909546,
      "logps/chosen": -627.2045288085938,
      "logps/rejected": -877.15234375,
      "loss": 0.164,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.1001508235931396,
      "rewards/margins": 4.346390247344971,
      "rewards/rejected": -7.446540832519531,
      "step": 240
    },
    {
      "epoch": 0.26,
      "grad_norm": 20.99257000654717,
      "learning_rate": 4.6138518605333664e-07,
      "logits/chosen": -1.7733532190322876,
      "logits/rejected": -1.5429089069366455,
      "logps/chosen": -534.9285888671875,
      "logps/rejected": -766.8218383789062,
      "loss": 0.1779,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.3250820636749268,
      "rewards/margins": 4.1490068435668945,
      "rewards/rejected": -6.4740891456604,
      "step": 250
    },
    {
      "epoch": 0.27,
      "grad_norm": 24.636105883207083,
      "learning_rate": 4.5636328249082514e-07,
      "logits/chosen": -1.7743794918060303,
      "logits/rejected": -1.6784734725952148,
      "logps/chosen": -502.83929443359375,
      "logps/rejected": -686.4469604492188,
      "loss": 0.1666,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.020374298095703,
      "rewards/margins": 3.5390701293945312,
      "rewards/rejected": -5.559444427490234,
      "step": 260
    },
    {
      "epoch": 0.28,
      "grad_norm": 26.442049627508165,
      "learning_rate": 4.510653863290871e-07,
      "logits/chosen": -1.8089313507080078,
      "logits/rejected": -1.504288673400879,
      "logps/chosen": -553.7681884765625,
      "logps/rejected": -773.63671875,
      "loss": 0.163,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.3301241397857666,
      "rewards/margins": 4.031432628631592,
      "rewards/rejected": -6.361557483673096,
      "step": 270
    },
    {
      "epoch": 0.29,
      "grad_norm": 21.752850388248913,
      "learning_rate": 4.4549858303465737e-07,
      "logits/chosen": -1.792066216468811,
      "logits/rejected": -1.5596303939819336,
      "logps/chosen": -521.0010986328125,
      "logps/rejected": -791.414794921875,
      "loss": 0.1514,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.1834633350372314,
      "rewards/margins": 4.415928363800049,
      "rewards/rejected": -6.599390983581543,
      "step": 280
    },
    {
      "epoch": 0.3,
      "grad_norm": 21.62051159722618,
      "learning_rate": 4.396703177135261e-07,
      "logits/chosen": -1.573697805404663,
      "logits/rejected": -1.1857482194900513,
      "logps/chosen": -611.6795654296875,
      "logps/rejected": -855.2744140625,
      "loss": 0.1493,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.884495258331299,
      "rewards/margins": 4.296299934387207,
      "rewards/rejected": -7.180795192718506,
      "step": 290
    },
    {
      "epoch": 0.31,
      "grad_norm": 16.812667194402408,
      "learning_rate": 4.335883851539693e-07,
      "logits/chosen": -1.738490343093872,
      "logits/rejected": -1.7563718557357788,
      "logps/chosen": -494.66253662109375,
      "logps/rejected": -731.6812744140625,
      "loss": 0.1608,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.9538425207138062,
      "rewards/margins": 3.9912962913513184,
      "rewards/rejected": -5.945138931274414,
      "step": 300
    },
    {
      "epoch": 0.31,
      "eval_logits/chosen": -1.4367032051086426,
      "eval_logits/rejected": -1.4021562337875366,
      "eval_logps/chosen": -454.9446105957031,
      "eval_logps/rejected": -422.9683532714844,
      "eval_loss": 1.0910040140151978,
      "eval_rewards/accuracies": 0.39453125,
      "eval_rewards/chosen": -1.4316813945770264,
      "eval_rewards/margins": -0.37724918127059937,
      "eval_rewards/rejected": -1.0544321537017822,
      "eval_runtime": 97.2015,
      "eval_samples_per_second": 20.576,
      "eval_steps_per_second": 0.329,
      "step": 300
    },
    {
      "epoch": 0.32,
      "grad_norm": 23.495796172679743,
      "learning_rate": 4.272609194017105e-07,
      "logits/chosen": -1.7330036163330078,
      "logits/rejected": -1.6350265741348267,
      "logps/chosen": -519.0903930664062,
      "logps/rejected": -720.5287475585938,
      "loss": 0.1552,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.7445989847183228,
      "rewards/margins": 4.112780570983887,
      "rewards/rejected": -5.857378959655762,
      "step": 310
    },
    {
      "epoch": 0.33,
      "grad_norm": 16.298880424996614,
      "learning_rate": 4.2069638288135547e-07,
      "logits/chosen": -1.7347389459609985,
      "logits/rejected": -1.7288787364959717,
      "logps/chosen": -510.26483154296875,
      "logps/rejected": -713.36572265625,
      "loss": 0.157,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.7514524459838867,
      "rewards/margins": 3.96923828125,
      "rewards/rejected": -5.720690727233887,
      "step": 320
    },
    {
      "epoch": 0.35,
      "grad_norm": 19.47125814373079,
      "learning_rate": 4.139035550786494e-07,
      "logits/chosen": -1.1601016521453857,
      "logits/rejected": -0.8797602653503418,
      "logps/chosen": -555.39990234375,
      "logps/rejected": -796.0065307617188,
      "loss": 0.1561,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -2.3634750843048096,
      "rewards/margins": 4.329623222351074,
      "rewards/rejected": -6.693098545074463,
      "step": 330
    },
    {
      "epoch": 0.36,
      "grad_norm": 28.613677806731356,
      "learning_rate": 4.0689152079869306e-07,
      "logits/chosen": -1.2572317123413086,
      "logits/rejected": -0.9607254862785339,
      "logps/chosen": -606.7630615234375,
      "logps/rejected": -846.2305908203125,
      "loss": 0.1416,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.5160038471221924,
      "rewards/margins": 4.53468132019043,
      "rewards/rejected": -7.050685882568359,
      "step": 340
    },
    {
      "epoch": 0.37,
      "grad_norm": 17.707474904024725,
      "learning_rate": 3.99669658015821e-07,
      "logits/chosen": -1.3649070262908936,
      "logits/rejected": -0.9824110269546509,
      "logps/chosen": -558.0640869140625,
      "logps/rejected": -775.701171875,
      "loss": 0.1563,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.48624849319458,
      "rewards/margins": 4.034348964691162,
      "rewards/rejected": -6.5205979347229,
      "step": 350
    },
    {
      "epoch": 0.38,
      "grad_norm": 17.329304967511433,
      "learning_rate": 3.92247625331392e-07,
      "logits/chosen": -1.5912964344024658,
      "logits/rejected": -1.5140416622161865,
      "logps/chosen": -473.7013244628906,
      "logps/rejected": -705.4163818359375,
      "loss": 0.1391,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.0327391624450684,
      "rewards/margins": 3.767451047897339,
      "rewards/rejected": -5.800190448760986,
      "step": 360
    },
    {
      "epoch": 0.39,
      "grad_norm": 21.039638710046997,
      "learning_rate": 3.846353490562664e-07,
      "logits/chosen": -1.3319923877716064,
      "logits/rejected": -1.090010404586792,
      "logps/chosen": -520.6598510742188,
      "logps/rejected": -788.8143310546875,
      "loss": 0.1403,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -2.3732516765594482,
      "rewards/margins": 4.287914276123047,
      "rewards/rejected": -6.661165714263916,
      "step": 370
    },
    {
      "epoch": 0.4,
      "grad_norm": 19.41739908042072,
      "learning_rate": 3.768430099352445e-07,
      "logits/chosen": -1.2049062252044678,
      "logits/rejected": -0.9995657205581665,
      "logps/chosen": -531.4547119140625,
      "logps/rejected": -863.2029418945312,
      "loss": 0.1359,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.5220770835876465,
      "rewards/margins": 4.725777626037598,
      "rewards/rejected": -7.247855186462402,
      "step": 380
    },
    {
      "epoch": 0.41,
      "grad_norm": 30.846420024935966,
      "learning_rate": 3.6888102953122304e-07,
      "logits/chosen": -1.4293053150177002,
      "logits/rejected": -1.129652976989746,
      "logps/chosen": -509.592529296875,
      "logps/rejected": -834.6383056640625,
      "loss": 0.1217,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.1002399921417236,
      "rewards/margins": 4.87478494644165,
      "rewards/rejected": -6.975025177001953,
      "step": 390
    },
    {
      "epoch": 0.42,
      "grad_norm": 26.99256302878051,
      "learning_rate": 3.607600562872785e-07,
      "logits/chosen": -1.3965580463409424,
      "logits/rejected": -0.9727011919021606,
      "logps/chosen": -591.9832763671875,
      "logps/rejected": -926.43701171875,
      "loss": 0.1368,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.8865580558776855,
      "rewards/margins": 5.026923656463623,
      "rewards/rejected": -7.913481712341309,
      "step": 400
    },
    {
      "epoch": 0.42,
      "eval_logits/chosen": -1.053754210472107,
      "eval_logits/rejected": -1.013121485710144,
      "eval_logps/chosen": -520.169921875,
      "eval_logps/rejected": -479.64556884765625,
      "eval_loss": 1.3009716272354126,
      "eval_rewards/accuracies": 0.41015625,
      "eval_rewards/chosen": -2.0839340686798096,
      "eval_rewards/margins": -0.46272987127304077,
      "eval_rewards/rejected": -1.6212042570114136,
      "eval_runtime": 97.208,
      "eval_samples_per_second": 20.574,
      "eval_steps_per_second": 0.329,
      "step": 400
    },
    {
      "epoch": 0.43,
      "grad_norm": 18.264964860581,
      "learning_rate": 3.5249095128531856e-07,
      "logits/chosen": -1.6575086116790771,
      "logits/rejected": -1.4926228523254395,
      "logps/chosen": -505.78448486328125,
      "logps/rejected": -805.1259155273438,
      "loss": 0.151,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.203605890274048,
      "rewards/margins": 4.460430145263672,
      "rewards/rejected": -6.664034843444824,
      "step": 410
    },
    {
      "epoch": 0.44,
      "grad_norm": 19.09047976589624,
      "learning_rate": 3.4408477372034736e-07,
      "logits/chosen": -1.6263633966445923,
      "logits/rejected": -1.4956812858581543,
      "logps/chosen": -479.7705993652344,
      "logps/rejected": -766.1605224609375,
      "loss": 0.1448,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.093071937561035,
      "rewards/margins": 4.394181728363037,
      "rewards/rejected": -6.4872541427612305,
      "step": 420
    },
    {
      "epoch": 0.45,
      "grad_norm": 26.836781627013515,
      "learning_rate": 3.3555276610977276e-07,
      "logits/chosen": -1.662540078163147,
      "logits/rejected": -1.4674957990646362,
      "logps/chosen": -527.6272583007812,
      "logps/rejected": -827.05224609375,
      "loss": 0.1477,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.207035541534424,
      "rewards/margins": 4.785927772521973,
      "rewards/rejected": -6.9929633140563965,
      "step": 430
    },
    {
      "epoch": 0.46,
      "grad_norm": 24.71697296764681,
      "learning_rate": 3.269063392575352e-07,
      "logits/chosen": -1.682995080947876,
      "logits/rejected": -1.625732660293579,
      "logps/chosen": -521.5386352539062,
      "logps/rejected": -861.8421630859375,
      "loss": 0.1431,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.1410486698150635,
      "rewards/margins": 5.073235511779785,
      "rewards/rejected": -7.214284420013428,
      "step": 440
    },
    {
      "epoch": 0.47,
      "grad_norm": 35.22220664866878,
      "learning_rate": 3.1815705699316964e-07,
      "logits/chosen": -1.578249216079712,
      "logits/rejected": -1.4307564496994019,
      "logps/chosen": -618.6419067382812,
      "logps/rejected": -917.3260498046875,
      "loss": 0.1288,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.068641185760498,
      "rewards/margins": 4.769894599914551,
      "rewards/rejected": -7.838536262512207,
      "step": 450
    },
    {
      "epoch": 0.48,
      "grad_norm": 15.368374650378705,
      "learning_rate": 3.0931662070620794e-07,
      "logits/chosen": -1.8674280643463135,
      "logits/rejected": -1.8419768810272217,
      "logps/chosen": -574.4186401367188,
      "logps/rejected": -899.6975708007812,
      "loss": 0.1317,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.381072998046875,
      "rewards/margins": 5.151331424713135,
      "rewards/rejected": -7.53240442276001,
      "step": 460
    },
    {
      "epoch": 0.49,
      "grad_norm": 32.45699366987169,
      "learning_rate": 3.003968536966078e-07,
      "logits/chosen": -1.779637098312378,
      "logits/rejected": -1.7716691493988037,
      "logps/chosen": -548.851806640625,
      "logps/rejected": -895.9444580078125,
      "loss": 0.1467,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.4585914611816406,
      "rewards/margins": 5.207429885864258,
      "rewards/rejected": -7.666021823883057,
      "step": 470
    },
    {
      "epoch": 0.5,
      "grad_norm": 20.390767179507876,
      "learning_rate": 2.9140968536213693e-07,
      "logits/chosen": -1.919952630996704,
      "logits/rejected": -1.9492056369781494,
      "logps/chosen": -536.4271240234375,
      "logps/rejected": -884.7716064453125,
      "loss": 0.1285,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.535160541534424,
      "rewards/margins": 5.057857990264893,
      "rewards/rejected": -7.593018531799316,
      "step": 480
    },
    {
      "epoch": 0.51,
      "grad_norm": 23.067489784018093,
      "learning_rate": 2.823671352438608e-07,
      "logits/chosen": -1.9916563034057617,
      "logits/rejected": -2.021616220474243,
      "logps/chosen": -518.0189208984375,
      "logps/rejected": -824.4202880859375,
      "loss": 0.1367,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.1029324531555176,
      "rewards/margins": 4.754405498504639,
      "rewards/rejected": -6.857337951660156,
      "step": 490
    },
    {
      "epoch": 0.52,
      "grad_norm": 25.320584902838384,
      "learning_rate": 2.73281296951072e-07,
      "logits/chosen": -1.8573198318481445,
      "logits/rejected": -1.8858684301376343,
      "logps/chosen": -541.9103393554688,
      "logps/rejected": -806.6531372070312,
      "loss": 0.1364,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.396160840988159,
      "rewards/margins": 4.32682466506958,
      "rewards/rejected": -6.722985744476318,
      "step": 500
    },
    {
      "epoch": 0.52,
      "eval_logits/chosen": -1.6466461420059204,
      "eval_logits/rejected": -1.6090116500854492,
      "eval_logps/chosen": -470.0933532714844,
      "eval_logps/rejected": -430.8614196777344,
      "eval_loss": 1.1773370504379272,
      "eval_rewards/accuracies": 0.37109375,
      "eval_rewards/chosen": -1.5831681489944458,
      "eval_rewards/margins": -0.44980528950691223,
      "eval_rewards/rejected": -1.133362889289856,
      "eval_runtime": 97.142,
      "eval_samples_per_second": 20.588,
      "eval_steps_per_second": 0.329,
      "step": 500
    },
    {
      "epoch": 0.53,
      "grad_norm": 22.389152396843905,
      "learning_rate": 2.641643219871597e-07,
      "logits/chosen": -1.9270741939544678,
      "logits/rejected": -1.8253233432769775,
      "logps/chosen": -544.8509521484375,
      "logps/rejected": -834.5119018554688,
      "loss": 0.1273,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.5420873165130615,
      "rewards/margins": 4.583073616027832,
      "rewards/rejected": -7.125161170959473,
      "step": 510
    },
    {
      "epoch": 0.54,
      "grad_norm": 29.91148010858245,
      "learning_rate": 2.550284034980507e-07,
      "logits/chosen": -1.8461729288101196,
      "logits/rejected": -1.9183883666992188,
      "logps/chosen": -536.2467041015625,
      "logps/rejected": -846.56689453125,
      "loss": 0.1301,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.3210761547088623,
      "rewards/margins": 4.834765911102295,
      "rewards/rejected": -7.1558427810668945,
      "step": 520
    },
    {
      "epoch": 0.55,
      "grad_norm": 24.244708458502263,
      "learning_rate": 2.4588575996495794e-07,
      "logits/chosen": -1.80156672000885,
      "logits/rejected": -1.815596580505371,
      "logps/chosen": -494.86798095703125,
      "logps/rejected": -798.2820434570312,
      "loss": 0.1237,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.3462131023406982,
      "rewards/margins": 4.387810230255127,
      "rewards/rejected": -6.7340240478515625,
      "step": 530
    },
    {
      "epoch": 0.57,
      "grad_norm": 19.910786436303987,
      "learning_rate": 2.367486188632446e-07,
      "logits/chosen": -1.9172916412353516,
      "logits/rejected": -1.9127076864242554,
      "logps/chosen": -525.8649291992188,
      "logps/rejected": -845.7303466796875,
      "loss": 0.1138,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.059016704559326,
      "rewards/margins": 4.951017379760742,
      "rewards/rejected": -7.010035037994385,
      "step": 540
    },
    {
      "epoch": 0.58,
      "grad_norm": 26.12771211985874,
      "learning_rate": 2.276292003092593e-07,
      "logits/chosen": -1.8340644836425781,
      "logits/rejected": -1.8532531261444092,
      "logps/chosen": -547.7066650390625,
      "logps/rejected": -828.9231567382812,
      "loss": 0.1309,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.3669962882995605,
      "rewards/margins": 4.561534404754639,
      "rewards/rejected": -6.928530216217041,
      "step": 550
    },
    {
      "epoch": 0.59,
      "grad_norm": 23.903525728545898,
      "learning_rate": 2.185397007170141e-07,
      "logits/chosen": -1.7198997735977173,
      "logits/rejected": -1.5677220821380615,
      "logps/chosen": -525.4856567382812,
      "logps/rejected": -855.2352294921875,
      "loss": 0.1157,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.4956746101379395,
      "rewards/margins": 4.861161708831787,
      "rewards/rejected": -7.356836795806885,
      "step": 560
    },
    {
      "epoch": 0.6,
      "grad_norm": 24.886700178362616,
      "learning_rate": 2.094922764865619e-07,
      "logits/chosen": -1.7758210897445679,
      "logits/rejected": -1.836538314819336,
      "logps/chosen": -506.49053955078125,
      "logps/rejected": -846.2745971679688,
      "loss": 0.1366,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.2135753631591797,
      "rewards/margins": 4.973353385925293,
      "rewards/rejected": -7.186928749084473,
      "step": 570
    },
    {
      "epoch": 0.61,
      "grad_norm": 21.212825758656418,
      "learning_rate": 2.0049902774588797e-07,
      "logits/chosen": -1.7731355428695679,
      "logits/rejected": -1.777611494064331,
      "logps/chosen": -581.9010009765625,
      "logps/rejected": -871.5733642578125,
      "loss": 0.123,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.585883378982544,
      "rewards/margins": 4.742143154144287,
      "rewards/rejected": -7.32802677154541,
      "step": 580
    },
    {
      "epoch": 0.62,
      "grad_norm": 24.501331083650943,
      "learning_rate": 1.9157198216806238e-07,
      "logits/chosen": -1.7602859735488892,
      "logits/rejected": -1.84162175655365,
      "logps/chosen": -452.37420654296875,
      "logps/rejected": -707.836181640625,
      "loss": 0.1305,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.025653600692749,
      "rewards/margins": 3.844447374343872,
      "rewards/rejected": -5.870100975036621,
      "step": 590
    },
    {
      "epoch": 0.63,
      "grad_norm": 31.082842710792463,
      "learning_rate": 1.8272307888529274e-07,
      "logits/chosen": -1.5975382328033447,
      "logits/rejected": -1.4109210968017578,
      "logps/chosen": -539.9990234375,
      "logps/rejected": -849.1795654296875,
      "loss": 0.1223,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.722317934036255,
      "rewards/margins": 4.622250556945801,
      "rewards/rejected": -7.344568729400635,
      "step": 600
    },
    {
      "epoch": 0.63,
      "eval_logits/chosen": -1.1879667043685913,
      "eval_logits/rejected": -1.1541048288345337,
      "eval_logps/chosen": -541.48828125,
      "eval_logps/rejected": -500.4929504394531,
      "eval_loss": 1.3205512762069702,
      "eval_rewards/accuracies": 0.4140625,
      "eval_rewards/chosen": -2.2971181869506836,
      "eval_rewards/margins": -0.46744003891944885,
      "eval_rewards/rejected": -1.829677939414978,
      "eval_runtime": 97.2287,
      "eval_samples_per_second": 20.57,
      "eval_steps_per_second": 0.329,
      "step": 600
    },
    {
      "epoch": 0.64,
      "grad_norm": 25.74237035768959,
      "learning_rate": 1.7396415252139288e-07,
      "logits/chosen": -1.5792185068130493,
      "logits/rejected": -1.6271593570709229,
      "logps/chosen": -576.2874755859375,
      "logps/rejected": -917.5999755859375,
      "loss": 0.1228,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -2.802475929260254,
      "rewards/margins": 5.048992156982422,
      "rewards/rejected": -7.851468563079834,
      "step": 610
    },
    {
      "epoch": 0.65,
      "grad_norm": 29.519623894853613,
      "learning_rate": 1.6530691736402316e-07,
      "logits/chosen": -1.6389156579971313,
      "logits/rejected": -1.6191425323486328,
      "logps/chosen": -530.8707275390625,
      "logps/rejected": -890.0936279296875,
      "loss": 0.1254,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.3094658851623535,
      "rewards/margins": 5.2504377365112305,
      "rewards/rejected": -7.559903144836426,
      "step": 620
    },
    {
      "epoch": 0.66,
      "grad_norm": 50.42769593704326,
      "learning_rate": 1.5676295169786864e-07,
      "logits/chosen": -1.7326772212982178,
      "logits/rejected": -1.5138975381851196,
      "logps/chosen": -561.8060302734375,
      "logps/rejected": -941.2083740234375,
      "loss": 0.1236,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.4162166118621826,
      "rewards/margins": 5.7043328285217285,
      "rewards/rejected": -8.120549201965332,
      "step": 630
    },
    {
      "epoch": 0.67,
      "grad_norm": 27.38270007193824,
      "learning_rate": 1.483436823197092e-07,
      "logits/chosen": -1.5837154388427734,
      "logits/rejected": -1.4245226383209229,
      "logps/chosen": -562.3756103515625,
      "logps/rejected": -949.3406982421875,
      "loss": 0.1163,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.040762186050415,
      "rewards/margins": 5.183028697967529,
      "rewards/rejected": -8.223790168762207,
      "step": 640
    },
    {
      "epoch": 0.68,
      "grad_norm": 35.27601401211646,
      "learning_rate": 1.4006036925609243e-07,
      "logits/chosen": -1.7516834735870361,
      "logits/rejected": -1.7616602182388306,
      "logps/chosen": -558.3771362304688,
      "logps/rejected": -879.00341796875,
      "loss": 0.1139,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.68957257270813,
      "rewards/margins": 4.818731784820557,
      "rewards/rejected": -7.508304595947266,
      "step": 650
    },
    {
      "epoch": 0.69,
      "grad_norm": 22.77394327537456,
      "learning_rate": 1.319240907040458e-07,
      "logits/chosen": -1.7446712255477905,
      "logits/rejected": -1.6567370891571045,
      "logps/chosen": -559.461669921875,
      "logps/rejected": -941.3621826171875,
      "loss": 0.1117,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.522426128387451,
      "rewards/margins": 5.564127445220947,
      "rewards/rejected": -8.086553573608398,
      "step": 660
    },
    {
      "epoch": 0.7,
      "grad_norm": 30.36694985683218,
      "learning_rate": 1.239457282149695e-07,
      "logits/chosen": -1.4815702438354492,
      "logits/rejected": -1.3673474788665771,
      "logps/chosen": -548.4630737304688,
      "logps/rejected": -930.783203125,
      "loss": 0.1092,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.9377596378326416,
      "rewards/margins": 5.2314453125,
      "rewards/rejected": -8.169205665588379,
      "step": 670
    },
    {
      "epoch": 0.71,
      "grad_norm": 28.757425831930576,
      "learning_rate": 1.1613595214152711e-07,
      "logits/chosen": -1.5986506938934326,
      "logits/rejected": -1.5326309204101562,
      "logps/chosen": -513.4288940429688,
      "logps/rejected": -861.1954345703125,
      "loss": 0.1105,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.518094062805176,
      "rewards/margins": 4.961692810058594,
      "rewards/rejected": -7.4797868728637695,
      "step": 680
    },
    {
      "epoch": 0.72,
      "grad_norm": 23.459702688906162,
      "learning_rate": 1.0850520736699362e-07,
      "logits/chosen": -1.6289697885513306,
      "logits/rejected": -1.527553677558899,
      "logps/chosen": -537.8997192382812,
      "logps/rejected": -878.1143798828125,
      "loss": 0.113,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.5290098190307617,
      "rewards/margins": 4.979551315307617,
      "rewards/rejected": -7.508561611175537,
      "step": 690
    },
    {
      "epoch": 0.73,
      "grad_norm": 25.531163769236304,
      "learning_rate": 1.0106369933615042e-07,
      "logits/chosen": -1.5353871583938599,
      "logits/rejected": -1.3200981616973877,
      "logps/chosen": -662.8204956054688,
      "logps/rejected": -1062.7938232421875,
      "loss": 0.0971,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.5107734203338623,
      "rewards/margins": 5.729162693023682,
      "rewards/rejected": -9.239934921264648,
      "step": 700
    },
    {
      "epoch": 0.73,
      "eval_logits/chosen": -0.9711934924125671,
      "eval_logits/rejected": -0.9391850829124451,
      "eval_logps/chosen": -577.3128051757812,
      "eval_logps/rejected": -533.4666748046875,
      "eval_loss": 1.463817834854126,
      "eval_rewards/accuracies": 0.390625,
      "eval_rewards/chosen": -2.6553633213043213,
      "eval_rewards/margins": -0.4959481358528137,
      "eval_rewards/rejected": -2.1594154834747314,
      "eval_runtime": 97.1589,
      "eval_samples_per_second": 20.585,
      "eval_steps_per_second": 0.329,
      "step": 700
    },
    {
      "epoch": 0.74,
      "grad_norm": 43.118837668077326,
      "learning_rate": 9.382138040640714e-08,
      "logits/chosen": -1.6447147130966187,
      "logits/rejected": -1.4446865320205688,
      "logps/chosen": -644.3197021484375,
      "logps/rejected": -1070.079345703125,
      "loss": 0.11,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.044210910797119,
      "rewards/margins": 6.259350776672363,
      "rewards/rejected": -9.303561210632324,
      "step": 710
    },
    {
      "epoch": 0.75,
      "grad_norm": 32.87349411722058,
      "learning_rate": 8.678793653740632e-08,
      "logits/chosen": -1.7681348323822021,
      "logits/rejected": -1.6581518650054932,
      "logps/chosen": -622.3300170898438,
      "logps/rejected": -1050.2943115234375,
      "loss": 0.1069,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.925344944000244,
      "rewards/margins": 6.121405601501465,
      "rewards/rejected": -9.04675006866455,
      "step": 720
    },
    {
      "epoch": 0.76,
      "grad_norm": 36.992204810711705,
      "learning_rate": 7.997277433690983e-08,
      "logits/chosen": -1.623159646987915,
      "logits/rejected": -1.4811782836914062,
      "logps/chosen": -612.2554931640625,
      "logps/rejected": -995.8507690429688,
      "loss": 0.118,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.1586315631866455,
      "rewards/margins": 5.526102066040039,
      "rewards/rejected": -8.684733390808105,
      "step": 730
    },
    {
      "epoch": 0.77,
      "grad_norm": 31.945634867150687,
      "learning_rate": 7.338500848029602e-08,
      "logits/chosen": -1.51155686378479,
      "logits/rejected": -1.349172592163086,
      "logps/chosen": -575.31103515625,
      "logps/rejected": -1048.4927978515625,
      "loss": 0.1135,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.0343711376190186,
      "rewards/margins": 6.1768388748168945,
      "rewards/rejected": -9.211209297180176,
      "step": 740
    },
    {
      "epoch": 0.78,
      "grad_norm": 26.030459327741653,
      "learning_rate": 6.70334495204884e-08,
      "logits/chosen": -1.6075376272201538,
      "logits/rejected": -1.2717111110687256,
      "logps/chosen": -635.0384521484375,
      "logps/rejected": -1016.2355346679688,
      "loss": 0.0957,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.0112128257751465,
      "rewards/margins": 5.901360511779785,
      "rewards/rejected": -8.912572860717773,
      "step": 750
    },
    {
      "epoch": 0.8,
      "grad_norm": 34.627238331382905,
      "learning_rate": 6.092659210462231e-08,
      "logits/chosen": -1.448529839515686,
      "logits/rejected": -1.4409586191177368,
      "logps/chosen": -625.353271484375,
      "logps/rejected": -1024.951416015625,
      "loss": 0.1095,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.2976653575897217,
      "rewards/margins": 5.690510272979736,
      "rewards/rejected": -8.988176345825195,
      "step": 760
    },
    {
      "epoch": 0.81,
      "grad_norm": 29.968791403480516,
      "learning_rate": 5.507260361320737e-08,
      "logits/chosen": -1.5798556804656982,
      "logits/rejected": -1.5097671747207642,
      "logps/chosen": -600.1854248046875,
      "logps/rejected": -997.0427856445312,
      "loss": 0.1033,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.1436641216278076,
      "rewards/margins": 5.432265281677246,
      "rewards/rejected": -8.575929641723633,
      "step": 770
    },
    {
      "epoch": 0.82,
      "grad_norm": 23.15982358200479,
      "learning_rate": 4.947931323697982e-08,
      "logits/chosen": -1.4408910274505615,
      "logits/rejected": -1.3072535991668701,
      "logps/chosen": -556.1472778320312,
      "logps/rejected": -968.1648559570312,
      "loss": 0.0969,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.2009377479553223,
      "rewards/margins": 5.246421813964844,
      "rewards/rejected": -8.447359085083008,
      "step": 780
    },
    {
      "epoch": 0.83,
      "grad_norm": 23.663576108460163,
      "learning_rate": 4.415420150605398e-08,
      "logits/chosen": -1.5656832456588745,
      "logits/rejected": -1.3935534954071045,
      "logps/chosen": -601.1109008789062,
      "logps/rejected": -999.2574462890625,
      "loss": 0.1023,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.033841609954834,
      "rewards/margins": 5.674118995666504,
      "rewards/rejected": -8.70796012878418,
      "step": 790
    },
    {
      "epoch": 0.84,
      "grad_norm": 23.462966354133016,
      "learning_rate": 3.9104390285376374e-08,
      "logits/chosen": -1.466312289237976,
      "logits/rejected": -1.2542184591293335,
      "logps/chosen": -619.60009765625,
      "logps/rejected": -1041.6527099609375,
      "loss": 0.1035,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.135887861251831,
      "rewards/margins": 5.890833854675293,
      "rewards/rejected": -9.026721954345703,
      "step": 800
    },
    {
      "epoch": 0.84,
      "eval_logits/chosen": -0.9231772422790527,
      "eval_logits/rejected": -0.8901588916778564,
      "eval_logps/chosen": -569.3816528320312,
      "eval_logps/rejected": -532.9067993164062,
      "eval_loss": 1.4475353956222534,
      "eval_rewards/accuracies": 0.39453125,
      "eval_rewards/chosen": -2.5760514736175537,
      "eval_rewards/margins": -0.42223480343818665,
      "eval_rewards/rejected": -2.1538166999816895,
      "eval_runtime": 97.1382,
      "eval_samples_per_second": 20.589,
      "eval_steps_per_second": 0.329,
      "step": 800
    },
    {
      "epoch": 0.85,
      "grad_norm": 30.508507386848592,
      "learning_rate": 3.433663324986208e-08,
      "logits/chosen": -1.4120627641677856,
      "logits/rejected": -1.3580305576324463,
      "logps/chosen": -642.4114990234375,
      "logps/rejected": -1046.66357421875,
      "loss": 0.1086,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -3.5207436084747314,
      "rewards/margins": 5.544968605041504,
      "rewards/rejected": -9.065712928771973,
      "step": 810
    },
    {
      "epoch": 0.86,
      "grad_norm": 15.450308092025814,
      "learning_rate": 2.9857306851953897e-08,
      "logits/chosen": -1.5075926780700684,
      "logits/rejected": -1.286818504333496,
      "logps/chosen": -577.96533203125,
      "logps/rejected": -1041.372802734375,
      "loss": 0.0844,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.9870734214782715,
      "rewards/margins": 6.117280960083008,
      "rewards/rejected": -9.104352951049805,
      "step": 820
    },
    {
      "epoch": 0.87,
      "grad_norm": 32.428690976585585,
      "learning_rate": 2.567240179368185e-08,
      "logits/chosen": -1.5189392566680908,
      "logits/rejected": -1.3501880168914795,
      "logps/chosen": -603.621337890625,
      "logps/rejected": -1007.1226806640625,
      "loss": 0.0982,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.046630620956421,
      "rewards/margins": 5.773746967315674,
      "rewards/rejected": -8.820378303527832,
      "step": 830
    },
    {
      "epoch": 0.88,
      "grad_norm": 35.07586716910879,
      "learning_rate": 2.1787515014630357e-08,
      "logits/chosen": -1.406670331954956,
      "logits/rejected": -1.1134653091430664,
      "logps/chosen": -552.0911254882812,
      "logps/rejected": -1012.5986328125,
      "loss": 0.1141,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.7988593578338623,
      "rewards/margins": 6.124402046203613,
      "rewards/rejected": -8.923261642456055,
      "step": 840
    },
    {
      "epoch": 0.89,
      "grad_norm": 35.12000542720853,
      "learning_rate": 1.820784220652766e-08,
      "logits/chosen": -1.5011646747589111,
      "logits/rejected": -1.4681178331375122,
      "logps/chosen": -629.8143310546875,
      "logps/rejected": -1053.095703125,
      "loss": 0.0947,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.13895583152771,
      "rewards/margins": 5.925333499908447,
      "rewards/rejected": -9.064289093017578,
      "step": 850
    },
    {
      "epoch": 0.9,
      "grad_norm": 37.505156397985395,
      "learning_rate": 1.4938170864468636e-08,
      "logits/chosen": -1.5276473760604858,
      "logits/rejected": -1.3087430000305176,
      "logps/chosen": -611.999267578125,
      "logps/rejected": -982.8528442382812,
      "loss": 0.1089,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.328166961669922,
      "rewards/margins": 5.248373031616211,
      "rewards/rejected": -8.576539993286133,
      "step": 860
    },
    {
      "epoch": 0.91,
      "grad_norm": 33.59787382725122,
      "learning_rate": 1.1982873884064465e-08,
      "logits/chosen": -1.5041674375534058,
      "logits/rejected": -1.186408281326294,
      "logps/chosen": -543.9867553710938,
      "logps/rejected": -960.0665893554688,
      "loss": 0.1023,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.6006431579589844,
      "rewards/margins": 5.71505069732666,
      "rewards/rejected": -8.315693855285645,
      "step": 870
    },
    {
      "epoch": 0.92,
      "grad_norm": 24.198197782455043,
      "learning_rate": 9.345903713082304e-09,
      "logits/chosen": -1.5898497104644775,
      "logits/rejected": -1.4362118244171143,
      "logps/chosen": -581.5143432617188,
      "logps/rejected": -1014.4615478515625,
      "loss": 0.0837,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.9443726539611816,
      "rewards/margins": 5.875336647033691,
      "rewards/rejected": -8.819709777832031,
      "step": 880
    },
    {
      "epoch": 0.93,
      "grad_norm": 32.4653792997969,
      "learning_rate": 7.030787065396865e-09,
      "logits/chosen": -1.5886590480804443,
      "logits/rejected": -1.2676339149475098,
      "logps/chosen": -612.6450805664062,
      "logps/rejected": -982.3916015625,
      "loss": 0.1056,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -3.1008591651916504,
      "rewards/margins": 5.476729393005371,
      "rewards/rejected": -8.57758903503418,
      "step": 890
    },
    {
      "epoch": 0.94,
      "grad_norm": 28.89242092867656,
      "learning_rate": 5.04062020432286e-09,
      "logits/chosen": -1.5399090051651,
      "logits/rejected": -1.42655611038208,
      "logps/chosen": -595.1466064453125,
      "logps/rejected": -1033.9515380859375,
      "loss": 0.088,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.967661142349243,
      "rewards/margins": 6.027240753173828,
      "rewards/rejected": -8.994901657104492,
      "step": 900
    },
    {
      "epoch": 0.94,
      "eval_logits/chosen": -1.1238834857940674,
      "eval_logits/rejected": -1.0823395252227783,
      "eval_logps/chosen": -554.9180297851562,
      "eval_logps/rejected": -517.7515869140625,
      "eval_loss": 1.3947275876998901,
      "eval_rewards/accuracies": 0.38671875,
      "eval_rewards/chosen": -2.43141508102417,
      "eval_rewards/margins": -0.4291508197784424,
      "eval_rewards/rejected": -2.0022642612457275,
      "eval_runtime": 97.126,
      "eval_samples_per_second": 20.592,
      "eval_steps_per_second": 0.329,
      "step": 900
    },
    {
      "epoch": 0.95,
      "grad_norm": 26.806730695970433,
      "learning_rate": 3.3780648016376866e-09,
      "logits/chosen": -1.658847451210022,
      "logits/rejected": -1.3762922286987305,
      "logps/chosen": -603.6043701171875,
      "logps/rejected": -1017.3605346679688,
      "loss": 0.0898,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.103484630584717,
      "rewards/margins": 5.833575248718262,
      "rewards/rejected": -8.93705940246582,
      "step": 910
    },
    {
      "epoch": 0.96,
      "grad_norm": 15.136819931212546,
      "learning_rate": 2.0453443778310766e-09,
      "logits/chosen": -1.627918004989624,
      "logits/rejected": -1.4132583141326904,
      "logps/chosen": -563.746337890625,
      "logps/rejected": -1013.9581909179688,
      "loss": 0.0936,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.8101205825805664,
      "rewards/margins": 6.073071479797363,
      "rewards/rejected": -8.88319206237793,
      "step": 920
    },
    {
      "epoch": 0.97,
      "grad_norm": 36.0925338348187,
      "learning_rate": 1.0442413283435758e-09,
      "logits/chosen": -1.5613000392913818,
      "logits/rejected": -1.3666173219680786,
      "logps/chosen": -583.2642822265625,
      "logps/rejected": -1070.8175048828125,
      "loss": 0.0947,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.103260040283203,
      "rewards/margins": 6.282073020935059,
      "rewards/rejected": -9.385333061218262,
      "step": 930
    },
    {
      "epoch": 0.98,
      "grad_norm": 30.829530604495584,
      "learning_rate": 3.760945397705828e-10,
      "logits/chosen": -1.6098768711090088,
      "logits/rejected": -1.4137991666793823,
      "logps/chosen": -652.3986206054688,
      "logps/rejected": -1045.8623046875,
      "loss": 0.1041,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.067474365234375,
      "rewards/margins": 6.041169166564941,
      "rewards/rejected": -9.108643531799316,
      "step": 940
    },
    {
      "epoch": 0.99,
      "grad_norm": 28.95439781379653,
      "learning_rate": 4.17975992204056e-11,
      "logits/chosen": -1.6131422519683838,
      "logits/rejected": -1.4869133234024048,
      "logps/chosen": -605.0281372070312,
      "logps/rejected": -1031.686279296875,
      "loss": 0.0954,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.1735126972198486,
      "rewards/margins": 5.8094305992126465,
      "rewards/rejected": -8.982942581176758,
      "step": 950
    },
    {
      "epoch": 1.0,
      "step": 955,
      "total_flos": 0.0,
      "train_loss": 0.05187356284775659,
      "train_runtime": 7314.4586,
      "train_samples_per_second": 16.716,
      "train_steps_per_second": 0.131
    }
  ],
  "logging_steps": 10,
  "max_steps": 955,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}