{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 5.0,
  "eval_steps": 1000,
  "global_step": 1145,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.004366812227074236,
      "grad_norm": 0.47458936036527805,
      "learning_rate": 4.347826086956522e-08,
      "logits/chosen": -1.077685832977295,
      "logits/rejected": -0.9262686967849731,
      "logps/chosen": -270.19927978515625,
      "logps/rejected": -284.0829772949219,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.043668122270742356,
      "grad_norm": 0.523539374370341,
      "learning_rate": 4.347826086956522e-07,
      "logits/chosen": -0.9952558875083923,
      "logits/rejected": -1.0596376657485962,
      "logps/chosen": -300.6241760253906,
      "logps/rejected": -255.2275390625,
      "loss": 0.6929,
      "rewards/accuracies": 0.4583333432674408,
      "rewards/chosen": 0.0006611866410821676,
      "rewards/margins": 0.0006012354861013591,
      "rewards/rejected": 5.995116225676611e-05,
      "step": 10
    },
    {
      "epoch": 0.08733624454148471,
      "grad_norm": 0.5365126766074364,
      "learning_rate": 8.695652173913044e-07,
      "logits/chosen": -1.1060526371002197,
      "logits/rejected": -1.0136970281600952,
      "logps/chosen": -282.16522216796875,
      "logps/rejected": -316.65264892578125,
      "loss": 0.6925,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.0014794444432482123,
      "rewards/margins": 0.0008522253483533859,
      "rewards/rejected": 0.0006272192695178092,
      "step": 20
    },
    {
      "epoch": 0.13100436681222707,
      "grad_norm": 0.580247902784444,
      "learning_rate": 1.3043478260869566e-06,
      "logits/chosen": -1.0645825862884521,
      "logits/rejected": -1.1114238500595093,
      "logps/chosen": -287.91937255859375,
      "logps/rejected": -264.2998352050781,
      "loss": 0.69,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.00400827219709754,
      "rewards/margins": 0.006094515323638916,
      "rewards/rejected": -0.002086243825033307,
      "step": 30
    },
    {
      "epoch": 0.17467248908296942,
      "grad_norm": 0.45391259957771296,
      "learning_rate": 1.7391304347826088e-06,
      "logits/chosen": -1.0716311931610107,
      "logits/rejected": -1.1120277643203735,
      "logps/chosen": -285.3992919921875,
      "logps/rejected": -272.4431457519531,
      "loss": 0.6833,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.008399474434554577,
      "rewards/margins": 0.01975000463426113,
      "rewards/rejected": -0.011350530199706554,
      "step": 40
    },
    {
      "epoch": 0.2183406113537118,
      "grad_norm": 0.46277317667378487,
      "learning_rate": 2.173913043478261e-06,
      "logits/chosen": -1.111750841140747,
      "logits/rejected": -1.0212160348892212,
      "logps/chosen": -264.515380859375,
      "logps/rejected": -304.44281005859375,
      "loss": 0.6728,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.018858566880226135,
      "rewards/margins": 0.04448834806680679,
      "rewards/rejected": -0.02562977746129036,
      "step": 50
    },
    {
      "epoch": 0.26200873362445415,
      "grad_norm": 0.538292399167113,
      "learning_rate": 2.6086956521739132e-06,
      "logits/chosen": -1.070939302444458,
      "logits/rejected": -1.1072248220443726,
      "logps/chosen": -285.0442810058594,
      "logps/rejected": -262.81597900390625,
      "loss": 0.654,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.03614560887217522,
      "rewards/margins": 0.09129359573125839,
      "rewards/rejected": -0.05514797568321228,
      "step": 60
    },
    {
      "epoch": 0.3056768558951965,
      "grad_norm": 0.8400838184943603,
      "learning_rate": 3.043478260869566e-06,
      "logits/chosen": -1.1011688709259033,
      "logits/rejected": -1.1285476684570312,
      "logps/chosen": -279.41290283203125,
      "logps/rejected": -276.6390686035156,
      "loss": 0.6152,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.04447701945900917,
      "rewards/margins": 0.16786250472068787,
      "rewards/rejected": -0.123385488986969,
      "step": 70
    },
    {
      "epoch": 0.34934497816593885,
      "grad_norm": 1.363956535309491,
      "learning_rate": 3.4782608695652175e-06,
      "logits/chosen": -1.1735438108444214,
      "logits/rejected": -1.0770442485809326,
      "logps/chosen": -256.97564697265625,
      "logps/rejected": -330.7611083984375,
      "loss": 0.514,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10425807535648346,
      "rewards/margins": 0.4447970986366272,
      "rewards/rejected": -0.34053897857666016,
      "step": 80
    },
    {
      "epoch": 0.3930131004366812,
      "grad_norm": 1.0652559890564237,
      "learning_rate": 3.91304347826087e-06,
      "logits/chosen": -1.0946518182754517,
      "logits/rejected": -1.0877329111099243,
      "logps/chosen": -266.5739440917969,
      "logps/rejected": -361.3409729003906,
      "loss": 0.278,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2085898369550705,
      "rewards/margins": 1.1870415210723877,
      "rewards/rejected": -0.9784518480300903,
      "step": 90
    },
    {
      "epoch": 0.4366812227074236,
      "grad_norm": 0.3433521660739356,
      "learning_rate": 4.347826086956522e-06,
      "logits/chosen": -1.055832862854004,
      "logits/rejected": -0.9789533615112305,
      "logps/chosen": -219.6240692138672,
      "logps/rejected": -508.79132080078125,
      "loss": 0.0735,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.38825297355651855,
      "rewards/margins": 2.7440783977508545,
      "rewards/rejected": -2.355825662612915,
      "step": 100
    },
    {
      "epoch": 0.48034934497816595,
      "grad_norm": 0.11711005718135792,
      "learning_rate": 4.782608695652174e-06,
      "logits/chosen": -0.9485265016555786,
      "logits/rejected": -0.9038203358650208,
      "logps/chosen": -269.6318664550781,
      "logps/rejected": -716.6309814453125,
      "loss": 0.018,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2308163344860077,
      "rewards/margins": 4.760537147521973,
      "rewards/rejected": -4.529721260070801,
      "step": 110
    },
    {
      "epoch": 0.5240174672489083,
      "grad_norm": 0.056145921913412625,
      "learning_rate": 4.999709285361594e-06,
      "logits/chosen": -0.9236040115356445,
      "logits/rejected": -0.8355329632759094,
      "logps/chosen": -271.9773254394531,
      "logps/rejected": -864.2019653320312,
      "loss": 0.0057,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2535458505153656,
      "rewards/margins": 6.195461273193359,
      "rewards/rejected": -5.941915035247803,
      "step": 120
    },
    {
      "epoch": 0.5676855895196506,
      "grad_norm": 0.03098316075816059,
      "learning_rate": 4.997383973910631e-06,
      "logits/chosen": -0.8370451927185059,
      "logits/rejected": -0.6872082948684692,
      "logps/chosen": -263.49090576171875,
      "logps/rejected": -959.0076293945312,
      "loss": 0.0029,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2465527057647705,
      "rewards/margins": 7.134703159332275,
      "rewards/rejected": -6.888150215148926,
      "step": 130
    },
    {
      "epoch": 0.611353711790393,
      "grad_norm": 0.01781004140188894,
      "learning_rate": 4.9927355140895775e-06,
      "logits/chosen": -0.8542898297309875,
      "logits/rejected": -0.6182370185852051,
      "logps/chosen": -256.8946533203125,
      "logps/rejected": -1113.857666015625,
      "loss": 0.002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1456744223833084,
      "rewards/margins": 8.21844482421875,
      "rewards/rejected": -8.072771072387695,
      "step": 140
    },
    {
      "epoch": 0.6550218340611353,
      "grad_norm": 0.0626793578117912,
      "learning_rate": 4.985768230048011e-06,
      "logits/chosen": -0.8041971325874329,
      "logits/rejected": -0.645085334777832,
      "logps/chosen": -274.4373779296875,
      "logps/rejected": -1066.285400390625,
      "loss": 0.0019,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16038499772548676,
      "rewards/margins": 8.415651321411133,
      "rewards/rejected": -8.255266189575195,
      "step": 150
    },
    {
      "epoch": 0.6986899563318777,
      "grad_norm": 0.016666200992927045,
      "learning_rate": 4.976488602981748e-06,
      "logits/chosen": -0.8367046117782593,
      "logits/rejected": -0.6429373025894165,
      "logps/chosen": -258.5212707519531,
      "logps/rejected": -1201.1590576171875,
      "loss": 0.001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2543124556541443,
      "rewards/margins": 9.140569686889648,
      "rewards/rejected": -8.88625717163086,
      "step": 160
    },
    {
      "epoch": 0.74235807860262,
      "grad_norm": 0.07994746225935201,
      "learning_rate": 4.9649052651038255e-06,
      "logits/chosen": -0.7291176915168762,
      "logits/rejected": -0.5780690908432007,
      "logps/chosen": -279.2346496582031,
      "logps/rejected": -1126.931396484375,
      "loss": 0.0009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16093984246253967,
      "rewards/margins": 8.925050735473633,
      "rewards/rejected": -8.764110565185547,
      "step": 170
    },
    {
      "epoch": 0.7860262008733624,
      "grad_norm": 0.01171334168304518,
      "learning_rate": 4.9510289916145295e-06,
      "logits/chosen": -0.7453992962837219,
      "logits/rejected": -0.5254510641098022,
      "logps/chosen": -256.3668518066406,
      "logps/rejected": -1209.734619140625,
      "loss": 0.0006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1778395026922226,
      "rewards/margins": 9.462778091430664,
      "rewards/rejected": -9.28493881225586,
      "step": 180
    },
    {
      "epoch": 0.8296943231441049,
      "grad_norm": 0.017372174016347157,
      "learning_rate": 4.934872690677953e-06,
      "logits/chosen": -0.7389127612113953,
      "logits/rejected": -0.5030776858329773,
      "logps/chosen": -271.6781921386719,
      "logps/rejected": -1233.863037109375,
      "loss": 0.0007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.13182204961776733,
      "rewards/margins": 9.499404907226562,
      "rewards/rejected": -9.367582321166992,
      "step": 190
    },
    {
      "epoch": 0.8733624454148472,
      "grad_norm": 0.005099638792861995,
      "learning_rate": 4.9164513914144005e-06,
      "logits/chosen": -0.6438565254211426,
      "logits/rejected": -0.3725091516971588,
      "logps/chosen": -258.71893310546875,
      "logps/rejected": -1232.365478515625,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.11995329707860947,
      "rewards/margins": 9.731719017028809,
      "rewards/rejected": -9.611766815185547,
      "step": 200
    },
    {
      "epoch": 0.9170305676855895,
      "grad_norm": 0.007449286013455516,
      "learning_rate": 4.8957822299198045e-06,
      "logits/chosen": -0.7947207689285278,
      "logits/rejected": -0.5452377200126648,
      "logps/chosen": -277.94647216796875,
      "logps/rejected": -1344.936767578125,
      "loss": 0.0006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.07659594714641571,
      "rewards/margins": 10.560900688171387,
      "rewards/rejected": -10.484306335449219,
      "step": 210
    },
    {
      "epoch": 0.9606986899563319,
      "grad_norm": 0.025567537238137088,
      "learning_rate": 4.872884433325169e-06,
      "logits/chosen": -0.7050657272338867,
      "logits/rejected": -0.49923476576805115,
      "logps/chosen": -286.5049743652344,
      "logps/rejected": -1234.7254638671875,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.08888520300388336,
      "rewards/margins": 9.898573875427246,
      "rewards/rejected": -9.809688568115234,
      "step": 220
    },
    {
      "epoch": 1.0043668122270741,
      "grad_norm": 0.03935313476723805,
      "learning_rate": 4.847779301910868e-06,
      "logits/chosen": -0.7885675430297852,
      "logits/rejected": -0.48259955644607544,
      "logps/chosen": -235.9558563232422,
      "logps/rejected": -1333.6160888671875,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1478557288646698,
      "rewards/margins": 10.591659545898438,
      "rewards/rejected": -10.443804740905762,
      "step": 230
    },
    {
      "epoch": 1.0480349344978166,
      "grad_norm": 0.019331140450817564,
      "learning_rate": 4.820490189292415e-06,
      "logits/chosen": -0.6384273767471313,
      "logits/rejected": -0.32141706347465515,
      "logps/chosen": -256.14813232421875,
      "logps/rejected": -1328.8856201171875,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.03197110444307327,
      "rewards/margins": 10.722203254699707,
      "rewards/rejected": -10.690231323242188,
      "step": 240
    },
    {
      "epoch": 1.091703056768559,
      "grad_norm": 0.001917220346998511,
      "learning_rate": 4.791042480696179e-06,
      "logits/chosen": -0.8021818399429321,
      "logits/rejected": -0.5436561107635498,
      "logps/chosen": -285.2548828125,
      "logps/rejected": -1359.247802734375,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09929420799016953,
      "rewards/margins": 10.846672058105469,
      "rewards/rejected": -10.747377395629883,
      "step": 250
    },
    {
      "epoch": 1.1353711790393013,
      "grad_norm": 0.009816737966176867,
      "learning_rate": 4.759463569345205e-06,
      "logits/chosen": -0.7113679051399231,
      "logits/rejected": -0.45295238494873047,
      "logps/chosen": -268.17083740234375,
      "logps/rejected": -1343.6666259765625,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10034344345331192,
      "rewards/margins": 10.877279281616211,
      "rewards/rejected": -10.776936531066895,
      "step": 260
    },
    {
      "epoch": 1.1790393013100438,
      "grad_norm": 0.003536564037828518,
      "learning_rate": 4.725782830977145e-06,
      "logits/chosen": -0.7203564047813416,
      "logits/rejected": -0.4096035957336426,
      "logps/chosen": -257.78900146484375,
      "logps/rejected": -1433.7529296875,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.05666704103350639,
      "rewards/margins": 11.476617813110352,
      "rewards/rejected": -11.419949531555176,
      "step": 270
    },
    {
      "epoch": 1.222707423580786,
      "grad_norm": 0.0031083495470266283,
      "learning_rate": 4.690031596517984e-06,
      "logits/chosen": -0.6830414533615112,
      "logits/rejected": -0.3940260410308838,
      "logps/chosen": -252.91641235351562,
      "logps/rejected": -1419.6212158203125,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10215254127979279,
      "rewards/margins": 11.425537109375,
      "rewards/rejected": -11.323384284973145,
      "step": 280
    },
    {
      "epoch": 1.2663755458515285,
      "grad_norm": 0.0031010184167246683,
      "learning_rate": 4.652243122936987e-06,
      "logits/chosen": -0.7523640394210815,
      "logits/rejected": -0.4601876735687256,
      "logps/chosen": -267.94482421875,
      "logps/rejected": -1480.731201171875,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.141164630651474,
      "rewards/margins": 12.00882339477539,
      "rewards/rejected": -11.867658615112305,
      "step": 290
    },
    {
      "epoch": 1.3100436681222707,
      "grad_norm": 0.03893958758819322,
      "learning_rate": 4.612452562309975e-06,
      "logits/chosen": -0.6891266703605652,
      "logits/rejected": -0.3029942512512207,
      "logps/chosen": -241.3924560546875,
      "logps/rejected": -1440.799072265625,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.11804157495498657,
      "rewards/margins": 11.638472557067871,
      "rewards/rejected": -11.520431518554688,
      "step": 300
    },
    {
      "epoch": 1.3537117903930131,
      "grad_norm": 0.0020487177308325163,
      "learning_rate": 4.570696929119717e-06,
      "logits/chosen": -0.6615210771560669,
      "logits/rejected": -0.40224489569664,
      "logps/chosen": -264.6934814453125,
      "logps/rejected": -1409.147216796875,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0794425830245018,
      "rewards/margins": 11.551889419555664,
      "rewards/rejected": -11.47244644165039,
      "step": 310
    },
    {
      "epoch": 1.3973799126637554,
      "grad_norm": 0.0017833767780479168,
      "learning_rate": 4.527015065823841e-06,
      "logits/chosen": -0.6528028249740601,
      "logits/rejected": -0.4198477864265442,
      "logps/chosen": -277.9957580566406,
      "logps/rejected": -1393.375244140625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09100461006164551,
      "rewards/margins": 11.455917358398438,
      "rewards/rejected": -11.364913940429688,
      "step": 320
    },
    {
      "epoch": 1.4410480349344978,
      "grad_norm": 0.007670342609009089,
      "learning_rate": 4.481447606722309e-06,
      "logits/chosen": -0.6851311326026917,
      "logits/rejected": -0.44172611832618713,
      "logps/chosen": -280.6171875,
      "logps/rejected": -1444.6007080078125,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1056002750992775,
      "rewards/margins": 11.923059463500977,
      "rewards/rejected": -11.817459106445312,
      "step": 330
    },
    {
      "epoch": 1.48471615720524,
      "grad_norm": 0.004356336133031361,
      "learning_rate": 4.434036940158062e-06,
      "logits/chosen": -0.7159875631332397,
      "logits/rejected": -0.26825979351997375,
      "logps/chosen": -226.5546875,
      "logps/rejected": -1620.3592529296875,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.12559999525547028,
      "rewards/margins": 13.081733703613281,
      "rewards/rejected": -12.956133842468262,
      "step": 340
    },
    {
      "epoch": 1.5283842794759825,
      "grad_norm": 0.006924018476345942,
      "learning_rate": 4.384827169085993e-06,
      "logits/chosen": -0.722930908203125,
      "logits/rejected": -0.54079270362854,
      "logps/chosen": -305.4542541503906,
      "logps/rejected": -1445.2802734375,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.036442067474126816,
      "rewards/margins": 11.9239501953125,
      "rewards/rejected": -11.887507438659668,
      "step": 350
    },
    {
      "epoch": 1.572052401746725,
      "grad_norm": 0.00824202881851081,
      "learning_rate": 4.333864070046938e-06,
      "logits/chosen": -0.6452621221542358,
      "logits/rejected": -0.2813284993171692,
      "logps/chosen": -266.8562927246094,
      "logps/rejected": -1508.1966552734375,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.023952916264533997,
      "rewards/margins": 12.330470085144043,
      "rewards/rejected": -12.306516647338867,
      "step": 360
    },
    {
      "epoch": 1.6157205240174672,
      "grad_norm": 0.0028510754034206916,
      "learning_rate": 4.28119505058483e-06,
      "logits/chosen": -0.5809464454650879,
      "logits/rejected": -0.29732316732406616,
      "logps/chosen": -292.64154052734375,
      "logps/rejected": -1430.4862060546875,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08253034949302673,
      "rewards/margins": 11.704535484313965,
      "rewards/rejected": -11.787064552307129,
      "step": 370
    },
    {
      "epoch": 1.6593886462882095,
      "grad_norm": 0.004071842447232605,
      "learning_rate": 4.226869105146658e-06,
      "logits/chosen": -0.6976742148399353,
      "logits/rejected": -0.374077707529068,
      "logps/chosen": -267.6322326660156,
      "logps/rejected": -1627.9765625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.027708619832992554,
      "rewards/margins": 13.307157516479492,
      "rewards/rejected": -13.279451370239258,
      "step": 380
    },
    {
      "epoch": 1.703056768558952,
      "grad_norm": 0.0030757362239471637,
      "learning_rate": 4.170936769506222e-06,
      "logits/chosen": -0.6652247309684753,
      "logits/rejected": -0.3628092110157013,
      "logps/chosen": -288.57257080078125,
      "logps/rejected": -1520.739990234375,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.052382566034793854,
      "rewards/margins": 12.51613998413086,
      "rewards/rejected": -12.568521499633789,
      "step": 390
    },
    {
      "epoch": 1.7467248908296944,
      "grad_norm": 0.003598764645985237,
      "learning_rate": 4.1134500737541026e-06,
      "logits/chosen": -0.6975280046463013,
      "logits/rejected": -0.3748314380645752,
      "logps/chosen": -277.32843017578125,
      "logps/rejected": -1550.5653076171875,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.06987209618091583,
      "rewards/margins": 12.744722366333008,
      "rewards/rejected": -12.674848556518555,
      "step": 400
    },
    {
      "epoch": 1.7903930131004366,
      "grad_norm": 0.007268762647014337,
      "learning_rate": 4.054462493897569e-06,
      "logits/chosen": -0.7424625754356384,
      "logits/rejected": -0.30923086404800415,
      "logps/chosen": -234.93551635742188,
      "logps/rejected": -1707.387451171875,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10221578180789948,
      "rewards/margins": 13.916183471679688,
      "rewards/rejected": -13.81396770477295,
      "step": 410
    },
    {
      "epoch": 1.8340611353711789,
      "grad_norm": 0.0008727719544739926,
      "learning_rate": 3.994028902115439e-06,
      "logits/chosen": -0.6766982674598694,
      "logits/rejected": -0.22497108578681946,
      "logps/chosen": -246.74490356445312,
      "logps/rejected": -1593.43115234375,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.03030253015458584,
      "rewards/margins": 12.960153579711914,
      "rewards/rejected": -12.990457534790039,
      "step": 420
    },
    {
      "epoch": 1.8777292576419216,
      "grad_norm": 0.0015384551125673842,
      "learning_rate": 3.932205515714189e-06,
      "logits/chosen": -0.7006328701972961,
      "logits/rejected": -0.38360628485679626,
      "logps/chosen": -298.765380859375,
      "logps/rejected": -1612.6402587890625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06020375341176987,
      "rewards/margins": 13.307812690734863,
      "rewards/rejected": -13.368016242980957,
      "step": 430
    },
    {
      "epoch": 1.9213973799126638,
      "grad_norm": 0.008179209300348753,
      "learning_rate": 3.86904984483277e-06,
      "logits/chosen": -0.7087584137916565,
      "logits/rejected": -0.4246200919151306,
      "logps/chosen": -304.3985900878906,
      "logps/rejected": -1534.143798828125,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08279000222682953,
      "rewards/margins": 12.747483253479004,
      "rewards/rejected": -12.830273628234863,
      "step": 440
    },
    {
      "epoch": 1.965065502183406,
      "grad_norm": 0.0014160258405897827,
      "learning_rate": 3.8046206389447916e-06,
      "logits/chosen": -0.6982067823410034,
      "logits/rejected": -0.3726228177547455,
      "logps/chosen": -276.84039306640625,
      "logps/rejected": -1593.4991455078125,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.04725092276930809,
      "rewards/margins": 13.1595458984375,
      "rewards/rejected": -13.20679759979248,
      "step": 450
    },
    {
      "epoch": 2.0087336244541483,
      "grad_norm": 0.0017767298206001502,
      "learning_rate": 3.738977832207839e-06,
      "logits/chosen": -0.6970980763435364,
      "logits/rejected": -0.3680294156074524,
      "logps/chosen": -261.3289794921875,
      "logps/rejected": -1555.7623291015625,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0009517133003100753,
      "rewards/margins": 12.788881301879883,
      "rewards/rejected": -12.787931442260742,
      "step": 460
    },
    {
      "epoch": 2.052401746724891,
      "grad_norm": 0.001535205426650719,
      "learning_rate": 3.6721824877107588e-06,
      "logits/chosen": -0.7333104014396667,
      "logits/rejected": -0.45936185121536255,
      "logps/chosen": -308.5245666503906,
      "logps/rejected": -1585.037841796875,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0660587027668953,
      "rewards/margins": 13.10444450378418,
      "rewards/rejected": -13.170503616333008,
      "step": 470
    },
    {
      "epoch": 2.096069868995633,
      "grad_norm": 0.005158531675148358,
      "learning_rate": 3.604296740670768e-06,
      "logits/chosen": -0.7633829116821289,
      "logits/rejected": -0.33872467279434204,
      "logps/chosen": -260.2208557128906,
      "logps/rejected": -1724.6669921875,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.010257990099489689,
      "rewards/margins": 14.2053804397583,
      "rewards/rejected": -14.195123672485352,
      "step": 480
    },
    {
      "epoch": 2.1397379912663754,
      "grad_norm": 0.006727466994391036,
      "learning_rate": 3.5353837406332464e-06,
      "logits/chosen": -0.7862290143966675,
      "logits/rejected": -0.45894408226013184,
      "logps/chosen": -251.74417114257812,
      "logps/rejected": -1686.0699462890625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0987875908613205,
      "rewards/margins": 13.866643905639648,
      "rewards/rejected": -13.767855644226074,
      "step": 490
    },
    {
      "epoch": 2.183406113537118,
      "grad_norm": 0.005878784292188106,
      "learning_rate": 3.4655075927279576e-06,
      "logits/chosen": -0.7339245080947876,
      "logits/rejected": -0.42568492889404297,
      "logps/chosen": -266.9426574707031,
      "logps/rejected": -1574.525634765625,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.030187349766492844,
      "rewards/margins": 13.049545288085938,
      "rewards/rejected": -13.079732894897461,
      "step": 500
    },
    {
      "epoch": 2.2270742358078603,
      "grad_norm": 0.0017376152792194543,
      "learning_rate": 3.3947332980363552e-06,
      "logits/chosen": -0.5695234537124634,
      "logits/rejected": -0.19781272113323212,
      "logps/chosen": -270.3812561035156,
      "logps/rejected": -1615.6109619140625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.023705072700977325,
      "rewards/margins": 13.492143630981445,
      "rewards/rejected": -13.515848159790039,
      "step": 510
    },
    {
      "epoch": 2.2707423580786026,
      "grad_norm": 0.0008884061380883483,
      "learning_rate": 3.3231266931254546e-06,
      "logits/chosen": -0.7268373370170593,
      "logits/rejected": -0.4064566493034363,
      "logps/chosen": -288.7190246582031,
      "logps/rejected": -1659.5142822265625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.004935880657285452,
      "rewards/margins": 13.8540620803833,
      "rewards/rejected": -13.858996391296387,
      "step": 520
    },
    {
      "epoch": 2.314410480349345,
      "grad_norm": 0.0012752069272254601,
      "learning_rate": 3.250754388804495e-06,
      "logits/chosen": -0.7004576921463013,
      "logits/rejected": -0.3160724639892578,
      "logps/chosen": -260.39801025390625,
      "logps/rejected": -1695.2933349609375,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0040903836488723755,
      "rewards/margins": 14.074589729309082,
      "rewards/rejected": -14.078680038452148,
      "step": 530
    },
    {
      "epoch": 2.3580786026200875,
      "grad_norm": 0.0010551393838541235,
      "learning_rate": 3.1776837081613893e-06,
      "logits/chosen": -0.6237174868583679,
      "logits/rejected": -0.2775794267654419,
      "logps/chosen": -275.13775634765625,
      "logps/rejected": -1654.786376953125,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.028180230408906937,
      "rewards/margins": 13.729413032531738,
      "rewards/rejected": -13.757593154907227,
      "step": 540
    },
    {
      "epoch": 2.4017467248908297,
      "grad_norm": 0.010736168566735408,
      "learning_rate": 3.1039826239365754e-06,
      "logits/chosen": -0.6697626113891602,
      "logits/rejected": -0.3747546076774597,
      "logps/chosen": -265.6369323730469,
      "logps/rejected": -1629.97998046875,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.04310118779540062,
      "rewards/margins": 13.556437492370605,
      "rewards/rejected": -13.513336181640625,
      "step": 550
    },
    {
      "epoch": 2.445414847161572,
      "grad_norm": 0.001439648164211988,
      "learning_rate": 3.0297196952925533e-06,
      "logits/chosen": -0.6776083111763,
      "logits/rejected": -0.4190545678138733,
      "logps/chosen": -304.7749328613281,
      "logps/rejected": -1601.60791015625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.04750397801399231,
      "rewards/margins": 13.419174194335938,
      "rewards/rejected": -13.466680526733398,
      "step": 560
    },
    {
      "epoch": 2.489082969432314,
      "grad_norm": 0.0013072815191190771,
      "learning_rate": 2.9549640040379043e-06,
      "logits/chosen": -0.7228536605834961,
      "logits/rejected": -0.4134379029273987,
      "logps/chosen": -293.1953430175781,
      "logps/rejected": -1718.9398193359375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06104196235537529,
      "rewards/margins": 14.203579902648926,
      "rewards/rejected": -14.264620780944824,
      "step": 570
    },
    {
      "epoch": 2.532751091703057,
      "grad_norm": 0.0008461963585448364,
      "learning_rate": 2.8797850903651274e-06,
      "logits/chosen": -0.6659802794456482,
      "logits/rejected": -0.360958993434906,
      "logps/chosen": -270.3370056152344,
      "logps/rejected": -1665.4697265625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0008804827812127769,
      "rewards/margins": 13.956453323364258,
      "rewards/rejected": -13.9573335647583,
      "step": 580
    },
    {
      "epoch": 2.576419213973799,
      "grad_norm": 0.0023387701751764562,
      "learning_rate": 2.804252888162079e-06,
      "logits/chosen": -0.8161109685897827,
      "logits/rejected": -0.4084259569644928,
      "logps/chosen": -272.53216552734375,
      "logps/rejected": -1786.5172119140625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.03314363956451416,
      "rewards/margins": 14.708500862121582,
      "rewards/rejected": -14.741642951965332,
      "step": 590
    },
    {
      "epoch": 2.6200873362445414,
      "grad_norm": 0.0018232538424911746,
      "learning_rate": 2.7284376599571776e-06,
      "logits/chosen": -0.7031590342521667,
      "logits/rejected": -0.37507158517837524,
      "logps/chosen": -274.2117919921875,
      "logps/rejected": -1641.595458984375,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.04686348885297775,
      "rewards/margins": 13.67021656036377,
      "rewards/rejected": -13.717081069946289,
      "step": 600
    },
    {
      "epoch": 2.6637554585152836,
      "grad_norm": 0.000990528476960238,
      "learning_rate": 2.652409931558898e-06,
      "logits/chosen": -0.7374690175056458,
      "logits/rejected": -0.43666791915893555,
      "logps/chosen": -283.0863342285156,
      "logps/rejected": -1705.676513671875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.040116313844919205,
      "rewards/margins": 14.263372421264648,
      "rewards/rejected": -14.303491592407227,
      "step": 610
    },
    {
      "epoch": 2.7074235807860263,
      "grad_norm": 0.0009319562615286963,
      "learning_rate": 2.5762404264503538e-06,
      "logits/chosen": -0.6617239713668823,
      "logits/rejected": -0.28903594613075256,
      "logps/chosen": -267.43609619140625,
      "logps/rejected": -1748.5576171875,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.03135783225297928,
      "rewards/margins": 14.624079704284668,
      "rewards/rejected": -14.592721939086914,
      "step": 620
    },
    {
      "epoch": 2.7510917030567685,
      "grad_norm": 0.013526422059541143,
      "learning_rate": 2.5e-06,
      "logits/chosen": -0.7105010151863098,
      "logits/rejected": -0.4328455328941345,
      "logps/chosen": -281.528564453125,
      "logps/rejected": -1675.9830322265625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0196510162204504,
      "rewards/margins": 13.972375869750977,
      "rewards/rejected": -13.992025375366211,
      "step": 630
    },
    {
      "epoch": 2.7947598253275108,
      "grad_norm": 0.0003686574185361761,
      "learning_rate": 2.423759573549647e-06,
      "logits/chosen": -0.6781324148178101,
      "logits/rejected": -0.2962037920951843,
      "logps/chosen": -263.18975830078125,
      "logps/rejected": -1710.5943603515625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.022569647058844566,
      "rewards/margins": 14.243792533874512,
      "rewards/rejected": -14.266362190246582,
      "step": 640
    },
    {
      "epoch": 2.8384279475982535,
      "grad_norm": 0.0009413555719320386,
      "learning_rate": 2.3475900684411027e-06,
      "logits/chosen": -0.6239091157913208,
      "logits/rejected": -0.22716188430786133,
      "logps/chosen": -257.69842529296875,
      "logps/rejected": -1695.5068359375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07734155654907227,
      "rewards/margins": 14.082303047180176,
      "rewards/rejected": -14.159643173217773,
      "step": 650
    },
    {
      "epoch": 2.8820960698689957,
      "grad_norm": 0.000759592485033725,
      "learning_rate": 2.2715623400428228e-06,
      "logits/chosen": -0.694677472114563,
      "logits/rejected": -0.3776795268058777,
      "logps/chosen": -282.89202880859375,
      "logps/rejected": -1710.0386962890625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0032756985165178776,
      "rewards/margins": 14.365133285522461,
      "rewards/rejected": -14.361857414245605,
      "step": 660
    },
    {
      "epoch": 2.925764192139738,
      "grad_norm": 0.019784253772424126,
      "learning_rate": 2.1957471118379213e-06,
      "logits/chosen": -0.7745624780654907,
      "logits/rejected": -0.4288128912448883,
      "logps/chosen": -304.70574951171875,
      "logps/rejected": -1791.9283447265625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.05531958490610123,
      "rewards/margins": 14.97740364074707,
      "rewards/rejected": -15.032722473144531,
      "step": 670
    },
    {
      "epoch": 2.96943231441048,
      "grad_norm": 0.004147931478685181,
      "learning_rate": 2.120214909634873e-06,
      "logits/chosen": -0.7153472304344177,
      "logits/rejected": -0.43083199858665466,
      "logps/chosen": -297.9048767089844,
      "logps/rejected": -1724.3980712890625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.03991527482867241,
      "rewards/margins": 14.473901748657227,
      "rewards/rejected": -14.513816833496094,
      "step": 680
    },
    {
      "epoch": 3.013100436681223,
      "grad_norm": 0.0013738234416668116,
      "learning_rate": 2.045035995962097e-06,
      "logits/chosen": -0.7798169255256653,
      "logits/rejected": -0.43747028708457947,
      "logps/chosen": -279.0494079589844,
      "logps/rejected": -1814.386962890625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06170396879315376,
      "rewards/margins": 15.159332275390625,
      "rewards/rejected": -15.221035957336426,
      "step": 690
    },
    {
      "epoch": 3.056768558951965,
      "grad_norm": 0.0031209059954659443,
      "learning_rate": 1.970280304707447e-06,
      "logits/chosen": -0.714453399181366,
      "logits/rejected": -0.36034974455833435,
      "logps/chosen": -269.9892272949219,
      "logps/rejected": -1740.939208984375,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.002551588462665677,
      "rewards/margins": 14.63127326965332,
      "rewards/rejected": -14.62872314453125,
      "step": 700
    },
    {
      "epoch": 3.1004366812227073,
      "grad_norm": 0.00034210696214912053,
      "learning_rate": 1.8960173760634257e-06,
      "logits/chosen": -0.7932115793228149,
      "logits/rejected": -0.5189129114151001,
      "logps/chosen": -322.1393737792969,
      "logps/rejected": -1711.671875,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08139355480670929,
      "rewards/margins": 14.367878913879395,
      "rewards/rejected": -14.449272155761719,
      "step": 710
    },
    {
      "epoch": 3.14410480349345,
      "grad_norm": 0.0013002362609120892,
      "learning_rate": 1.8223162918386122e-06,
      "logits/chosen": -0.7495840787887573,
      "logits/rejected": -0.3989501893520355,
      "logps/chosen": -285.207275390625,
      "logps/rejected": -1773.8642578125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09717712551355362,
      "rewards/margins": 14.81560230255127,
      "rewards/rejected": -14.9127779006958,
      "step": 720
    },
    {
      "epoch": 3.1877729257641922,
      "grad_norm": 0.008009764157245521,
      "learning_rate": 1.7492456111955052e-06,
      "logits/chosen": -0.6571623086929321,
      "logits/rejected": -0.30920231342315674,
      "logps/chosen": -278.0492248535156,
      "logps/rejected": -1723.248046875,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0555524006485939,
      "rewards/margins": 14.493664741516113,
      "rewards/rejected": -14.549217224121094,
      "step": 730
    },
    {
      "epoch": 3.2314410480349345,
      "grad_norm": 0.002618631891345944,
      "learning_rate": 1.6768733068745468e-06,
      "logits/chosen": -0.7488084435462952,
      "logits/rejected": -0.4106837809085846,
      "logps/chosen": -282.20538330078125,
      "logps/rejected": -1826.658203125,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08377159386873245,
      "rewards/margins": 15.273724555969238,
      "rewards/rejected": -15.35749626159668,
      "step": 740
    },
    {
      "epoch": 3.2751091703056767,
      "grad_norm": 0.00023217650976632204,
      "learning_rate": 1.6052667019636462e-06,
      "logits/chosen": -0.7862998247146606,
      "logits/rejected": -0.44631749391555786,
      "logps/chosen": -276.70867919921875,
      "logps/rejected": -1797.5888671875,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.024743851274251938,
      "rewards/margins": 15.001612663269043,
      "rewards/rejected": -15.026357650756836,
      "step": 750
    },
    {
      "epoch": 3.3187772925764194,
      "grad_norm": 0.01865811559978352,
      "learning_rate": 1.5344924072720434e-06,
      "logits/chosen": -0.7188819050788879,
      "logits/rejected": -0.4172704815864563,
      "logps/chosen": -285.13140869140625,
      "logps/rejected": -1677.7828369140625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.061395276337862015,
      "rewards/margins": 14.092081069946289,
      "rewards/rejected": -14.153475761413574,
      "step": 760
    },
    {
      "epoch": 3.3624454148471616,
      "grad_norm": 0.00039610874442709104,
      "learning_rate": 1.4646162593667535e-06,
      "logits/chosen": -0.7479511499404907,
      "logits/rejected": -0.4791211187839508,
      "logps/chosen": -314.30169677734375,
      "logps/rejected": -1737.6015625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.053110092878341675,
      "rewards/margins": 14.654457092285156,
      "rewards/rejected": -14.70756721496582,
      "step": 770
    },
    {
      "epoch": 3.406113537117904,
      "grad_norm": 0.0020264522877018994,
      "learning_rate": 1.3957032593292319e-06,
      "logits/chosen": -0.6951030492782593,
      "logits/rejected": -0.34219890832901,
      "logps/chosen": -285.89373779296875,
      "logps/rejected": -1782.4566650390625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08678509294986725,
      "rewards/margins": 14.822759628295898,
      "rewards/rejected": -14.9095458984375,
      "step": 780
    },
    {
      "epoch": 3.449781659388646,
      "grad_norm": 0.0018935186827857544,
      "learning_rate": 1.3278175122892416e-06,
      "logits/chosen": -0.6852272748947144,
      "logits/rejected": -0.33605360984802246,
      "logps/chosen": -259.1459045410156,
      "logps/rejected": -1725.716796875,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.036944665014743805,
      "rewards/margins": 14.497613906860352,
      "rewards/rejected": -14.534558296203613,
      "step": 790
    },
    {
      "epoch": 3.493449781659389,
      "grad_norm": 0.015223829550391095,
      "learning_rate": 1.261022167792161e-06,
      "logits/chosen": -0.7397433519363403,
      "logits/rejected": -0.36608046293258667,
      "logps/chosen": -281.9190673828125,
      "logps/rejected": -1852.1806640625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.03298696130514145,
      "rewards/margins": 15.54675006866455,
      "rewards/rejected": -15.579736709594727,
      "step": 800
    },
    {
      "epoch": 3.537117903930131,
      "grad_norm": 0.010378807533355724,
      "learning_rate": 1.195379361055209e-06,
      "logits/chosen": -0.6590791940689087,
      "logits/rejected": -0.28567662835121155,
      "logps/chosen": -273.66009521484375,
      "logps/rejected": -1854.084716796875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.010397704318165779,
      "rewards/margins": 15.640172004699707,
      "rewards/rejected": -15.650568962097168,
      "step": 810
    },
    {
      "epoch": 3.5807860262008733,
      "grad_norm": 0.016204955970958463,
      "learning_rate": 1.1309501551672303e-06,
      "logits/chosen": -0.5972493886947632,
      "logits/rejected": -0.2114742249250412,
      "logps/chosen": -269.32476806640625,
      "logps/rejected": -1736.829833984375,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.052963800728321075,
      "rewards/margins": 14.625813484191895,
      "rewards/rejected": -14.678777694702148,
      "step": 820
    },
    {
      "epoch": 3.6244541484716155,
      "grad_norm": 0.0008894376854101312,
      "learning_rate": 1.0677944842858112e-06,
      "logits/chosen": -0.6675132513046265,
      "logits/rejected": -0.26750123500823975,
      "logps/chosen": -279.586181640625,
      "logps/rejected": -1758.112060546875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.050345610827207565,
      "rewards/margins": 14.880023002624512,
      "rewards/rejected": -14.930368423461914,
      "step": 830
    },
    {
      "epoch": 3.668122270742358,
      "grad_norm": 0.0004087774037557016,
      "learning_rate": 1.005971097884561e-06,
      "logits/chosen": -0.705937922000885,
      "logits/rejected": -0.3553905785083771,
      "logps/chosen": -300.83929443359375,
      "logps/rejected": -1823.3248291015625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14001181721687317,
      "rewards/margins": 15.270605087280273,
      "rewards/rejected": -15.410616874694824,
      "step": 840
    },
    {
      "epoch": 3.7117903930131004,
      "grad_norm": 0.0006949799978363079,
      "learning_rate": 9.455375061024319e-07,
      "logits/chosen": -0.7351874709129333,
      "logits/rejected": -0.3912436366081238,
      "logps/chosen": -282.2884826660156,
      "logps/rejected": -1754.314453125,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06751924753189087,
      "rewards/margins": 14.702392578125,
      "rewards/rejected": -14.769912719726562,
      "step": 850
    },
    {
      "epoch": 3.7554585152838427,
      "grad_norm": 0.006463044951812143,
      "learning_rate": 8.86549926245898e-07,
      "logits/chosen": -0.6956661939620972,
      "logits/rejected": -0.3181700110435486,
      "logps/chosen": -287.1238708496094,
      "logps/rejected": -1798.494384765625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10201337188482285,
      "rewards/margins": 14.971240043640137,
      "rewards/rejected": -15.073251724243164,
      "step": 860
    },
    {
      "epoch": 3.7991266375545854,
      "grad_norm": 0.0007296140394687955,
      "learning_rate": 8.29063230493779e-07,
      "logits/chosen": -0.7505240440368652,
      "logits/rejected": -0.3770305812358856,
      "logps/chosen": -265.15679931640625,
      "logps/rejected": -1858.8929443359375,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.02774529531598091,
      "rewards/margins": 15.584371566772461,
      "rewards/rejected": -15.6121187210083,
      "step": 870
    },
    {
      "epoch": 3.8427947598253276,
      "grad_norm": 0.008816147256943409,
      "learning_rate": 7.731308948533431e-07,
      "logits/chosen": -0.7113201022148132,
      "logits/rejected": -0.4147794246673584,
      "logps/chosen": -302.5737609863281,
      "logps/rejected": -1753.0826416015625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.04450942575931549,
      "rewards/margins": 14.832493782043457,
      "rewards/rejected": -14.87700366973877,
      "step": 880
    },
    {
      "epoch": 3.88646288209607,
      "grad_norm": 0.005372270249774767,
      "learning_rate": 7.188049494151703e-07,
      "logits/chosen": -0.6462150812149048,
      "logits/rejected": -0.2772805094718933,
      "logps/chosen": -277.2967834472656,
      "logps/rejected": -1723.9173583984375,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0981786921620369,
      "rewards/margins": 14.48997688293457,
      "rewards/rejected": -14.588155746459961,
      "step": 890
    },
    {
      "epoch": 3.930131004366812,
      "grad_norm": 0.0007509813802026183,
      "learning_rate": 6.661359299530626e-07,
      "logits/chosen": -0.6561316847801208,
      "logits/rejected": -0.47288408875465393,
      "logps/chosen": -322.491455078125,
      "logps/rejected": -1582.517333984375,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14367668330669403,
      "rewards/margins": 13.482531547546387,
      "rewards/rejected": -13.626208305358887,
      "step": 900
    },
    {
      "epoch": 3.9737991266375547,
      "grad_norm": 0.002905923093028766,
      "learning_rate": 6.151728309140071e-07,
      "logits/chosen": -0.7203381061553955,
      "logits/rejected": -0.382064551115036,
      "logps/chosen": -285.7363586425781,
      "logps/rejected": -1801.4898681640625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07807151228189468,
      "rewards/margins": 15.15675163269043,
      "rewards/rejected": -15.234823226928711,
      "step": 910
    },
    {
      "epoch": 4.0174672489082965,
      "grad_norm": 0.0020680720730432436,
      "learning_rate": 5.659630598419391e-07,
      "logits/chosen": -0.724728524684906,
      "logits/rejected": -0.3629612326622009,
      "logps/chosen": -295.44073486328125,
      "logps/rejected": -1742.753173828125,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.13527199625968933,
      "rewards/margins": 14.602720260620117,
      "rewards/rejected": -14.737993240356445,
      "step": 920
    },
    {
      "epoch": 4.06113537117904,
      "grad_norm": 0.012272686016895323,
      "learning_rate": 5.185523932776923e-07,
      "logits/chosen": -0.6944893598556519,
      "logits/rejected": -0.3898335099220276,
      "logps/chosen": -287.7585144042969,
      "logps/rejected": -1794.3408203125,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.05893585830926895,
      "rewards/margins": 15.143773078918457,
      "rewards/rejected": -15.202710151672363,
      "step": 930
    },
    {
      "epoch": 4.104803493449782,
      "grad_norm": 0.0053926273432672945,
      "learning_rate": 4.7298493417616024e-07,
      "logits/chosen": -0.6781089305877686,
      "logits/rejected": -0.40882062911987305,
      "logps/chosen": -303.1540222167969,
      "logps/rejected": -1783.308837890625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1440161019563675,
      "rewards/margins": 14.9929780960083,
      "rewards/rejected": -15.136993408203125,
      "step": 940
    },
    {
      "epoch": 4.148471615720524,
      "grad_norm": 0.0010027396216351911,
      "learning_rate": 4.293030708802834e-07,
      "logits/chosen": -0.7281903028488159,
      "logits/rejected": -0.4005254805088043,
      "logps/chosen": -281.6249694824219,
      "logps/rejected": -1874.48828125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.086191326379776,
      "rewards/margins": 15.799471855163574,
      "rewards/rejected": -15.885663032531738,
      "step": 950
    },
    {
      "epoch": 4.192139737991266,
      "grad_norm": 0.0027350722737438755,
      "learning_rate": 3.875474376900254e-07,
      "logits/chosen": -0.6054420471191406,
      "logits/rejected": -0.2656559944152832,
      "logps/chosen": -283.4683532714844,
      "logps/rejected": -1737.546630859375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1006595641374588,
      "rewards/margins": 14.711895942687988,
      "rewards/rejected": -14.812556266784668,
      "step": 960
    },
    {
      "epoch": 4.235807860262009,
      "grad_norm": 0.006760599239543763,
      "learning_rate": 3.4775687706301437e-07,
      "logits/chosen": -0.7967843413352966,
      "logits/rejected": -0.42173296213150024,
      "logps/chosen": -270.96148681640625,
      "logps/rejected": -1896.1468505859375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.02015690691769123,
      "rewards/margins": 15.979083061218262,
      "rewards/rejected": -15.999239921569824,
      "step": 970
    },
    {
      "epoch": 4.279475982532751,
      "grad_norm": 0.0008520732124624599,
      "learning_rate": 3.0996840348201717e-07,
      "logits/chosen": -0.7633962631225586,
      "logits/rejected": -0.4977475106716156,
      "logps/chosen": -313.98822021484375,
      "logps/rejected": -1697.9468994140625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18659470975399017,
      "rewards/margins": 14.239301681518555,
      "rewards/rejected": -14.425895690917969,
      "step": 980
    },
    {
      "epoch": 4.323144104803493,
      "grad_norm": 0.0013051411886959893,
      "learning_rate": 2.742171690228562e-07,
      "logits/chosen": -0.7120814323425293,
      "logits/rejected": -0.3730967342853546,
      "logps/chosen": -298.5450134277344,
      "logps/rejected": -1744.666015625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15538612008094788,
      "rewards/margins": 14.598345756530762,
      "rewards/rejected": -14.75373363494873,
      "step": 990
    },
    {
      "epoch": 4.366812227074236,
      "grad_norm": 0.002698793985143946,
      "learning_rate": 2.405364306547955e-07,
      "logits/chosen": -0.7172932028770447,
      "logits/rejected": -0.47384780645370483,
      "logps/chosen": -322.2046813964844,
      "logps/rejected": -1694.6380615234375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10717038065195084,
      "rewards/margins": 14.218195915222168,
      "rewards/rejected": -14.325366020202637,
      "step": 1000
    },
    {
      "epoch": 4.4104803493449785,
      "grad_norm": 0.00023879898445148078,
      "learning_rate": 2.0895751930382125e-07,
      "logits/chosen": -0.6613215804100037,
      "logits/rejected": -0.33443623781204224,
      "logps/chosen": -295.16278076171875,
      "logps/rejected": -1768.1917724609375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.03670508787035942,
      "rewards/margins": 14.837631225585938,
      "rewards/rejected": -14.874334335327148,
      "step": 1010
    },
    {
      "epoch": 4.454148471615721,
      "grad_norm": 0.00023568210161956636,
      "learning_rate": 1.7950981070758488e-07,
      "logits/chosen": -0.6994235515594482,
      "logits/rejected": -0.41796016693115234,
      "logps/chosen": -287.3175048828125,
      "logps/rejected": -1685.6611328125,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08301335573196411,
      "rewards/margins": 14.271142959594727,
      "rewards/rejected": -14.354156494140625,
      "step": 1020
    },
    {
      "epoch": 4.497816593886463,
      "grad_norm": 0.0073133867021571005,
      "learning_rate": 1.5222069808913303e-07,
      "logits/chosen": -0.6575132012367249,
      "logits/rejected": -0.3197494447231293,
      "logps/chosen": -300.7660217285156,
      "logps/rejected": -1779.563232421875,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06065019965171814,
      "rewards/margins": 15.048309326171875,
      "rewards/rejected": -15.10895824432373,
      "step": 1030
    },
    {
      "epoch": 4.541484716157205,
      "grad_norm": 0.00034831682457422486,
      "learning_rate": 1.271155666748311e-07,
      "logits/chosen": -0.7639699578285217,
      "logits/rejected": -0.43667006492614746,
      "logps/chosen": -308.33502197265625,
      "logps/rejected": -1791.947998046875,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1204371303319931,
      "rewards/margins": 15.054000854492188,
      "rewards/rejected": -15.1744384765625,
      "step": 1040
    },
    {
      "epoch": 4.585152838427947,
      "grad_norm": 0.0003771930949580811,
      "learning_rate": 1.0421777008019663e-07,
      "logits/chosen": -0.7766512632369995,
      "logits/rejected": -0.3912786841392517,
      "logps/chosen": -293.3248596191406,
      "logps/rejected": -1899.7158203125,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07415080815553665,
      "rewards/margins": 15.951011657714844,
      "rewards/rejected": -16.02515983581543,
      "step": 1050
    },
    {
      "epoch": 4.62882096069869,
      "grad_norm": 0.0005156543022691973,
      "learning_rate": 8.354860858560021e-08,
      "logits/chosen": -0.7269026637077332,
      "logits/rejected": -0.3382445275783539,
      "logps/chosen": -295.33038330078125,
      "logps/rejected": -1880.168212890625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10000383853912354,
      "rewards/margins": 15.843725204467773,
      "rewards/rejected": -15.94372844696045,
      "step": 1060
    },
    {
      "epoch": 4.672489082969433,
      "grad_norm": 0.0012206160223788651,
      "learning_rate": 6.512730932204698e-08,
      "logits/chosen": -0.653832197189331,
      "logits/rejected": -0.3418962359428406,
      "logps/chosen": -320.64703369140625,
      "logps/rejected": -1710.6214599609375,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06000875309109688,
      "rewards/margins": 14.490182876586914,
      "rewards/rejected": -14.550191879272461,
      "step": 1070
    },
    {
      "epoch": 4.716157205240175,
      "grad_norm": 0.002712454997690386,
      "learning_rate": 4.897100838547081e-08,
      "logits/chosen": -0.6822378039360046,
      "logits/rejected": -0.3716619312763214,
      "logps/chosen": -306.8667907714844,
      "logps/rejected": -1840.0302734375,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12169905006885529,
      "rewards/margins": 15.402539253234863,
      "rewards/rejected": -15.524238586425781,
      "step": 1080
    },
    {
      "epoch": 4.759825327510917,
      "grad_norm": 0.00027359144331405783,
      "learning_rate": 3.5094734896174985e-08,
      "logits/chosen": -0.7298842668533325,
      "logits/rejected": -0.33024948835372925,
      "logps/chosen": -271.23095703125,
      "logps/rejected": -1889.5804443359375,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09704066812992096,
      "rewards/margins": 15.853654861450195,
      "rewards/rejected": -15.950695991516113,
      "step": 1090
    },
    {
      "epoch": 4.8034934497816595,
      "grad_norm": 0.0027199074701782094,
      "learning_rate": 2.351139701825267e-08,
      "logits/chosen": -0.6037005186080933,
      "logits/rejected": -0.26832693815231323,
      "logps/chosen": -272.974853515625,
      "logps/rejected": -1676.4384765625,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06281115859746933,
      "rewards/margins": 14.172335624694824,
      "rewards/rejected": -14.235147476196289,
      "step": 1100
    },
    {
      "epoch": 4.847161572052402,
      "grad_norm": 0.0007236973858890073,
      "learning_rate": 1.4231769951990326e-08,
      "logits/chosen": -0.6609109044075012,
      "logits/rejected": -0.33717867732048035,
      "logps/chosen": -295.76898193359375,
      "logps/rejected": -1793.028076171875,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19633208215236664,
      "rewards/margins": 15.07104206085205,
      "rewards/rejected": -15.267372131347656,
      "step": 1110
    },
    {
      "epoch": 4.890829694323144,
      "grad_norm": 0.0002222821524569821,
      "learning_rate": 7.264485910423447e-09,
      "logits/chosen": -0.6304144263267517,
      "logits/rejected": -0.2548811733722687,
      "logps/chosen": -274.11383056640625,
      "logps/rejected": -1818.213134765625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.09864435344934464,
      "rewards/margins": 15.282160758972168,
      "rewards/rejected": -15.380805969238281,
      "step": 1120
    },
    {
      "epoch": 4.934497816593886,
      "grad_norm": 0.0096819850101334,
      "learning_rate": 2.6160260893692833e-09,
      "logits/chosen": -0.6611170768737793,
      "logits/rejected": -0.354715496301651,
      "logps/chosen": -308.61688232421875,
      "logps/rejected": -1704.0927734375,
      "loss": 0.0001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.12653221189975739,
      "rewards/margins": 14.360208511352539,
      "rewards/rejected": -14.486740112304688,
      "step": 1130
    },
    {
      "epoch": 4.978165938864628,
      "grad_norm": 0.0002604769137792249,
      "learning_rate": 2.9071463840540936e-10,
      "logits/chosen": -0.7605892419815063,
      "logits/rejected": -0.5090384483337402,
      "logps/chosen": -336.0806579589844,
      "logps/rejected": -1813.3853759765625,
      "loss": 0.0,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.10771320015192032,
      "rewards/margins": 15.362016677856445,
      "rewards/rejected": -15.469728469848633,
      "step": 1140
    },
    {
      "epoch": 5.0,
      "step": 1145,
      "total_flos": 0.0,
      "train_loss": 0.04901049308554833,
      "train_runtime": 10890.5088,
      "train_samples_per_second": 6.722,
      "train_steps_per_second": 0.105
    }
  ],
  "logging_steps": 10,
  "max_steps": 1145,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 5,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}