{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 4.99746835443038,
  "eval_steps": 100,
  "global_step": 987,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.005063291139240506,
      "grad_norm": 520145.53108452284,
      "learning_rate": 1.4084507042253521e-09,
      "logits/chosen": -16.270591735839844,
      "logits/rejected": -16.343984603881836,
      "logps/chosen": -186.17276000976562,
      "logps/rejected": -175.8095703125,
      "loss": 122464.3125,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.05063291139240506,
      "grad_norm": 501181.05233525805,
      "learning_rate": 1.408450704225352e-08,
      "logits/chosen": -17.194263458251953,
      "logits/rejected": -17.04476547241211,
      "logps/chosen": -220.64031982421875,
      "logps/rejected": -220.79531860351562,
      "loss": 124716.2917,
      "rewards/accuracies": 0.4583333432674408,
      "rewards/chosen": 1.5937095554363623e-07,
      "rewards/margins": 1.5358187738456763e-05,
      "rewards/rejected": -1.5198814253380988e-05,
      "step": 10
    },
    {
      "epoch": 0.10126582278481013,
      "grad_norm": 537058.8643033113,
      "learning_rate": 2.816901408450704e-08,
      "logits/chosen": -16.468345642089844,
      "logits/rejected": -16.397050857543945,
      "logps/chosen": -238.16464233398438,
      "logps/rejected": -234.13320922851562,
      "loss": 125132.075,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -7.482715773221571e-06,
      "rewards/margins": 1.4337347238324583e-05,
      "rewards/rejected": -2.1820069378009066e-05,
      "step": 20
    },
    {
      "epoch": 0.1518987341772152,
      "grad_norm": 457257.68659374124,
      "learning_rate": 4.2253521126760564e-08,
      "logits/chosen": -16.952747344970703,
      "logits/rejected": -16.70650863647461,
      "logps/chosen": -242.9259490966797,
      "logps/rejected": -242.9457244873047,
      "loss": 124660.25,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -3.21022052958142e-05,
      "rewards/margins": 3.5706521885003895e-05,
      "rewards/rejected": -6.780872354283929e-05,
      "step": 30
    },
    {
      "epoch": 0.20253164556962025,
      "grad_norm": 520967.9129238899,
      "learning_rate": 5.633802816901408e-08,
      "logits/chosen": -16.920284271240234,
      "logits/rejected": -16.8529052734375,
      "logps/chosen": -243.7992706298828,
      "logps/rejected": -244.38906860351562,
      "loss": 124148.0625,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0001235240779351443,
      "rewards/margins": 8.850651647662744e-05,
      "rewards/rejected": -0.00021203060168772936,
      "step": 40
    },
    {
      "epoch": 0.25316455696202533,
      "grad_norm": 722258.4292859514,
      "learning_rate": 7.042253521126761e-08,
      "logits/chosen": -16.24307632446289,
      "logits/rejected": -16.294937133789062,
      "logps/chosen": -238.68148803710938,
      "logps/rejected": -240.46337890625,
      "loss": 125272.85,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.00024854010553099215,
      "rewards/margins": -7.368279329966754e-05,
      "rewards/rejected": -0.00017485734133515507,
      "step": 50
    },
    {
      "epoch": 0.3037974683544304,
      "grad_norm": 521725.51159479923,
      "learning_rate": 8.450704225352113e-08,
      "logits/chosen": -16.547048568725586,
      "logits/rejected": -16.562244415283203,
      "logps/chosen": -234.24453735351562,
      "logps/rejected": -236.03823852539062,
      "loss": 123692.1,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.00036723288940265775,
      "rewards/margins": 6.122588274592999e-06,
      "rewards/rejected": -0.00037335552042350173,
      "step": 60
    },
    {
      "epoch": 0.35443037974683544,
      "grad_norm": 446768.20251500694,
      "learning_rate": 9.859154929577463e-08,
      "logits/chosen": -16.514156341552734,
      "logits/rejected": -16.41303062438965,
      "logps/chosen": -240.8957061767578,
      "logps/rejected": -235.2915496826172,
      "loss": 125937.8,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.00028800699510611594,
      "rewards/margins": 0.00021416530944406986,
      "rewards/rejected": -0.0005021723336540163,
      "step": 70
    },
    {
      "epoch": 0.4050632911392405,
      "grad_norm": 463557.5011981856,
      "learning_rate": 1.1267605633802817e-07,
      "logits/chosen": -16.711376190185547,
      "logits/rejected": -16.489612579345703,
      "logps/chosen": -243.5523681640625,
      "logps/rejected": -228.8307342529297,
      "loss": 125818.525,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0005035396316088736,
      "rewards/margins": 6.90509841660969e-05,
      "rewards/rejected": -0.0005725906230509281,
      "step": 80
    },
    {
      "epoch": 0.45569620253164556,
      "grad_norm": 465137.87035599066,
      "learning_rate": 1.2676056338028167e-07,
      "logits/chosen": -17.326900482177734,
      "logits/rejected": -17.396936416625977,
      "logps/chosen": -240.1623077392578,
      "logps/rejected": -234.27578735351562,
      "loss": 123894.4,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.0008088796166703105,
      "rewards/margins": -0.00010466824460308999,
      "rewards/rejected": -0.0007042114739306271,
      "step": 90
    },
    {
      "epoch": 0.5063291139240507,
      "grad_norm": 505006.4054603859,
      "learning_rate": 1.4084507042253522e-07,
      "logits/chosen": -16.5346736907959,
      "logits/rejected": -16.46234893798828,
      "logps/chosen": -238.9674530029297,
      "logps/rejected": -235.36239624023438,
      "loss": 126640.2125,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.0006280581001192331,
      "rewards/margins": 2.4443055735900998e-05,
      "rewards/rejected": -0.0006525011267513037,
      "step": 100
    },
    {
      "epoch": 0.5569620253164557,
      "grad_norm": 475489.46555727004,
      "learning_rate": 1.549295774647887e-07,
      "logits/chosen": -16.67499351501465,
      "logits/rejected": -16.584075927734375,
      "logps/chosen": -240.5388946533203,
      "logps/rejected": -239.03366088867188,
      "loss": 122706.3,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.0006213908782228827,
      "rewards/margins": 0.00010261077841278166,
      "rewards/rejected": -0.0007240016711875796,
      "step": 110
    },
    {
      "epoch": 0.6075949367088608,
      "grad_norm": 492764.07090207015,
      "learning_rate": 1.6901408450704225e-07,
      "logits/chosen": -16.746532440185547,
      "logits/rejected": -16.617717742919922,
      "logps/chosen": -227.05398559570312,
      "logps/rejected": -225.60214233398438,
      "loss": 126588.925,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.0008889889577403665,
      "rewards/margins": 2.5076475139940158e-05,
      "rewards/rejected": -0.0009140653419308364,
      "step": 120
    },
    {
      "epoch": 0.6582278481012658,
      "grad_norm": 511084.4558498889,
      "learning_rate": 1.8309859154929577e-07,
      "logits/chosen": -16.747934341430664,
      "logits/rejected": -16.733430862426758,
      "logps/chosen": -240.7227325439453,
      "logps/rejected": -240.2967529296875,
      "loss": 125175.5125,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.0010187395382672548,
      "rewards/margins": 0.002705145161598921,
      "rewards/rejected": -0.003723885165527463,
      "step": 130
    },
    {
      "epoch": 0.7088607594936709,
      "grad_norm": 540454.6644647518,
      "learning_rate": 1.9718309859154927e-07,
      "logits/chosen": -16.1859073638916,
      "logits/rejected": -16.264835357666016,
      "logps/chosen": -231.37173461914062,
      "logps/rejected": -227.0606689453125,
      "loss": 126058.6375,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.0008466474828310311,
      "rewards/margins": -4.7403918870259076e-05,
      "rewards/rejected": -0.0007992436294443905,
      "step": 140
    },
    {
      "epoch": 0.759493670886076,
      "grad_norm": 503077.16971538117,
      "learning_rate": 2.112676056338028e-07,
      "logits/chosen": -17.280269622802734,
      "logits/rejected": -17.093780517578125,
      "logps/chosen": -238.0977325439453,
      "logps/rejected": -238.93212890625,
      "loss": 126646.125,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.0008783842204138637,
      "rewards/margins": 0.00026031016022898257,
      "rewards/rejected": -0.0011386943515390158,
      "step": 150
    },
    {
      "epoch": 0.810126582278481,
      "grad_norm": 541715.9624559938,
      "learning_rate": 2.2535211267605633e-07,
      "logits/chosen": -16.782550811767578,
      "logits/rejected": -16.79593276977539,
      "logps/chosen": -250.48593139648438,
      "logps/rejected": -249.44924926757812,
      "loss": 124718.425,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.0012445250758901238,
      "rewards/margins": 5.686017539119348e-05,
      "rewards/rejected": -0.0013013852294534445,
      "step": 160
    },
    {
      "epoch": 0.8607594936708861,
      "grad_norm": 548905.0358445289,
      "learning_rate": 2.394366197183098e-07,
      "logits/chosen": -17.04167938232422,
      "logits/rejected": -16.985572814941406,
      "logps/chosen": -255.06942749023438,
      "logps/rejected": -260.38128662109375,
      "loss": 125650.7625,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0021144188940525055,
      "rewards/margins": 0.001583110773935914,
      "rewards/rejected": -0.003697529900819063,
      "step": 170
    },
    {
      "epoch": 0.9113924050632911,
      "grad_norm": 561549.4959644328,
      "learning_rate": 2.5352112676056334e-07,
      "logits/chosen": -16.703407287597656,
      "logits/rejected": -16.487037658691406,
      "logps/chosen": -232.529052734375,
      "logps/rejected": -228.3297576904297,
      "loss": 127298.1375,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.002397818025201559,
      "rewards/margins": 0.0011578220874071121,
      "rewards/rejected": -0.003555640112608671,
      "step": 180
    },
    {
      "epoch": 0.9620253164556962,
      "grad_norm": 565071.0053763993,
      "learning_rate": 2.6760563380281686e-07,
      "logits/chosen": -16.11090660095215,
      "logits/rejected": -16.053157806396484,
      "logps/chosen": -239.39205932617188,
      "logps/rejected": -235.435791015625,
      "loss": 127009.225,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.0010291519574820995,
      "rewards/margins": 0.00019036220328416675,
      "rewards/rejected": -0.0012195140589028597,
      "step": 190
    },
    {
      "epoch": 1.0126582278481013,
      "grad_norm": 497332.98430491646,
      "learning_rate": 2.8169014084507043e-07,
      "logits/chosen": -16.127140045166016,
      "logits/rejected": -15.988116264343262,
      "logps/chosen": -225.9070587158203,
      "logps/rejected": -227.90145874023438,
      "loss": 126358.875,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0012379485415294766,
      "rewards/margins": 0.0006675361073575914,
      "rewards/rejected": -0.001905484707094729,
      "step": 200
    },
    {
      "epoch": 1.0632911392405062,
      "grad_norm": 935063.760892245,
      "learning_rate": 2.957746478873239e-07,
      "logits/chosen": -16.701793670654297,
      "logits/rejected": -16.669902801513672,
      "logps/chosen": -230.3677520751953,
      "logps/rejected": -229.03921508789062,
      "loss": 124250.775,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.0006614397279918194,
      "rewards/margins": 0.003238010685890913,
      "rewards/rejected": -0.003899450646713376,
      "step": 210
    },
    {
      "epoch": 1.1139240506329113,
      "grad_norm": 517399.2020129059,
      "learning_rate": 3.098591549295774e-07,
      "logits/chosen": -16.413972854614258,
      "logits/rejected": -16.371458053588867,
      "logps/chosen": -247.8984832763672,
      "logps/rejected": -249.5322723388672,
      "loss": 124993.7375,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.0012706981506198645,
      "rewards/margins": 0.003060612827539444,
      "rewards/rejected": -0.004331310745328665,
      "step": 220
    },
    {
      "epoch": 1.1645569620253164,
      "grad_norm": 499036.7717944408,
      "learning_rate": 3.23943661971831e-07,
      "logits/chosen": -15.908624649047852,
      "logits/rejected": -15.847338676452637,
      "logps/chosen": -236.7013397216797,
      "logps/rejected": -239.3136749267578,
      "loss": 122842.5,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.0006655483739450574,
      "rewards/margins": 0.0032406128011643887,
      "rewards/rejected": -0.0039061610586941242,
      "step": 230
    },
    {
      "epoch": 1.2151898734177216,
      "grad_norm": 540681.7856619481,
      "learning_rate": 3.380281690140845e-07,
      "logits/chosen": -16.052249908447266,
      "logits/rejected": -15.99653148651123,
      "logps/chosen": -229.74832153320312,
      "logps/rejected": -230.9803009033203,
      "loss": 124587.3625,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.0007962372037582099,
      "rewards/margins": 0.0025483998470008373,
      "rewards/rejected": -0.003344637108966708,
      "step": 240
    },
    {
      "epoch": 1.2658227848101267,
      "grad_norm": 1023950.8355601664,
      "learning_rate": 3.52112676056338e-07,
      "logits/chosen": -15.299288749694824,
      "logits/rejected": -15.215815544128418,
      "logps/chosen": -231.2301788330078,
      "logps/rejected": -232.03359985351562,
      "loss": 121822.4,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -7.512583579227794e-06,
      "rewards/margins": 0.003883513854816556,
      "rewards/rejected": -0.0038910270668566227,
      "step": 250
    },
    {
      "epoch": 1.3164556962025316,
      "grad_norm": 620253.8184950812,
      "learning_rate": 3.6619718309859155e-07,
      "logits/chosen": -16.167770385742188,
      "logits/rejected": -15.915590286254883,
      "logps/chosen": -238.9904327392578,
      "logps/rejected": -239.73953247070312,
      "loss": 123388.8625,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.00017356239550281316,
      "rewards/margins": 0.0050824107602238655,
      "rewards/rejected": -0.005255972500890493,
      "step": 260
    },
    {
      "epoch": 1.3670886075949367,
      "grad_norm": 575104.3218096169,
      "learning_rate": 3.8028169014084507e-07,
      "logits/chosen": -15.480558395385742,
      "logits/rejected": -15.386639595031738,
      "logps/chosen": -241.60879516601562,
      "logps/rejected": -250.003173828125,
      "loss": 123555.7,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.001139859901741147,
      "rewards/margins": 0.005077657289803028,
      "rewards/rejected": -0.0062175169587135315,
      "step": 270
    },
    {
      "epoch": 1.4177215189873418,
      "grad_norm": 601224.4433091934,
      "learning_rate": 3.9436619718309853e-07,
      "logits/chosen": -15.266016960144043,
      "logits/rejected": -15.313554763793945,
      "logps/chosen": -230.73397827148438,
      "logps/rejected": -237.3317108154297,
      "loss": 125556.675,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.0007209269679151475,
      "rewards/margins": 0.00534270191565156,
      "rewards/rejected": -0.004621774889528751,
      "step": 280
    },
    {
      "epoch": 1.4683544303797469,
      "grad_norm": 751936.3077706753,
      "learning_rate": 4.084507042253521e-07,
      "logits/chosen": -14.600263595581055,
      "logits/rejected": -14.538311958312988,
      "logps/chosen": -224.1177520751953,
      "logps/rejected": -226.97879028320312,
      "loss": 123584.675,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.0011863496620208025,
      "rewards/margins": 0.007649322040379047,
      "rewards/rejected": -0.006462973542511463,
      "step": 290
    },
    {
      "epoch": 1.518987341772152,
      "grad_norm": 575660.5828565176,
      "learning_rate": 4.225352112676056e-07,
      "logits/chosen": -14.935551643371582,
      "logits/rejected": -15.062429428100586,
      "logps/chosen": -235.7123565673828,
      "logps/rejected": -245.36181640625,
      "loss": 122562.1375,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0014863747637718916,
      "rewards/margins": 0.0057060932740569115,
      "rewards/rejected": -0.0042197187431156635,
      "step": 300
    },
    {
      "epoch": 1.5696202531645569,
      "grad_norm": 619514.1083852616,
      "learning_rate": 4.366197183098591e-07,
      "logits/chosen": -14.678690910339355,
      "logits/rejected": -14.617218017578125,
      "logps/chosen": -229.6386260986328,
      "logps/rejected": -234.1474151611328,
      "loss": 123630.225,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.0006864996394142509,
      "rewards/margins": 0.004933560267090797,
      "rewards/rejected": -0.004247060976922512,
      "step": 310
    },
    {
      "epoch": 1.620253164556962,
      "grad_norm": 738538.1512211321,
      "learning_rate": 4.5070422535211266e-07,
      "logits/chosen": -14.131611824035645,
      "logits/rejected": -14.156657218933105,
      "logps/chosen": -241.20156860351562,
      "logps/rejected": -248.2321319580078,
      "loss": 124158.6,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.0009155808947980404,
      "rewards/margins": 0.006913213524967432,
      "rewards/rejected": -0.007828795351088047,
      "step": 320
    },
    {
      "epoch": 1.6708860759493671,
      "grad_norm": 688317.7143989427,
      "learning_rate": 4.647887323943662e-07,
      "logits/chosen": -13.791796684265137,
      "logits/rejected": -13.970884323120117,
      "logps/chosen": -228.53079223632812,
      "logps/rejected": -235.5008087158203,
      "loss": 123378.175,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.0017698236042633653,
      "rewards/margins": 0.006004182621836662,
      "rewards/rejected": -0.004234359599649906,
      "step": 330
    },
    {
      "epoch": 1.721518987341772,
      "grad_norm": 693314.5034252935,
      "learning_rate": 4.788732394366196e-07,
      "logits/chosen": -13.555567741394043,
      "logits/rejected": -13.32630729675293,
      "logps/chosen": -227.0249481201172,
      "logps/rejected": -232.2772216796875,
      "loss": 122521.475,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.001143553527072072,
      "rewards/margins": 0.009070896543562412,
      "rewards/rejected": -0.00792734231799841,
      "step": 340
    },
    {
      "epoch": 1.7721518987341773,
      "grad_norm": 758709.6120906892,
      "learning_rate": 4.929577464788733e-07,
      "logits/chosen": -13.520563125610352,
      "logits/rejected": -13.633130073547363,
      "logps/chosen": -234.7182159423828,
      "logps/rejected": -248.12890625,
      "loss": 121557.575,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.00047356385039165616,
      "rewards/margins": 0.00813873577862978,
      "rewards/rejected": -0.008612299337983131,
      "step": 350
    },
    {
      "epoch": 1.8227848101265822,
      "grad_norm": 689974.393201542,
      "learning_rate": 4.992165465371357e-07,
      "logits/chosen": -12.841153144836426,
      "logits/rejected": -12.86094856262207,
      "logps/chosen": -232.314697265625,
      "logps/rejected": -232.64297485351562,
      "loss": 121436.65,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.0036194869317114353,
      "rewards/margins": 0.009506477043032646,
      "rewards/rejected": -0.005886988714337349,
      "step": 360
    },
    {
      "epoch": 1.8734177215189873,
      "grad_norm": 883375.543329047,
      "learning_rate": 4.976496396114071e-07,
      "logits/chosen": -12.77904224395752,
      "logits/rejected": -12.76900577545166,
      "logps/chosen": -239.8730010986328,
      "logps/rejected": -251.4569549560547,
      "loss": 122456.925,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0006393647054210305,
      "rewards/margins": 0.008665768429636955,
      "rewards/rejected": -0.009305133484303951,
      "step": 370
    },
    {
      "epoch": 1.9240506329113924,
      "grad_norm": 797554.0864386982,
      "learning_rate": 4.960827326856785e-07,
      "logits/chosen": -13.028135299682617,
      "logits/rejected": -13.148831367492676,
      "logps/chosen": -237.040771484375,
      "logps/rejected": -244.45181274414062,
      "loss": 124907.725,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.0025544934906065464,
      "rewards/margins": 0.008132859133183956,
      "rewards/rejected": -0.005578366108238697,
      "step": 380
    },
    {
      "epoch": 1.9746835443037973,
      "grad_norm": 793120.1180084129,
      "learning_rate": 4.945158257599498e-07,
      "logits/chosen": -12.312803268432617,
      "logits/rejected": -12.135167121887207,
      "logps/chosen": -235.60360717773438,
      "logps/rejected": -242.9219207763672,
      "loss": 121583.8,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.003660207614302635,
      "rewards/margins": 0.011001082137227058,
      "rewards/rejected": -0.007340874522924423,
      "step": 390
    },
    {
      "epoch": 2.0253164556962027,
      "grad_norm": 767339.6192091529,
      "learning_rate": 4.929489188342212e-07,
      "logits/chosen": -12.052891731262207,
      "logits/rejected": -11.94625473022461,
      "logps/chosen": -225.0377197265625,
      "logps/rejected": -243.81039428710938,
      "loss": 119737.85,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.006956162396818399,
      "rewards/margins": 0.01727995090186596,
      "rewards/rejected": -0.010323788039386272,
      "step": 400
    },
    {
      "epoch": 2.0759493670886076,
      "grad_norm": 936793.207320047,
      "learning_rate": 4.913820119084926e-07,
      "logits/chosen": -11.38767147064209,
      "logits/rejected": -11.339715957641602,
      "logps/chosen": -219.8796844482422,
      "logps/rejected": -252.80581665039062,
      "loss": 114021.05,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.008199459873139858,
      "rewards/margins": 0.031510110944509506,
      "rewards/rejected": -0.023310650140047073,
      "step": 410
    },
    {
      "epoch": 2.1265822784810124,
      "grad_norm": 1035986.8564166825,
      "learning_rate": 4.89815104982764e-07,
      "logits/chosen": -10.819408416748047,
      "logits/rejected": -10.774351119995117,
      "logps/chosen": -231.78854370117188,
      "logps/rejected": -260.20355224609375,
      "loss": 116051.6,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0056950985454022884,
      "rewards/margins": 0.027868490666151047,
      "rewards/rejected": -0.02217339165508747,
      "step": 420
    },
    {
      "epoch": 2.1772151898734178,
      "grad_norm": 1036991.7861177241,
      "learning_rate": 4.882481980570354e-07,
      "logits/chosen": -10.84526252746582,
      "logits/rejected": -10.708145141601562,
      "logps/chosen": -221.5430908203125,
      "logps/rejected": -257.36114501953125,
      "loss": 113501.175,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.005057200789451599,
      "rewards/margins": 0.038923002779483795,
      "rewards/rejected": -0.033865805715322495,
      "step": 430
    },
    {
      "epoch": 2.2278481012658227,
      "grad_norm": 1227488.243303788,
      "learning_rate": 4.866812911313068e-07,
      "logits/chosen": -10.5010986328125,
      "logits/rejected": -10.63232135772705,
      "logps/chosen": -233.42373657226562,
      "logps/rejected": -276.0982666015625,
      "loss": 112100.4,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.004779786802828312,
      "rewards/margins": 0.040522992610931396,
      "rewards/rejected": -0.03574320673942566,
      "step": 440
    },
    {
      "epoch": 2.278481012658228,
      "grad_norm": 1079397.6974786038,
      "learning_rate": 4.851143842055782e-07,
      "logits/chosen": -10.104026794433594,
      "logits/rejected": -10.142271995544434,
      "logps/chosen": -216.66940307617188,
      "logps/rejected": -258.98858642578125,
      "loss": 112483.4,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.0053299954161047935,
      "rewards/margins": 0.03484385460615158,
      "rewards/rejected": -0.029513856396079063,
      "step": 450
    },
    {
      "epoch": 2.329113924050633,
      "grad_norm": 1367054.8438774655,
      "learning_rate": 4.835474772798496e-07,
      "logits/chosen": -10.148681640625,
      "logits/rejected": -10.183786392211914,
      "logps/chosen": -233.730224609375,
      "logps/rejected": -278.64349365234375,
      "loss": 111561.6625,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.011530257761478424,
      "rewards/margins": 0.04578756168484688,
      "rewards/rejected": -0.034257303923368454,
      "step": 460
    },
    {
      "epoch": 2.379746835443038,
      "grad_norm": 1298484.9349088285,
      "learning_rate": 4.819805703541209e-07,
      "logits/chosen": -10.018949508666992,
      "logits/rejected": -10.097805976867676,
      "logps/chosen": -224.6026153564453,
      "logps/rejected": -270.0591735839844,
      "loss": 112710.1875,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.011801879853010178,
      "rewards/margins": 0.040784891694784164,
      "rewards/rejected": -0.028983011841773987,
      "step": 470
    },
    {
      "epoch": 2.430379746835443,
      "grad_norm": 1428524.6930006845,
      "learning_rate": 4.804136634283923e-07,
      "logits/chosen": -9.595979690551758,
      "logits/rejected": -9.634994506835938,
      "logps/chosen": -265.3009338378906,
      "logps/rejected": -315.98541259765625,
      "loss": 110031.3,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.0027348275762051344,
      "rewards/margins": 0.05151837319135666,
      "rewards/rejected": -0.048783544450998306,
      "step": 480
    },
    {
      "epoch": 2.481012658227848,
      "grad_norm": 1467649.8441612076,
      "learning_rate": 4.788467565026637e-07,
      "logits/chosen": -8.871723175048828,
      "logits/rejected": -8.764354705810547,
      "logps/chosen": -203.2312774658203,
      "logps/rejected": -241.612548828125,
      "loss": 110534.325,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.01302252896130085,
      "rewards/margins": 0.03907207027077675,
      "rewards/rejected": -0.02604953944683075,
      "step": 490
    },
    {
      "epoch": 2.5316455696202533,
      "grad_norm": 1382959.9591988046,
      "learning_rate": 4.772798495769351e-07,
      "logits/chosen": -8.468270301818848,
      "logits/rejected": -8.384966850280762,
      "logps/chosen": -226.46237182617188,
      "logps/rejected": -269.6461181640625,
      "loss": 110480.175,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.015191495418548584,
      "rewards/margins": 0.0456535741686821,
      "rewards/rejected": -0.030462080612778664,
      "step": 500
    },
    {
      "epoch": 2.5822784810126582,
      "grad_norm": 1369494.2190603705,
      "learning_rate": 4.757129426512065e-07,
      "logits/chosen": -8.634099006652832,
      "logits/rejected": -8.640868186950684,
      "logps/chosen": -232.20022583007812,
      "logps/rejected": -304.80352783203125,
      "loss": 109921.975,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.013228721916675568,
      "rewards/margins": 0.07378505170345306,
      "rewards/rejected": -0.060556329786777496,
      "step": 510
    },
    {
      "epoch": 2.632911392405063,
      "grad_norm": 1750255.0550240122,
      "learning_rate": 4.741460357254779e-07,
      "logits/chosen": -7.8379316329956055,
      "logits/rejected": -7.4784440994262695,
      "logps/chosen": -213.3401641845703,
      "logps/rejected": -258.43743896484375,
      "loss": 111730.3875,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.016342563554644585,
      "rewards/margins": 0.048144370317459106,
      "rewards/rejected": -0.03180180490016937,
      "step": 520
    },
    {
      "epoch": 2.6835443037974684,
      "grad_norm": 1447093.2174814222,
      "learning_rate": 4.7257912879974927e-07,
      "logits/chosen": -8.354089736938477,
      "logits/rejected": -7.889782905578613,
      "logps/chosen": -225.5243682861328,
      "logps/rejected": -276.7877502441406,
      "loss": 109226.9625,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.013667022809386253,
      "rewards/margins": 0.05627403408288956,
      "rewards/rejected": -0.042607005685567856,
      "step": 530
    },
    {
      "epoch": 2.7341772151898733,
      "grad_norm": 1477083.7533012358,
      "learning_rate": 4.710122218740207e-07,
      "logits/chosen": -7.921019077301025,
      "logits/rejected": -7.979846000671387,
      "logps/chosen": -237.23715209960938,
      "logps/rejected": -285.4289855957031,
      "loss": 109592.125,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.014952963218092918,
      "rewards/margins": 0.055934417992830276,
      "rewards/rejected": -0.040981464087963104,
      "step": 540
    },
    {
      "epoch": 2.7848101265822782,
      "grad_norm": 1486366.6324330876,
      "learning_rate": 4.6944531494829204e-07,
      "logits/chosen": -7.12634801864624,
      "logits/rejected": -7.396058082580566,
      "logps/chosen": -226.1304168701172,
      "logps/rejected": -276.8672790527344,
      "loss": 108245.925,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.00994019117206335,
      "rewards/margins": 0.0537477545440197,
      "rewards/rejected": -0.04380756989121437,
      "step": 550
    },
    {
      "epoch": 2.8354430379746836,
      "grad_norm": 1560304.698196799,
      "learning_rate": 4.6787840802256345e-07,
      "logits/chosen": -7.268878936767578,
      "logits/rejected": -7.414219856262207,
      "logps/chosen": -215.24661254882812,
      "logps/rejected": -276.79437255859375,
      "loss": 110187.5125,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.016926631331443787,
      "rewards/margins": 0.05572710186243057,
      "rewards/rejected": -0.03880046680569649,
      "step": 560
    },
    {
      "epoch": 2.8860759493670884,
      "grad_norm": 1647695.8714812996,
      "learning_rate": 4.663115010968348e-07,
      "logits/chosen": -8.584083557128906,
      "logits/rejected": -8.43793773651123,
      "logps/chosen": -239.3496856689453,
      "logps/rejected": -301.948974609375,
      "loss": 108493.15,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.007640582975000143,
      "rewards/margins": 0.06335236132144928,
      "rewards/rejected": -0.0557117760181427,
      "step": 570
    },
    {
      "epoch": 2.9367088607594938,
      "grad_norm": 1523200.3846012072,
      "learning_rate": 4.647445941711062e-07,
      "logits/chosen": -8.875934600830078,
      "logits/rejected": -8.860316276550293,
      "logps/chosen": -234.2982635498047,
      "logps/rejected": -293.39727783203125,
      "loss": 107204.65,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.0077833631075918674,
      "rewards/margins": 0.061719853430986404,
      "rewards/rejected": -0.05393648147583008,
      "step": 580
    },
    {
      "epoch": 2.9873417721518987,
      "grad_norm": 1605115.356703113,
      "learning_rate": 4.631776872453776e-07,
      "logits/chosen": -8.788633346557617,
      "logits/rejected": -8.637460708618164,
      "logps/chosen": -257.7025146484375,
      "logps/rejected": -303.82147216796875,
      "loss": 108959.225,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.0057020229287445545,
      "rewards/margins": 0.053022872656583786,
      "rewards/rejected": -0.04732084274291992,
      "step": 590
    },
    {
      "epoch": 3.037974683544304,
      "grad_norm": 1435515.2852262415,
      "learning_rate": 4.61610780319649e-07,
      "logits/chosen": -7.956998348236084,
      "logits/rejected": -7.496169090270996,
      "logps/chosen": -219.92410278320312,
      "logps/rejected": -310.20123291015625,
      "loss": 95986.4875,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.016220271587371826,
      "rewards/margins": 0.09167212247848511,
      "rewards/rejected": -0.07545184344053268,
      "step": 600
    },
    {
      "epoch": 3.088607594936709,
      "grad_norm": 1646011.901841717,
      "learning_rate": 4.6004387339392035e-07,
      "logits/chosen": -7.747580051422119,
      "logits/rejected": -7.5227952003479,
      "logps/chosen": -217.8295440673828,
      "logps/rejected": -343.4312438964844,
      "loss": 91538.925,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.02667585015296936,
      "rewards/margins": 0.12547221779823303,
      "rewards/rejected": -0.09879636764526367,
      "step": 610
    },
    {
      "epoch": 3.1392405063291138,
      "grad_norm": 1631989.4144731541,
      "learning_rate": 4.5847696646819176e-07,
      "logits/chosen": -6.8127121925354,
      "logits/rejected": -6.8090972900390625,
      "logps/chosen": -209.46859741210938,
      "logps/rejected": -332.0594482421875,
      "loss": 92242.9,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.026208167895674706,
      "rewards/margins": 0.12268342822790146,
      "rewards/rejected": -0.0964752584695816,
      "step": 620
    },
    {
      "epoch": 3.189873417721519,
      "grad_norm": 1627589.9925143481,
      "learning_rate": 4.569100595424631e-07,
      "logits/chosen": -6.631221771240234,
      "logits/rejected": -6.502354621887207,
      "logps/chosen": -211.57974243164062,
      "logps/rejected": -333.447265625,
      "loss": 89921.25,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.022689208388328552,
      "rewards/margins": 0.12395058572292328,
      "rewards/rejected": -0.10126137733459473,
      "step": 630
    },
    {
      "epoch": 3.240506329113924,
      "grad_norm": 1780107.5787213328,
      "learning_rate": 4.5534315261673453e-07,
      "logits/chosen": -7.868208885192871,
      "logits/rejected": -7.755393981933594,
      "logps/chosen": -209.3970184326172,
      "logps/rejected": -341.9508056640625,
      "loss": 89608.1875,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.027028566226363182,
      "rewards/margins": 0.133165642619133,
      "rewards/rejected": -0.10613708198070526,
      "step": 640
    },
    {
      "epoch": 3.291139240506329,
      "grad_norm": 1730512.4518714033,
      "learning_rate": 4.5377624569100595e-07,
      "logits/chosen": -7.359053134918213,
      "logits/rejected": -7.324367523193359,
      "logps/chosen": -193.1954803466797,
      "logps/rejected": -309.5513610839844,
      "loss": 93257.225,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.028996175155043602,
      "rewards/margins": 0.11760006099939346,
      "rewards/rejected": -0.08860386908054352,
      "step": 650
    },
    {
      "epoch": 3.3417721518987342,
      "grad_norm": 1692816.769511115,
      "learning_rate": 4.5220933876527736e-07,
      "logits/chosen": -8.043203353881836,
      "logits/rejected": -8.003018379211426,
      "logps/chosen": -211.73648071289062,
      "logps/rejected": -336.10455322265625,
      "loss": 88400.4688,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.024640550836920738,
      "rewards/margins": 0.12655004858970642,
      "rewards/rejected": -0.10190950334072113,
      "step": 660
    },
    {
      "epoch": 3.392405063291139,
      "grad_norm": 1906377.7496358757,
      "learning_rate": 4.506424318395487e-07,
      "logits/chosen": -7.25619649887085,
      "logits/rejected": -7.37869119644165,
      "logps/chosen": -197.8258819580078,
      "logps/rejected": -324.2138671875,
      "loss": 89983.5688,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.026263948529958725,
      "rewards/margins": 0.12702925503253937,
      "rewards/rejected": -0.10076530277729034,
      "step": 670
    },
    {
      "epoch": 3.4430379746835444,
      "grad_norm": 1785643.0594316572,
      "learning_rate": 4.4907552491382013e-07,
      "logits/chosen": -6.798577785491943,
      "logits/rejected": -6.7768073081970215,
      "logps/chosen": -208.5835723876953,
      "logps/rejected": -323.3017883300781,
      "loss": 89767.5,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.025741413235664368,
      "rewards/margins": 0.1167701929807663,
      "rewards/rejected": -0.09102877229452133,
      "step": 680
    },
    {
      "epoch": 3.4936708860759493,
      "grad_norm": 2393957.296937455,
      "learning_rate": 4.475086179880915e-07,
      "logits/chosen": -6.352355480194092,
      "logits/rejected": -6.526197910308838,
      "logps/chosen": -187.56597900390625,
      "logps/rejected": -306.5972595214844,
      "loss": 89036.6875,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.024061182513833046,
      "rewards/margins": 0.11990946531295776,
      "rewards/rejected": -0.09584827721118927,
      "step": 690
    },
    {
      "epoch": 3.5443037974683547,
      "grad_norm": 1811486.2204670438,
      "learning_rate": 4.459417110623629e-07,
      "logits/chosen": -5.7466630935668945,
      "logits/rejected": -5.797163486480713,
      "logps/chosen": -212.6585235595703,
      "logps/rejected": -364.36199951171875,
      "loss": 88031.3,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.027677077800035477,
      "rewards/margins": 0.14764061570167542,
      "rewards/rejected": -0.11996352672576904,
      "step": 700
    },
    {
      "epoch": 3.5949367088607596,
      "grad_norm": 1724684.5755440604,
      "learning_rate": 4.4437480413663426e-07,
      "logits/chosen": -5.412962436676025,
      "logits/rejected": -5.541121959686279,
      "logps/chosen": -202.39065551757812,
      "logps/rejected": -333.0758056640625,
      "loss": 86956.675,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.0247800350189209,
      "rewards/margins": 0.12825721502304077,
      "rewards/rejected": -0.10347716510295868,
      "step": 710
    },
    {
      "epoch": 3.6455696202531644,
      "grad_norm": 1933271.7611355048,
      "learning_rate": 4.4280789721090567e-07,
      "logits/chosen": -5.053005218505859,
      "logits/rejected": -4.886711597442627,
      "logps/chosen": -199.10885620117188,
      "logps/rejected": -317.7257385253906,
      "loss": 86655.0125,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.02152046002447605,
      "rewards/margins": 0.11774978786706924,
      "rewards/rejected": -0.09622932970523834,
      "step": 720
    },
    {
      "epoch": 3.6962025316455698,
      "grad_norm": 2267463.489494214,
      "learning_rate": 4.4124099028517703e-07,
      "logits/chosen": -6.616279602050781,
      "logits/rejected": -6.9615797996521,
      "logps/chosen": -200.58961486816406,
      "logps/rejected": -351.6376953125,
      "loss": 86181.3938,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.032253801822662354,
      "rewards/margins": 0.14937567710876465,
      "rewards/rejected": -0.1171218603849411,
      "step": 730
    },
    {
      "epoch": 3.7468354430379747,
      "grad_norm": 1734288.0953653858,
      "learning_rate": 4.3967408335944844e-07,
      "logits/chosen": -5.873335361480713,
      "logits/rejected": -5.689335823059082,
      "logps/chosen": -217.43637084960938,
      "logps/rejected": -350.2752990722656,
      "loss": 86780.825,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.031159091740846634,
      "rewards/margins": 0.13692796230316162,
      "rewards/rejected": -0.10576887428760529,
      "step": 740
    },
    {
      "epoch": 3.7974683544303796,
      "grad_norm": 1741715.9901586007,
      "learning_rate": 4.381071764337198e-07,
      "logits/chosen": -7.123785972595215,
      "logits/rejected": -7.188807487487793,
      "logps/chosen": -207.00045776367188,
      "logps/rejected": -336.5976867675781,
      "loss": 86139.5625,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.03052128478884697,
      "rewards/margins": 0.13043463230133057,
      "rewards/rejected": -0.0999133437871933,
      "step": 750
    },
    {
      "epoch": 3.848101265822785,
      "grad_norm": 1879351.8394690978,
      "learning_rate": 4.365402695079912e-07,
      "logits/chosen": -7.820990085601807,
      "logits/rejected": -7.7128729820251465,
      "logps/chosen": -213.57388305664062,
      "logps/rejected": -362.5634460449219,
      "loss": 87478.3625,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 0.03660900145769119,
      "rewards/margins": 0.1480773240327835,
      "rewards/rejected": -0.11146833002567291,
      "step": 760
    },
    {
      "epoch": 3.8987341772151898,
      "grad_norm": 1968713.4204386624,
      "learning_rate": 4.349733625822626e-07,
      "logits/chosen": -7.314540863037109,
      "logits/rejected": -7.363668918609619,
      "logps/chosen": -213.6930694580078,
      "logps/rejected": -367.44073486328125,
      "loss": 86825.5813,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.026752913370728493,
      "rewards/margins": 0.15061405301094055,
      "rewards/rejected": -0.1238611489534378,
      "step": 770
    },
    {
      "epoch": 3.9493670886075947,
      "grad_norm": 2163439.406665409,
      "learning_rate": 4.33406455656534e-07,
      "logits/chosen": -7.67099666595459,
      "logits/rejected": -7.536408424377441,
      "logps/chosen": -213.9747772216797,
      "logps/rejected": -344.7560119628906,
      "loss": 86913.0375,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.029844319447875023,
      "rewards/margins": 0.12930825352668762,
      "rewards/rejected": -0.09946390986442566,
      "step": 780
    },
    {
      "epoch": 4.0,
      "grad_norm": 1866234.1823014135,
      "learning_rate": 4.3183954873080535e-07,
      "logits/chosen": -7.922532081604004,
      "logits/rejected": -7.692726135253906,
      "logps/chosen": -211.41653442382812,
      "logps/rejected": -349.7116394042969,
      "loss": 86592.8938,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.027728911489248276,
      "rewards/margins": 0.1435452550649643,
      "rewards/rejected": -0.11581633985042572,
      "step": 790
    },
    {
      "epoch": 4.050632911392405,
      "grad_norm": 1782853.8797277175,
      "learning_rate": 4.3027264180507676e-07,
      "logits/chosen": -8.29829216003418,
      "logits/rejected": -8.205643653869629,
      "logps/chosen": -178.8797149658203,
      "logps/rejected": -378.06121826171875,
      "loss": 69143.425,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.05098045617341995,
      "rewards/margins": 0.1993386447429657,
      "rewards/rejected": -0.14835818111896515,
      "step": 800
    },
    {
      "epoch": 4.10126582278481,
      "grad_norm": 1719472.9461235409,
      "learning_rate": 4.287057348793481e-07,
      "logits/chosen": -7.558290958404541,
      "logits/rejected": -7.646592617034912,
      "logps/chosen": -186.36911010742188,
      "logps/rejected": -386.6961975097656,
      "loss": 67634.3375,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.04189852252602577,
      "rewards/margins": 0.19968575239181519,
      "rewards/rejected": -0.1577872335910797,
      "step": 810
    },
    {
      "epoch": 4.151898734177215,
      "grad_norm": 1571399.8942716653,
      "learning_rate": 4.2713882795361953e-07,
      "logits/chosen": -7.811161994934082,
      "logits/rejected": -7.783130645751953,
      "logps/chosen": -181.81602478027344,
      "logps/rejected": -402.1683654785156,
      "loss": 66806.9187,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.049001529812812805,
      "rewards/margins": 0.21849961578845978,
      "rewards/rejected": -0.16949808597564697,
      "step": 820
    },
    {
      "epoch": 4.2025316455696204,
      "grad_norm": 1992030.3917670588,
      "learning_rate": 4.255719210278909e-07,
      "logits/chosen": -7.349759101867676,
      "logits/rejected": -7.380797386169434,
      "logps/chosen": -175.21702575683594,
      "logps/rejected": -396.2167053222656,
      "loss": 67021.875,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.05283821374177933,
      "rewards/margins": 0.22190704941749573,
      "rewards/rejected": -0.169068843126297,
      "step": 830
    },
    {
      "epoch": 4.253164556962025,
      "grad_norm": 1859879.670487208,
      "learning_rate": 4.2400501410216235e-07,
      "logits/chosen": -7.482248783111572,
      "logits/rejected": -7.252910614013672,
      "logps/chosen": -187.070556640625,
      "logps/rejected": -401.1556701660156,
      "loss": 68463.9,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 0.05697192624211311,
      "rewards/margins": 0.21645841002464294,
      "rewards/rejected": -0.15948647260665894,
      "step": 840
    },
    {
      "epoch": 4.30379746835443,
      "grad_norm": 1688181.1410657803,
      "learning_rate": 4.224381071764337e-07,
      "logits/chosen": -5.693742275238037,
      "logits/rejected": -5.435591697692871,
      "logps/chosen": -198.21900939941406,
      "logps/rejected": -398.49981689453125,
      "loss": 67266.2,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.04546400159597397,
      "rewards/margins": 0.20465342700481415,
      "rewards/rejected": -0.15918943285942078,
      "step": 850
    },
    {
      "epoch": 4.3544303797468356,
      "grad_norm": 1750431.6432656392,
      "learning_rate": 4.208712002507051e-07,
      "logits/chosen": -8.664016723632812,
      "logits/rejected": -8.082508087158203,
      "logps/chosen": -178.05966186523438,
      "logps/rejected": -402.77093505859375,
      "loss": 65760.2625,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.056066203862428665,
      "rewards/margins": 0.22950176894664764,
      "rewards/rejected": -0.17343556880950928,
      "step": 860
    },
    {
      "epoch": 4.405063291139241,
      "grad_norm": 1904336.610304837,
      "learning_rate": 4.193042933249765e-07,
      "logits/chosen": -5.778517723083496,
      "logits/rejected": -5.432709693908691,
      "logps/chosen": -176.563720703125,
      "logps/rejected": -379.2276916503906,
      "loss": 67058.1125,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.05091014504432678,
      "rewards/margins": 0.2058809995651245,
      "rewards/rejected": -0.15497085452079773,
      "step": 870
    },
    {
      "epoch": 4.455696202531645,
      "grad_norm": 1779397.1811982268,
      "learning_rate": 4.177373863992479e-07,
      "logits/chosen": -6.937778472900391,
      "logits/rejected": -6.611588954925537,
      "logps/chosen": -180.23001098632812,
      "logps/rejected": -400.9800720214844,
      "loss": 67019.0875,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 0.05085798352956772,
      "rewards/margins": 0.2235671728849411,
      "rewards/rejected": -0.17270918190479279,
      "step": 880
    },
    {
      "epoch": 4.506329113924051,
      "grad_norm": 1755630.994265544,
      "learning_rate": 4.1617047947351925e-07,
      "logits/chosen": -6.663479804992676,
      "logits/rejected": -6.144991397857666,
      "logps/chosen": -189.93707275390625,
      "logps/rejected": -383.9622802734375,
      "loss": 66060.8813,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 0.053109876811504364,
      "rewards/margins": 0.20497091114521027,
      "rewards/rejected": -0.1518610268831253,
      "step": 890
    },
    {
      "epoch": 4.556962025316456,
      "grad_norm": 1729683.010514938,
      "learning_rate": 4.1460357254779067e-07,
      "logits/chosen": -7.10635232925415,
      "logits/rejected": -7.227837562561035,
      "logps/chosen": -184.3021240234375,
      "logps/rejected": -391.59930419921875,
      "loss": 67231.6313,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.050502438098192215,
      "rewards/margins": 0.20674797892570496,
      "rewards/rejected": -0.15624557435512543,
      "step": 900
    },
    {
      "epoch": 4.6075949367088604,
      "grad_norm": 1921064.671845176,
      "learning_rate": 4.13036665622062e-07,
      "logits/chosen": -7.409733772277832,
      "logits/rejected": -7.2668256759643555,
      "logps/chosen": -184.89645385742188,
      "logps/rejected": -395.2364501953125,
      "loss": 67370.1875,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.047733135521411896,
      "rewards/margins": 0.2108074128627777,
      "rewards/rejected": -0.1630742847919464,
      "step": 910
    },
    {
      "epoch": 4.658227848101266,
      "grad_norm": 1780170.6356310213,
      "learning_rate": 4.1146975869633344e-07,
      "logits/chosen": -8.294339179992676,
      "logits/rejected": -8.312765121459961,
      "logps/chosen": -185.74949645996094,
      "logps/rejected": -405.0606689453125,
      "loss": 64484.2438,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.05801473185420036,
      "rewards/margins": 0.21365991234779358,
      "rewards/rejected": -0.15564517676830292,
      "step": 920
    },
    {
      "epoch": 4.708860759493671,
      "grad_norm": 1755118.627079852,
      "learning_rate": 4.099028517706048e-07,
      "logits/chosen": -8.692441940307617,
      "logits/rejected": -8.729148864746094,
      "logps/chosen": -177.8703155517578,
      "logps/rejected": -410.15179443359375,
      "loss": 65960.6812,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.061922211199998856,
      "rewards/margins": 0.2333444058895111,
      "rewards/rejected": -0.17142215371131897,
      "step": 930
    },
    {
      "epoch": 4.759493670886076,
      "grad_norm": 1801666.0452341542,
      "learning_rate": 4.083359448448762e-07,
      "logits/chosen": -8.838138580322266,
      "logits/rejected": -8.679426193237305,
      "logps/chosen": -160.35488891601562,
      "logps/rejected": -387.3427429199219,
      "loss": 65957.3,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.061734091490507126,
      "rewards/margins": 0.2303626835346222,
      "rewards/rejected": -0.16862855851650238,
      "step": 940
    },
    {
      "epoch": 4.810126582278481,
      "grad_norm": 1823914.1164093877,
      "learning_rate": 4.0676903791914757e-07,
      "logits/chosen": -8.039133071899414,
      "logits/rejected": -8.235550880432129,
      "logps/chosen": -181.90818786621094,
      "logps/rejected": -390.46075439453125,
      "loss": 65100.0437,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.05453425645828247,
      "rewards/margins": 0.20622405409812927,
      "rewards/rejected": -0.1516897976398468,
      "step": 950
    },
    {
      "epoch": 4.860759493670886,
      "grad_norm": 2552504.752187401,
      "learning_rate": 4.05202130993419e-07,
      "logits/chosen": -8.228861808776855,
      "logits/rejected": -8.044200897216797,
      "logps/chosen": -175.62306213378906,
      "logps/rejected": -387.7801818847656,
      "loss": 65251.5563,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.05643890053033829,
      "rewards/margins": 0.2162017822265625,
      "rewards/rejected": -0.15976287424564362,
      "step": 960
    },
    {
      "epoch": 4.911392405063291,
      "grad_norm": 2112562.829549655,
      "learning_rate": 4.0363522406769034e-07,
      "logits/chosen": -8.678482055664062,
      "logits/rejected": -8.680012702941895,
      "logps/chosen": -180.9581298828125,
      "logps/rejected": -402.48944091796875,
      "loss": 65731.7188,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 0.05988938361406326,
      "rewards/margins": 0.22270476818084717,
      "rewards/rejected": -0.1628153920173645,
      "step": 970
    },
    {
      "epoch": 4.962025316455696,
      "grad_norm": 1800725.2761679955,
      "learning_rate": 4.0206831714196175e-07,
      "logits/chosen": -9.068916320800781,
      "logits/rejected": -8.908533096313477,
      "logps/chosen": -191.30018615722656,
      "logps/rejected": -433.2850036621094,
      "loss": 64987.5125,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 0.0664498582482338,
      "rewards/margins": 0.24509286880493164,
      "rewards/rejected": -0.17864301800727844,
      "step": 980
    }
  ],
  "logging_steps": 10,
  "max_steps": 3546,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 18,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}