{
  "best_metric": 0.5813568234443665,
  "best_model_checkpoint": "models/llama-3.2-3b-dpo-2/checkpoint-1500",
  "epoch": 2.998693948628646,
  "eval_steps": 100,
  "global_step": 1722,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0017414018284719198,
      "grad_norm": 16.910327090065376,
      "learning_rate": 9.615384615384615e-09,
      "logits/chosen": 0.5875190496444702,
      "logits/rejected": 0.8128950595855713,
      "logps/chosen": -403.25274658203125,
      "logps/rejected": -280.435302734375,
      "loss": 1.0,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.0174140182847192,
      "grad_norm": 19.954384726122814,
      "learning_rate": 9.615384615384616e-08,
      "logits/chosen": 0.5852577686309814,
      "logits/rejected": 0.6753351092338562,
      "logps/chosen": -359.52099609375,
      "logps/rejected": -325.59088134765625,
      "loss": 1.0009,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.00044315060949884355,
      "rewards/margins": -0.0015449941856786609,
      "rewards/rejected": 0.001101843430660665,
      "step": 10
    },
    {
      "epoch": 0.0348280365694384,
      "grad_norm": 18.013297709246753,
      "learning_rate": 1.9230769230769231e-07,
      "logits/chosen": 0.5446847677230835,
      "logits/rejected": 0.5242202877998352,
      "logps/chosen": -337.18963623046875,
      "logps/rejected": -339.19342041015625,
      "loss": 0.9995,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.001996212173253298,
      "rewards/margins": -0.0010775469709187746,
      "rewards/rejected": -0.0009186647948808968,
      "step": 20
    },
    {
      "epoch": 0.052242054854157595,
      "grad_norm": 17.103619387941173,
      "learning_rate": 2.884615384615384e-07,
      "logits/chosen": 0.5122469067573547,
      "logits/rejected": 0.6459108591079712,
      "logps/chosen": -354.17877197265625,
      "logps/rejected": -327.595947265625,
      "loss": 0.9962,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.006955576129257679,
      "rewards/margins": 0.014386234804987907,
      "rewards/rejected": -0.007430657744407654,
      "step": 30
    },
    {
      "epoch": 0.0696560731388768,
      "grad_norm": 19.165758688739448,
      "learning_rate": 3.8461538461538463e-07,
      "logits/chosen": 0.5354681015014648,
      "logits/rejected": 0.6567710638046265,
      "logps/chosen": -320.5543212890625,
      "logps/rejected": -303.88934326171875,
      "loss": 0.9841,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.008432401344180107,
      "rewards/margins": 0.04507141932845116,
      "rewards/rejected": -0.0366390161216259,
      "step": 40
    },
    {
      "epoch": 0.087070091423596,
      "grad_norm": 15.004492365630895,
      "learning_rate": 4.807692307692307e-07,
      "logits/chosen": 0.7950223684310913,
      "logits/rejected": 0.7804186940193176,
      "logps/chosen": -311.0839538574219,
      "logps/rejected": -302.98931884765625,
      "loss": 0.9613,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.035525619983673096,
      "rewards/margins": 0.1604817509651184,
      "rewards/rejected": -0.12495611608028412,
      "step": 50
    },
    {
      "epoch": 0.10448410970831519,
      "grad_norm": 14.20109250939024,
      "learning_rate": 4.99971689431323e-07,
      "logits/chosen": 0.6672784686088562,
      "logits/rejected": 0.7780674695968628,
      "logps/chosen": -336.57861328125,
      "logps/rejected": -307.4258728027344,
      "loss": 0.9291,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.24874690175056458,
      "rewards/margins": 0.37877365946769714,
      "rewards/rejected": -0.13002678751945496,
      "step": 60
    },
    {
      "epoch": 0.12189812799303439,
      "grad_norm": 13.148974871311646,
      "learning_rate": 4.998566887351406e-07,
      "logits/chosen": 0.4873111844062805,
      "logits/rejected": 0.6285614967346191,
      "logps/chosen": -360.8282775878906,
      "logps/rejected": -331.92291259765625,
      "loss": 0.878,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.38520747423171997,
      "rewards/margins": 0.48126062750816345,
      "rewards/rejected": -0.09605316817760468,
      "step": 70
    },
    {
      "epoch": 0.1393121462777536,
      "grad_norm": 12.357509982546857,
      "learning_rate": 4.996532691658581e-07,
      "logits/chosen": 0.6317829489707947,
      "logits/rejected": 0.6504614353179932,
      "logps/chosen": -367.9935607910156,
      "logps/rejected": -340.3227844238281,
      "loss": 0.8333,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.40676337480545044,
      "rewards/margins": 1.0554689168930054,
      "rewards/rejected": -0.6487056016921997,
      "step": 80
    },
    {
      "epoch": 0.1567261645624728,
      "grad_norm": 11.26570144090341,
      "learning_rate": 4.993615027092575e-07,
      "logits/chosen": 0.5946422815322876,
      "logits/rejected": 0.6343314051628113,
      "logps/chosen": -284.96160888671875,
      "logps/rejected": -307.95623779296875,
      "loss": 0.7985,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.30818304419517517,
      "rewards/margins": 0.9103001356124878,
      "rewards/rejected": -0.6021171808242798,
      "step": 90
    },
    {
      "epoch": 0.174140182847192,
      "grad_norm": 10.947878449098555,
      "learning_rate": 4.989814926151712e-07,
      "logits/chosen": 0.5656188726425171,
      "logits/rejected": 0.5697994232177734,
      "logps/chosen": -331.9007873535156,
      "logps/rejected": -353.94854736328125,
      "loss": 0.7596,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.4680165648460388,
      "rewards/margins": 1.718173623085022,
      "rewards/rejected": -1.2501569986343384,
      "step": 100
    },
    {
      "epoch": 0.174140182847192,
      "eval_logits/chosen": 0.5481991171836853,
      "eval_logits/rejected": 0.6135243773460388,
      "eval_logps/chosen": -354.6434020996094,
      "eval_logps/rejected": -360.88714599609375,
      "eval_loss": 0.7588136792182922,
      "eval_rewards/accuracies": 0.699367105960846,
      "eval_rewards/chosen": 0.13485638797283173,
      "eval_rewards/margins": 1.574654459953308,
      "eval_rewards/rejected": -1.4397982358932495,
      "eval_runtime": 74.5995,
      "eval_samples_per_second": 33.512,
      "eval_steps_per_second": 1.059,
      "step": 100
    },
    {
      "epoch": 0.19155420113191118,
      "grad_norm": 10.162134567361168,
      "learning_rate": 4.985133733609451e-07,
      "logits/chosen": 0.590889573097229,
      "logits/rejected": 0.6486071944236755,
      "logps/chosen": -326.9929504394531,
      "logps/rejected": -329.12506103515625,
      "loss": 0.7376,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": 0.4178914427757263,
      "rewards/margins": 2.201124668121338,
      "rewards/rejected": -1.7832332849502563,
      "step": 110
    },
    {
      "epoch": 0.20896821941663038,
      "grad_norm": 10.257743402467643,
      "learning_rate": 4.979573106038489e-07,
      "logits/chosen": 0.5782706141471863,
      "logits/rejected": 0.6307928562164307,
      "logps/chosen": -318.9755859375,
      "logps/rejected": -345.50799560546875,
      "loss": 0.7276,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.34923794865608215,
      "rewards/margins": 2.149462938308716,
      "rewards/rejected": -1.800225019454956,
      "step": 120
    },
    {
      "epoch": 0.22638223770134958,
      "grad_norm": 11.74103724012099,
      "learning_rate": 4.973135011224546e-07,
      "logits/chosen": 0.4907318651676178,
      "logits/rejected": 0.5648502707481384,
      "logps/chosen": -326.4444580078125,
      "logps/rejected": -352.13140869140625,
      "loss": 0.7176,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.4452616572380066,
      "rewards/margins": 2.2069687843322754,
      "rewards/rejected": -1.7617069482803345,
      "step": 130
    },
    {
      "epoch": 0.24379625598606877,
      "grad_norm": 10.814311644424995,
      "learning_rate": 4.965821727469999e-07,
      "logits/chosen": 0.5636430382728577,
      "logits/rejected": 0.653995156288147,
      "logps/chosen": -360.30804443359375,
      "logps/rejected": -338.9974365234375,
      "loss": 0.7097,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.4193500876426697,
      "rewards/margins": 2.334038257598877,
      "rewards/rejected": -1.9146884679794312,
      "step": 140
    },
    {
      "epoch": 0.26121027427078797,
      "grad_norm": 9.638215913082206,
      "learning_rate": 4.957635842787648e-07,
      "logits/chosen": 0.4618745744228363,
      "logits/rejected": 0.5876752138137817,
      "logps/chosen": -342.8825988769531,
      "logps/rejected": -373.37945556640625,
      "loss": 0.7149,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.020968124270439148,
      "rewards/margins": 2.3450706005096436,
      "rewards/rejected": -2.3241021633148193,
      "step": 150
    },
    {
      "epoch": 0.2786242925555072,
      "grad_norm": 11.08344519424176,
      "learning_rate": 4.948580253984868e-07,
      "logits/chosen": 0.47446388006210327,
      "logits/rejected": 0.5650290250778198,
      "logps/chosen": -307.9524230957031,
      "logps/rejected": -348.2874450683594,
      "loss": 0.6961,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.5717116594314575,
      "rewards/margins": 3.2698848247528076,
      "rewards/rejected": -2.6981732845306396,
      "step": 160
    },
    {
      "epoch": 0.29603831084022636,
      "grad_norm": 9.33184052198387,
      "learning_rate": 4.938658165638496e-07,
      "logits/chosen": 0.4030691981315613,
      "logits/rejected": 0.5435680150985718,
      "logps/chosen": -353.84063720703125,
      "logps/rejected": -383.1701354980469,
      "loss": 0.6827,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.5485956072807312,
      "rewards/margins": 3.1341984272003174,
      "rewards/rejected": -2.5856029987335205,
      "step": 170
    },
    {
      "epoch": 0.3134523291249456,
      "grad_norm": 9.495740140284331,
      "learning_rate": 4.9278730889608e-07,
      "logits/chosen": 0.5938032269477844,
      "logits/rejected": 0.6139369606971741,
      "logps/chosen": -321.7276916503906,
      "logps/rejected": -362.5476989746094,
      "loss": 0.6581,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": 1.118109941482544,
      "rewards/margins": 3.9192118644714355,
      "rewards/rejected": -2.8011021614074707,
      "step": 180
    },
    {
      "epoch": 0.3308663474096648,
      "grad_norm": 9.373432885518516,
      "learning_rate": 4.916228840556932e-07,
      "logits/chosen": 0.49382227659225464,
      "logits/rejected": 0.6422325968742371,
      "logps/chosen": -365.77471923828125,
      "logps/rejected": -338.041015625,
      "loss": 0.6736,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 0.5264456272125244,
      "rewards/margins": 3.0396347045898438,
      "rewards/rejected": -2.5131890773773193,
      "step": 190
    },
    {
      "epoch": 0.348280365694384,
      "grad_norm": 10.853280262051877,
      "learning_rate": 4.903729541074324e-07,
      "logits/chosen": 0.4458913207054138,
      "logits/rejected": 0.560121476650238,
      "logps/chosen": -360.0127258300781,
      "logps/rejected": -370.9368896484375,
      "loss": 0.6725,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.29008936882019043,
      "rewards/margins": 2.945146083831787,
      "rewards/rejected": -2.655057191848755,
      "step": 200
    },
    {
      "epoch": 0.348280365694384,
      "eval_logits/chosen": 0.47184163331985474,
      "eval_logits/rejected": 0.533531904220581,
      "eval_logps/chosen": -349.7451477050781,
      "eval_logps/rejected": -373.811767578125,
      "eval_loss": 0.6680147051811218,
      "eval_rewards/accuracies": 0.7278481125831604,
      "eval_rewards/chosen": 0.6246804594993591,
      "eval_rewards/margins": 3.356940984725952,
      "eval_rewards/rejected": -2.7322607040405273,
      "eval_runtime": 73.458,
      "eval_samples_per_second": 34.033,
      "eval_steps_per_second": 1.075,
      "step": 200
    },
    {
      "epoch": 0.3656943839791032,
      "grad_norm": 9.744674847212522,
      "learning_rate": 4.890379613744469e-07,
      "logits/chosen": 0.5317627787590027,
      "logits/rejected": 0.6278539896011353,
      "logps/chosen": -336.4283142089844,
      "logps/rejected": -345.31683349609375,
      "loss": 0.6653,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.5727043747901917,
      "rewards/margins": 3.1886203289031982,
      "rewards/rejected": -2.6159162521362305,
      "step": 210
    },
    {
      "epoch": 0.38310840226382237,
      "grad_norm": 9.347496298393207,
      "learning_rate": 4.87618378281764e-07,
      "logits/chosen": 0.4476338326931,
      "logits/rejected": 0.559643566608429,
      "logps/chosen": -353.37103271484375,
      "logps/rejected": -345.65435791015625,
      "loss": 0.6793,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.609833836555481,
      "rewards/margins": 3.4250805377960205,
      "rewards/rejected": -2.81524658203125,
      "step": 220
    },
    {
      "epoch": 0.4005224205485416,
      "grad_norm": 10.64305700872199,
      "learning_rate": 4.861147071891086e-07,
      "logits/chosen": 0.5430992245674133,
      "logits/rejected": 0.5794849395751953,
      "logps/chosen": -322.3576965332031,
      "logps/rejected": -377.4114074707031,
      "loss": 0.6652,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 0.870169997215271,
      "rewards/margins": 4.160640716552734,
      "rewards/rejected": -3.290470838546753,
      "step": 230
    },
    {
      "epoch": 0.41793643883326076,
      "grad_norm": 9.671652973719253,
      "learning_rate": 4.845274802131277e-07,
      "logits/chosen": 0.5680447816848755,
      "logits/rejected": 0.604617178440094,
      "logps/chosen": -293.92315673828125,
      "logps/rejected": -329.440673828125,
      "loss": 0.6565,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 0.4282473623752594,
      "rewards/margins": 3.685936689376831,
      "rewards/rejected": -3.2576892375946045,
      "step": 240
    },
    {
      "epoch": 0.43535045711798,
      "grad_norm": 10.210911077399032,
      "learning_rate": 4.828572590390871e-07,
      "logits/chosen": 0.46727150678634644,
      "logits/rejected": 0.6490423083305359,
      "logps/chosen": -329.9657287597656,
      "logps/rejected": -337.4406433105469,
      "loss": 0.6563,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.10539127886295319,
      "rewards/margins": 2.982442855834961,
      "rewards/rejected": -2.877051591873169,
      "step": 250
    },
    {
      "epoch": 0.45276447540269915,
      "grad_norm": 11.398369421316715,
      "learning_rate": 4.811046347221025e-07,
      "logits/chosen": 0.403046190738678,
      "logits/rejected": 0.6530165672302246,
      "logps/chosen": -334.22760009765625,
      "logps/rejected": -334.1841735839844,
      "loss": 0.6755,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.48005884885787964,
      "rewards/margins": 3.5819506645202637,
      "rewards/rejected": -3.1018917560577393,
      "step": 260
    },
    {
      "epoch": 0.4701784936874184,
      "grad_norm": 10.244143897450304,
      "learning_rate": 4.792702274779775e-07,
      "logits/chosen": 0.5057202577590942,
      "logits/rejected": 0.6287623643875122,
      "logps/chosen": -352.79278564453125,
      "logps/rejected": -397.2536926269531,
      "loss": 0.6706,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.09758170694112778,
      "rewards/margins": 2.9382576942443848,
      "rewards/rejected": -2.8406760692596436,
      "step": 270
    },
    {
      "epoch": 0.48759251197213754,
      "grad_norm": 11.19817676192306,
      "learning_rate": 4.773546864637237e-07,
      "logits/chosen": 0.3693320155143738,
      "logits/rejected": 0.47319895029067993,
      "logps/chosen": -362.92987060546875,
      "logps/rejected": -393.7507019042969,
      "loss": 0.6669,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.7348660826683044,
      "rewards/margins": 3.7446212768554688,
      "rewards/rejected": -3.0097551345825195,
      "step": 280
    },
    {
      "epoch": 0.5050065302568568,
      "grad_norm": 11.48370881876219,
      "learning_rate": 4.7535868954783724e-07,
      "logits/chosen": 0.5243381261825562,
      "logits/rejected": 0.5710750818252563,
      "logps/chosen": -337.2316589355469,
      "logps/rejected": -344.5747375488281,
      "loss": 0.6147,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 1.3425434827804565,
      "rewards/margins": 4.192605495452881,
      "rewards/rejected": -2.8500616550445557,
      "step": 290
    },
    {
      "epoch": 0.5224205485415759,
      "grad_norm": 11.566809258335352,
      "learning_rate": 4.732829430704162e-07,
      "logits/chosen": 0.5479331016540527,
      "logits/rejected": 0.6467794179916382,
      "logps/chosen": -357.7979736328125,
      "logps/rejected": -371.5728759765625,
      "loss": 0.6452,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 1.06717848777771,
      "rewards/margins": 4.3429388999938965,
      "rewards/rejected": -3.2757606506347656,
      "step": 300
    },
    {
      "epoch": 0.5224205485415759,
      "eval_logits/chosen": 0.48656851053237915,
      "eval_logits/rejected": 0.5476837158203125,
      "eval_logps/chosen": -354.2216491699219,
      "eval_logps/rejected": -384.525634765625,
      "eval_loss": 0.6514016389846802,
      "eval_rewards/accuracies": 0.75,
      "eval_rewards/chosen": 0.1770300567150116,
      "eval_rewards/margins": 3.98067569732666,
      "eval_rewards/rejected": -3.803645610809326,
      "eval_runtime": 70.9012,
      "eval_samples_per_second": 35.26,
      "eval_steps_per_second": 1.114,
      "step": 300
    },
    {
      "epoch": 0.5398345668262952,
      "grad_norm": 10.500375745607627,
      "learning_rate": 4.7112818159320187e-07,
      "logits/chosen": 0.5276615023612976,
      "logits/rejected": 0.5516486167907715,
      "logps/chosen": -326.74200439453125,
      "logps/rejected": -359.82073974609375,
      "loss": 0.6401,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": 1.0380756855010986,
      "rewards/margins": 4.506098747253418,
      "rewards/rejected": -3.4680233001708984,
      "step": 310
    },
    {
      "epoch": 0.5572485851110144,
      "grad_norm": 10.088286466293443,
      "learning_rate": 4.6889516763963356e-07,
      "logits/chosen": 0.5277293920516968,
      "logits/rejected": 0.5241761207580566,
      "logps/chosen": -315.60980224609375,
      "logps/rejected": -370.9276428222656,
      "loss": 0.6594,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 1.2694261074066162,
      "rewards/margins": 4.256104469299316,
      "rewards/rejected": -2.9866783618927,
      "step": 320
    },
    {
      "epoch": 0.5746626033957336,
      "grad_norm": 9.154194830865354,
      "learning_rate": 4.665846914250079e-07,
      "logits/chosen": 0.36231085658073425,
      "logits/rejected": 0.5745257139205933,
      "logps/chosen": -350.9997863769531,
      "logps/rejected": -364.403564453125,
      "loss": 0.6387,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 0.9238360524177551,
      "rewards/margins": 3.7312512397766113,
      "rewards/rejected": -2.807415008544922,
      "step": 330
    },
    {
      "epoch": 0.5920766216804527,
      "grad_norm": 10.584885840971705,
      "learning_rate": 4.641975705768386e-07,
      "logits/chosen": 0.521003782749176,
      "logits/rejected": 0.566765546798706,
      "logps/chosen": -316.1827392578125,
      "logps/rejected": -351.51727294921875,
      "loss": 0.6389,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.7287189364433289,
      "rewards/margins": 3.9497039318084717,
      "rewards/rejected": -3.2209854125976562,
      "step": 340
    },
    {
      "epoch": 0.6094906399651719,
      "grad_norm": 12.315232300296437,
      "learning_rate": 4.6173464984551615e-07,
      "logits/chosen": 0.4950438439846039,
      "logits/rejected": 0.5969202518463135,
      "logps/chosen": -326.7777099609375,
      "logps/rejected": -357.15496826171875,
      "loss": 0.6432,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.4777583181858063,
      "rewards/margins": 4.0313310623168945,
      "rewards/rejected": -3.5535731315612793,
      "step": 350
    },
    {
      "epoch": 0.6269046582498912,
      "grad_norm": 10.162516049642594,
      "learning_rate": 4.591968008053685e-07,
      "logits/chosen": 0.47471824288368225,
      "logits/rejected": 0.5946096181869507,
      "logps/chosen": -313.26104736328125,
      "logps/rejected": -352.19842529296875,
      "loss": 0.6269,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.6822806596755981,
      "rewards/margins": 3.897700548171997,
      "rewards/rejected": -3.2154202461242676,
      "step": 360
    },
    {
      "epoch": 0.6443186765346104,
      "grad_norm": 12.196841859188872,
      "learning_rate": 4.5658492154623025e-07,
      "logits/chosen": 0.30948159098625183,
      "logits/rejected": 0.3537755012512207,
      "logps/chosen": -361.9256896972656,
      "logps/rejected": -385.45635986328125,
      "loss": 0.6485,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 1.5001431703567505,
      "rewards/margins": 4.353220462799072,
      "rewards/rejected": -2.853076457977295,
      "step": 370
    },
    {
      "epoch": 0.6617326948193296,
      "grad_norm": 14.472880692058368,
      "learning_rate": 4.538999363556276e-07,
      "logits/chosen": 0.4529440999031067,
      "logits/rejected": 0.5350080728530884,
      "logps/chosen": -315.1925048828125,
      "logps/rejected": -350.2633361816406,
      "loss": 0.624,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 1.4465656280517578,
      "rewards/margins": 4.741408348083496,
      "rewards/rejected": -3.294842481613159,
      "step": 380
    },
    {
      "epoch": 0.6791467131040487,
      "grad_norm": 10.826712989058954,
      "learning_rate": 4.5114279539169353e-07,
      "logits/chosen": 0.4620792269706726,
      "logits/rejected": 0.4736354351043701,
      "logps/chosen": -323.7712707519531,
      "logps/rejected": -383.2652282714844,
      "loss": 0.6394,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.8169974088668823,
      "rewards/margins": 5.20931339263916,
      "rewards/rejected": -4.39231538772583,
      "step": 390
    },
    {
      "epoch": 0.696560731388768,
      "grad_norm": 10.67277873114678,
      "learning_rate": 4.483144743469268e-07,
      "logits/chosen": 0.38168865442276,
      "logits/rejected": 0.49841055274009705,
      "logps/chosen": -322.7072448730469,
      "logps/rejected": -355.303955078125,
      "loss": 0.6259,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": 0.9774483442306519,
      "rewards/margins": 3.831352949142456,
      "rewards/rejected": -2.8539042472839355,
      "step": 400
    },
    {
      "epoch": 0.696560731388768,
      "eval_logits/chosen": 0.39267322421073914,
      "eval_logits/rejected": 0.45306530594825745,
      "eval_logps/chosen": -346.1069641113281,
      "eval_logps/rejected": -381.87127685546875,
      "eval_loss": 0.6327958703041077,
      "eval_rewards/accuracies": 0.7721518874168396,
      "eval_rewards/chosen": 0.9885040521621704,
      "eval_rewards/margins": 4.526716232299805,
      "eval_rewards/rejected": -3.538212299346924,
      "eval_runtime": 70.7229,
      "eval_samples_per_second": 35.349,
      "eval_steps_per_second": 1.117,
      "step": 400
    },
    {
      "epoch": 0.7139747496734872,
      "grad_norm": 9.610566425967104,
      "learning_rate": 4.454159741029162e-07,
      "logits/chosen": 0.363937646150589,
      "logits/rejected": 0.43550723791122437,
      "logps/chosen": -343.9782409667969,
      "logps/rejected": -362.17962646484375,
      "loss": 0.6254,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 1.068337082862854,
      "rewards/margins": 4.6207051277160645,
      "rewards/rejected": -3.5523681640625,
      "step": 410
    },
    {
      "epoch": 0.7313887679582064,
      "grad_norm": 12.035966522406508,
      "learning_rate": 4.4244832037614953e-07,
      "logits/chosen": 0.41232219338417053,
      "logits/rejected": 0.47881880402565,
      "logps/chosen": -335.7079772949219,
      "logps/rejected": -382.82403564453125,
      "loss": 0.5908,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 1.1208299398422241,
      "rewards/margins": 5.079546928405762,
      "rewards/rejected": -3.9587173461914062,
      "step": 420
    },
    {
      "epoch": 0.7488027862429255,
      "grad_norm": 10.889845296101432,
      "learning_rate": 4.394125633550352e-07,
      "logits/chosen": 0.44531410932540894,
      "logits/rejected": 0.5164756774902344,
      "logps/chosen": -337.1859436035156,
      "logps/rejected": -394.09906005859375,
      "loss": 0.6163,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.9268868565559387,
      "rewards/margins": 5.295614719390869,
      "rewards/rejected": -4.368727684020996,
      "step": 430
    },
    {
      "epoch": 0.7662168045276447,
      "grad_norm": 11.219285319452393,
      "learning_rate": 4.363097773282631e-07,
      "logits/chosen": 0.4676145613193512,
      "logits/rejected": 0.46276092529296875,
      "logps/chosen": -303.9361877441406,
      "logps/rejected": -363.8067321777344,
      "loss": 0.6134,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 1.4285213947296143,
      "rewards/margins": 5.457498550415039,
      "rewards/rejected": -4.028977394104004,
      "step": 440
    },
    {
      "epoch": 0.783630822812364,
      "grad_norm": 13.505959690725264,
      "learning_rate": 4.3314106030463736e-07,
      "logits/chosen": 0.4363015592098236,
      "logits/rejected": 0.5514327883720398,
      "logps/chosen": -316.35748291015625,
      "logps/rejected": -357.69384765625,
      "loss": 0.6506,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 1.2509381771087646,
      "rewards/margins": 4.098021507263184,
      "rewards/rejected": -2.847083330154419,
      "step": 450
    },
    {
      "epoch": 0.8010448410970832,
      "grad_norm": 10.355065488972091,
      "learning_rate": 4.299075336245144e-07,
      "logits/chosen": 0.3801150918006897,
      "logits/rejected": 0.44872745871543884,
      "logps/chosen": -349.3464050292969,
      "logps/rejected": -385.3382873535156,
      "loss": 0.644,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 1.4206287860870361,
      "rewards/margins": 5.481839656829834,
      "rewards/rejected": -4.061211109161377,
      "step": 460
    },
    {
      "epoch": 0.8184588593818024,
      "grad_norm": 11.847424597987043,
      "learning_rate": 4.2661034156298474e-07,
      "logits/chosen": 0.4114614427089691,
      "logits/rejected": 0.48444080352783203,
      "logps/chosen": -317.6768493652344,
      "logps/rejected": -355.9494934082031,
      "loss": 0.6044,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": 1.9520959854125977,
      "rewards/margins": 5.045262813568115,
      "rewards/rejected": -3.0931668281555176,
      "step": 470
    },
    {
      "epoch": 0.8358728776665215,
      "grad_norm": 12.525368413543276,
      "learning_rate": 4.232506509249397e-07,
      "logits/chosen": 0.3970574736595154,
      "logits/rejected": 0.5053227543830872,
      "logps/chosen": -333.1136474609375,
      "logps/rejected": -341.1883544921875,
      "loss": 0.5974,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 1.5816986560821533,
      "rewards/margins": 4.520462989807129,
      "rewards/rejected": -2.9387643337249756,
      "step": 480
    },
    {
      "epoch": 0.8532868959512407,
      "grad_norm": 13.142971908163307,
      "learning_rate": 4.1982965063216353e-07,
      "logits/chosen": 0.3946320712566376,
      "logits/rejected": 0.48155736923217773,
      "logps/chosen": -375.7856140136719,
      "logps/rejected": -376.10791015625,
      "loss": 0.644,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 1.4027400016784668,
      "rewards/margins": 4.7545166015625,
      "rewards/rejected": -3.351776599884033,
      "step": 490
    },
    {
      "epoch": 0.87070091423596,
      "grad_norm": 14.103469850176968,
      "learning_rate": 4.1634855130259996e-07,
      "logits/chosen": 0.4162313938140869,
      "logits/rejected": 0.4359792172908783,
      "logps/chosen": -310.9317321777344,
      "logps/rejected": -360.1820373535156,
      "loss": 0.5709,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 2.0291037559509277,
      "rewards/margins": 5.953708648681641,
      "rewards/rejected": -3.924605131149292,
      "step": 500
    },
    {
      "epoch": 0.87070091423596,
      "eval_logits/chosen": 0.3562653958797455,
      "eval_logits/rejected": 0.41477009654045105,
      "eval_logps/chosen": -346.84149169921875,
      "eval_logps/rejected": -386.58038330078125,
      "eval_loss": 0.6218530535697937,
      "eval_rewards/accuracies": 0.7816455960273743,
      "eval_rewards/chosen": 0.915044367313385,
      "eval_rewards/margins": 4.924162864685059,
      "eval_rewards/rejected": -4.009118556976318,
      "eval_runtime": 70.6374,
      "eval_samples_per_second": 35.392,
      "eval_steps_per_second": 1.118,
      "step": 500
    },
    {
      "epoch": 0.8881149325206792,
      "grad_norm": 12.576666962608346,
      "learning_rate": 4.128085848219408e-07,
      "logits/chosen": 0.3370770215988159,
      "logits/rejected": 0.47335267066955566,
      "logps/chosen": -318.4114990234375,
      "logps/rejected": -353.0730895996094,
      "loss": 0.6228,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.7541842460632324,
      "rewards/margins": 3.9734835624694824,
      "rewards/rejected": -3.21929931640625,
      "step": 510
    },
    {
      "epoch": 0.9055289508053983,
      "grad_norm": 12.733877298048826,
      "learning_rate": 4.092110039076884e-07,
      "logits/chosen": 0.4031103253364563,
      "logits/rejected": 0.4697874188423157,
      "logps/chosen": -305.5635681152344,
      "logps/rejected": -355.68865966796875,
      "loss": 0.6122,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": 1.5306105613708496,
      "rewards/margins": 5.436779499053955,
      "rewards/rejected": -3.9061686992645264,
      "step": 520
    },
    {
      "epoch": 0.9229429690901175,
      "grad_norm": 15.20563525082569,
      "learning_rate": 4.055570816658457e-07,
      "logits/chosen": 0.2911894917488098,
      "logits/rejected": 0.41246724128723145,
      "logps/chosen": -349.1706848144531,
      "logps/rejected": -364.3077087402344,
      "loss": 0.5931,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 1.3659398555755615,
      "rewards/margins": 4.93996524810791,
      "rewards/rejected": -3.5740256309509277,
      "step": 530
    },
    {
      "epoch": 0.9403569873748368,
      "grad_norm": 11.280219686189042,
      "learning_rate": 4.0184811114039184e-07,
      "logits/chosen": 0.42698025703430176,
      "logits/rejected": 0.5653012990951538,
      "logps/chosen": -307.64794921875,
      "logps/rejected": -315.4483337402344,
      "loss": 0.6033,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 1.7323646545410156,
      "rewards/margins": 4.723189353942871,
      "rewards/rejected": -2.9908242225646973,
      "step": 540
    },
    {
      "epoch": 0.957771005659556,
      "grad_norm": 16.19771019567853,
      "learning_rate": 3.9808540485570195e-07,
      "logits/chosen": 0.46557921171188354,
      "logits/rejected": 0.5172798037528992,
      "logps/chosen": -310.8573303222656,
      "logps/rejected": -352.49908447265625,
      "loss": 0.6182,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 1.6635421514511108,
      "rewards/margins": 4.843376636505127,
      "rewards/rejected": -3.1798348426818848,
      "step": 550
    },
    {
      "epoch": 0.9751850239442751,
      "grad_norm": 9.424230661957003,
      "learning_rate": 3.9427029435207295e-07,
      "logits/chosen": 0.18435801565647125,
      "logits/rejected": 0.24499377608299255,
      "logps/chosen": -365.77032470703125,
      "logps/rejected": -416.8031311035156,
      "loss": 0.6237,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.7337768077850342,
      "rewards/margins": 5.349893093109131,
      "rewards/rejected": -3.6161162853240967,
      "step": 560
    },
    {
      "epoch": 0.9925990422289943,
      "grad_norm": 12.241682769598608,
      "learning_rate": 3.904041297145201e-07,
      "logits/chosen": 0.3250972628593445,
      "logits/rejected": 0.4511636793613434,
      "logps/chosen": -360.701904296875,
      "logps/rejected": -388.0128479003906,
      "loss": 0.6399,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 1.4425889253616333,
      "rewards/margins": 4.446944236755371,
      "rewards/rejected": -3.0043554306030273,
      "step": 570
    },
    {
      "epoch": 1.0100130605137136,
      "grad_norm": 11.181442404802889,
      "learning_rate": 3.8648827909501176e-07,
      "logits/chosen": 0.29660564661026,
      "logits/rejected": 0.44677844643592834,
      "logps/chosen": -361.0950012207031,
      "logps/rejected": -377.21856689453125,
      "loss": 0.602,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 1.9495298862457275,
      "rewards/margins": 4.22812557220459,
      "rewards/rejected": -2.2785956859588623,
      "step": 580
    },
    {
      "epoch": 1.0274270787984328,
      "grad_norm": 13.05290689111488,
      "learning_rate": 3.825241282283096e-07,
      "logits/chosen": 0.27790385484695435,
      "logits/rejected": 0.3375425934791565,
      "logps/chosen": -330.90374755859375,
      "logps/rejected": -361.35601806640625,
      "loss": 0.5808,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 1.607259750366211,
      "rewards/margins": 4.201107025146484,
      "rewards/rejected": -2.5938477516174316,
      "step": 590
    },
    {
      "epoch": 1.0448410970831519,
      "grad_norm": 11.463096590603662,
      "learning_rate": 3.785130799415871e-07,
      "logits/chosen": 0.24200713634490967,
      "logits/rejected": 0.35302695631980896,
      "logps/chosen": -299.9937744140625,
      "logps/rejected": -359.466064453125,
      "loss": 0.5835,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 1.2321202754974365,
      "rewards/margins": 5.646553039550781,
      "rewards/rejected": -4.414433002471924,
      "step": 600
    },
    {
      "epoch": 1.0448410970831519,
      "eval_logits/chosen": 0.2933425307273865,
      "eval_logits/rejected": 0.35039564967155457,
      "eval_logps/chosen": -340.9584045410156,
      "eval_logps/rejected": -382.8789978027344,
      "eval_loss": 0.6093774437904358,
      "eval_rewards/accuracies": 0.7721518874168396,
      "eval_rewards/chosen": 1.5033540725708008,
      "eval_rewards/margins": 5.142338275909424,
      "eval_rewards/rejected": -3.638984441757202,
      "eval_runtime": 70.631,
      "eval_samples_per_second": 35.395,
      "eval_steps_per_second": 1.118,
      "step": 600
    },
    {
      "epoch": 1.0622551153678712,
      "grad_norm": 12.526851438420609,
      "learning_rate": 3.7445655365799936e-07,
      "logits/chosen": 0.20473480224609375,
      "logits/rejected": 0.28544989228248596,
      "logps/chosen": -334.8023986816406,
      "logps/rejected": -346.4919128417969,
      "loss": 0.5649,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 2.128657579421997,
      "rewards/margins": 5.316029071807861,
      "rewards/rejected": -3.187371015548706,
      "step": 610
    },
    {
      "epoch": 1.0796691336525903,
      "grad_norm": 11.47710312237594,
      "learning_rate": 3.7035598489438006e-07,
      "logits/chosen": 0.233549565076828,
      "logits/rejected": 0.35726696252822876,
      "logps/chosen": -349.82196044921875,
      "logps/rejected": -364.85955810546875,
      "loss": 0.58,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.5395448803901672,
      "rewards/margins": 4.185884952545166,
      "rewards/rejected": -3.6463401317596436,
      "step": 620
    },
    {
      "epoch": 1.0970831519373094,
      "grad_norm": 12.961022529112094,
      "learning_rate": 3.662128247532433e-07,
      "logits/chosen": 0.290211945772171,
      "logits/rejected": 0.35047486424446106,
      "logps/chosen": -325.6573181152344,
      "logps/rejected": -386.93597412109375,
      "loss": 0.577,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 1.4243555068969727,
      "rewards/margins": 5.428494930267334,
      "rewards/rejected": -4.004139423370361,
      "step": 630
    },
    {
      "epoch": 1.1144971702220288,
      "grad_norm": 17.398617895045366,
      "learning_rate": 3.6202853940926934e-07,
      "logits/chosen": 0.3017658293247223,
      "logits/rejected": 0.4195927083492279,
      "logps/chosen": -324.07952880859375,
      "logps/rejected": -373.904296875,
      "loss": 0.5541,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": 1.8470207452774048,
      "rewards/margins": 6.674212455749512,
      "rewards/rejected": -4.827191352844238,
      "step": 640
    },
    {
      "epoch": 1.1319111885067479,
      "grad_norm": 10.562372952225951,
      "learning_rate": 3.5780460959045754e-07,
      "logits/chosen": 0.2920629680156708,
      "logits/rejected": 0.47484827041625977,
      "logps/chosen": -352.8973693847656,
      "logps/rejected": -350.5732421875,
      "loss": 0.5682,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 1.6669800281524658,
      "rewards/margins": 5.411717891693115,
      "rewards/rejected": -3.7447381019592285,
      "step": 650
    },
    {
      "epoch": 1.1493252067914672,
      "grad_norm": 13.783936557369373,
      "learning_rate": 3.5354253005412884e-07,
      "logits/chosen": 0.2738053798675537,
      "logits/rejected": 0.35663852095603943,
      "logps/chosen": -324.64068603515625,
      "logps/rejected": -380.22943115234375,
      "loss": 0.5667,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 2.069589614868164,
      "rewards/margins": 6.046194553375244,
      "rewards/rejected": -3.9766056537628174,
      "step": 660
    },
    {
      "epoch": 1.1667392250761863,
      "grad_norm": 13.97568005507501,
      "learning_rate": 3.49243809057963e-07,
      "logits/chosen": 0.4103721082210541,
      "logits/rejected": 0.4501830041408539,
      "logps/chosen": -294.91845703125,
      "logps/rejected": -333.655517578125,
      "loss": 0.5689,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": 1.485319972038269,
      "rewards/margins": 5.04614782333374,
      "rewards/rejected": -3.5608277320861816,
      "step": 670
    },
    {
      "epoch": 1.1841532433609054,
      "grad_norm": 16.332889829547256,
      "learning_rate": 3.449099678262595e-07,
      "logits/chosen": 0.3343231976032257,
      "logits/rejected": 0.3634433150291443,
      "logps/chosen": -338.9833984375,
      "logps/rejected": -394.47454833984375,
      "loss": 0.5436,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 1.4713348150253296,
      "rewards/margins": 4.879803657531738,
      "rewards/rejected": -3.408468723297119,
      "step": 680
    },
    {
      "epoch": 1.2015672616456248,
      "grad_norm": 12.006988661080449,
      "learning_rate": 3.4054254001160854e-07,
      "logits/chosen": 0.19994381070137024,
      "logits/rejected": 0.29035601019859314,
      "logps/chosen": -322.84503173828125,
      "logps/rejected": -359.9244384765625,
      "loss": 0.5763,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 2.1871232986450195,
      "rewards/margins": 5.3036580085754395,
      "rewards/rejected": -3.11653470993042,
      "step": 690
    },
    {
      "epoch": 1.2189812799303439,
      "grad_norm": 12.099919696795636,
      "learning_rate": 3.3614307115216525e-07,
      "logits/chosen": 0.3054899573326111,
      "logits/rejected": 0.3647094964981079,
      "logps/chosen": -335.3572998046875,
      "logps/rejected": -434.33526611328125,
      "loss": 0.5571,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 2.1948211193084717,
      "rewards/margins": 6.700743675231934,
      "rewards/rejected": -4.505923271179199,
      "step": 700
    },
    {
      "epoch": 1.2189812799303439,
      "eval_logits/chosen": 0.26492393016815186,
      "eval_logits/rejected": 0.3217140734195709,
      "eval_logps/chosen": -340.2962341308594,
      "eval_logps/rejected": -383.6949157714844,
      "eval_loss": 0.5991856455802917,
      "eval_rewards/accuracies": 0.7689873576164246,
      "eval_rewards/chosen": 1.569573163986206,
      "eval_rewards/margins": 5.29014778137207,
      "eval_rewards/rejected": -3.7205748558044434,
      "eval_runtime": 71.1179,
      "eval_samples_per_second": 35.153,
      "eval_steps_per_second": 1.111,
      "step": 700
    },
    {
      "epoch": 1.2363952982150632,
      "grad_norm": 12.738566526927597,
      "learning_rate": 3.317131181247166e-07,
      "logits/chosen": 0.37958696484565735,
      "logits/rejected": 0.35304760932922363,
      "logps/chosen": -288.7842712402344,
      "logps/rejected": -364.54840087890625,
      "loss": 0.5859,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 2.730753183364868,
      "rewards/margins": 6.293435096740723,
      "rewards/rejected": -3.5626816749572754,
      "step": 710
    },
    {
      "epoch": 1.2538093164997823,
      "grad_norm": 13.111265189779358,
      "learning_rate": 3.272542485937368e-07,
      "logits/chosen": 0.2904764413833618,
      "logits/rejected": 0.4464187026023865,
      "logps/chosen": -304.67669677734375,
      "logps/rejected": -352.59979248046875,
      "loss": 0.5557,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.094412326812744,
      "rewards/margins": 5.624947547912598,
      "rewards/rejected": -3.5305354595184326,
      "step": 720
    },
    {
      "epoch": 1.2712233347845014,
      "grad_norm": 10.886620928807497,
      "learning_rate": 3.2276804045662483e-07,
      "logits/chosen": 0.27517110109329224,
      "logits/rejected": 0.3347955644130707,
      "logps/chosen": -316.2988586425781,
      "logps/rejected": -356.43768310546875,
      "loss": 0.5636,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 1.5192517042160034,
      "rewards/margins": 5.492722511291504,
      "rewards/rejected": -3.973471164703369,
      "step": 730
    },
    {
      "epoch": 1.2886373530692208,
      "grad_norm": 13.8403479629244,
      "learning_rate": 3.182560812853202e-07,
      "logits/chosen": 0.11149139702320099,
      "logits/rejected": 0.3251429498195648,
      "logps/chosen": -330.1148376464844,
      "logps/rejected": -378.6295471191406,
      "loss": 0.5628,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 1.484678864479065,
      "rewards/margins": 6.525158882141113,
      "rewards/rejected": -5.0404791831970215,
      "step": 740
    },
    {
      "epoch": 1.30605137135394,
      "grad_norm": 12.534702754120891,
      "learning_rate": 3.1371996776449617e-07,
      "logits/chosen": 0.23182816803455353,
      "logits/rejected": 0.28866177797317505,
      "logps/chosen": -328.12646484375,
      "logps/rejected": -370.0992126464844,
      "loss": 0.5467,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 2.179534435272217,
      "rewards/margins": 6.180920600891113,
      "rewards/rejected": -4.0013861656188965,
      "step": 750
    },
    {
      "epoch": 1.3234653896386592,
      "grad_norm": 11.31518460314422,
      "learning_rate": 3.091613051265273e-07,
      "logits/chosen": 0.31086990237236023,
      "logits/rejected": 0.4172425866127014,
      "logps/chosen": -297.3329772949219,
      "logps/rejected": -360.5836181640625,
      "loss": 0.5642,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 2.1698195934295654,
      "rewards/margins": 5.839700222015381,
      "rewards/rejected": -3.6698811054229736,
      "step": 760
    },
    {
      "epoch": 1.3408794079233783,
      "grad_norm": 10.379774861709702,
      "learning_rate": 3.045817065834326e-07,
      "logits/chosen": 0.29344993829727173,
      "logits/rejected": 0.32599514722824097,
      "logps/chosen": -327.0237121582031,
      "logps/rejected": -358.82928466796875,
      "loss": 0.5485,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.6931324005126953,
      "rewards/margins": 5.127728462219238,
      "rewards/rejected": -3.4345970153808594,
      "step": 770
    },
    {
      "epoch": 1.3582934262080975,
      "grad_norm": 13.434946271526595,
      "learning_rate": 2.999827927559946e-07,
      "logits/chosen": 0.2696860730648041,
      "logits/rejected": 0.34511059522628784,
      "logps/chosen": -296.4570617675781,
      "logps/rejected": -353.8194274902344,
      "loss": 0.5431,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": 2.2521262168884277,
      "rewards/margins": 5.2349138259887695,
      "rewards/rejected": -2.982787609100342,
      "step": 780
    },
    {
      "epoch": 1.3757074444928168,
      "grad_norm": 11.291499182850764,
      "learning_rate": 2.9536619110025697e-07,
      "logits/chosen": 0.18604174256324768,
      "logits/rejected": 0.3162630796432495,
      "logps/chosen": -326.1578369140625,
      "logps/rejected": -353.59295654296875,
      "loss": 0.5544,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 1.702147126197815,
      "rewards/margins": 5.362542152404785,
      "rewards/rejected": -3.6603941917419434,
      "step": 790
    },
    {
      "epoch": 1.393121462777536,
      "grad_norm": 12.702999923621702,
      "learning_rate": 2.907335353316027e-07,
      "logits/chosen": 0.26579901576042175,
      "logits/rejected": 0.41910189390182495,
      "logps/chosen": -320.19598388671875,
      "logps/rejected": -349.5892028808594,
      "loss": 0.5532,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 1.7423750162124634,
      "rewards/margins": 5.454329490661621,
      "rewards/rejected": -3.7119553089141846,
      "step": 800
    },
    {
      "epoch": 1.393121462777536,
      "eval_logits/chosen": 0.2383224368095398,
      "eval_logits/rejected": 0.29605910181999207,
      "eval_logps/chosen": -338.84527587890625,
      "eval_logps/rejected": -383.7506408691406,
      "eval_loss": 0.5953884720802307,
      "eval_rewards/accuracies": 0.7784810066223145,
      "eval_rewards/chosen": 1.714667797088623,
      "eval_rewards/margins": 5.440816879272461,
      "eval_rewards/rejected": -3.726149082183838,
      "eval_runtime": 70.5705,
      "eval_samples_per_second": 35.426,
      "eval_steps_per_second": 1.119,
      "step": 800
    },
    {
      "epoch": 1.4105354810622552,
      "grad_norm": 15.05280589745863,
      "learning_rate": 2.860864648466179e-07,
      "logits/chosen": 0.27184057235717773,
      "logits/rejected": 0.3507101535797119,
      "logps/chosen": -332.3158264160156,
      "logps/rejected": -380.0777587890625,
      "loss": 0.5504,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": 1.7377653121948242,
      "rewards/margins": 6.386282920837402,
      "rewards/rejected": -4.648517608642578,
      "step": 810
    },
    {
      "epoch": 1.4279494993469743,
      "grad_norm": 14.457454699405789,
      "learning_rate": 2.814266241429447e-07,
      "logits/chosen": 0.15302935242652893,
      "logits/rejected": 0.317359060049057,
      "logps/chosen": -344.7713317871094,
      "logps/rejected": -361.2349548339844,
      "loss": 0.5695,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 1.5794878005981445,
      "rewards/margins": 5.287659645080566,
      "rewards/rejected": -3.708172559738159,
      "step": 820
    },
    {
      "epoch": 1.4453635176316935,
      "grad_norm": 12.867419385191441,
      "learning_rate": 2.767556622373292e-07,
      "logits/chosen": 0.22975726425647736,
      "logits/rejected": 0.23659822344779968,
      "logps/chosen": -314.1470642089844,
      "logps/rejected": -377.7834167480469,
      "loss": 0.5733,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": 2.079237222671509,
      "rewards/margins": 5.952226161956787,
      "rewards/rejected": -3.8729889392852783,
      "step": 830
    },
    {
      "epoch": 1.4627775359164126,
      "grad_norm": 14.1466386443094,
      "learning_rate": 2.7207523208207013e-07,
      "logits/chosen": 0.1673222780227661,
      "logits/rejected": 0.2645789682865143,
      "logps/chosen": -338.6355285644531,
      "logps/rejected": -376.7158203125,
      "loss": 0.5545,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": 1.4028663635253906,
      "rewards/margins": 5.203114032745361,
      "rewards/rejected": -3.8002476692199707,
      "step": 840
    },
    {
      "epoch": 1.480191554201132,
      "grad_norm": 11.594267715010666,
      "learning_rate": 2.6738698998007466e-07,
      "logits/chosen": 0.15234848856925964,
      "logits/rejected": 0.2801158130168915,
      "logps/chosen": -370.71966552734375,
      "logps/rejected": -379.37567138671875,
      "loss": 0.5661,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": 2.7123475074768066,
      "rewards/margins": 5.849883079528809,
      "rewards/rejected": -3.137535572052002,
      "step": 850
    },
    {
      "epoch": 1.4976055724858512,
      "grad_norm": 15.37507940467823,
      "learning_rate": 2.62692594998729e-07,
      "logits/chosen": 0.1588628888130188,
      "logits/rejected": 0.29955458641052246,
      "logps/chosen": -316.0282897949219,
      "logps/rejected": -358.241455078125,
      "loss": 0.5643,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": 2.1699862480163574,
      "rewards/margins": 5.592405796051025,
      "rewards/rejected": -3.422419786453247,
      "step": 860
    },
    {
      "epoch": 1.5150195907705704,
      "grad_norm": 9.693159960410714,
      "learning_rate": 2.5799370838278996e-07,
      "logits/chosen": 0.3118472099304199,
      "logits/rejected": 0.4519326686859131,
      "logps/chosen": -304.1625061035156,
      "logps/rejected": -326.2646484375,
      "loss": 0.5422,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": 2.2456982135772705,
      "rewards/margins": 6.166834831237793,
      "rewards/rejected": -3.9211363792419434,
      "step": 870
    },
    {
      "epoch": 1.5324336090552895,
      "grad_norm": 11.9031904060217,
      "learning_rate": 2.5329199296650667e-07,
      "logits/chosen": 0.3134709596633911,
      "logits/rejected": 0.28860020637512207,
      "logps/chosen": -309.3736572265625,
      "logps/rejected": -387.1235656738281,
      "loss": 0.5562,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 2.08250093460083,
      "rewards/margins": 5.88448429107666,
      "rewards/rejected": -3.80198335647583,
      "step": 880
    },
    {
      "epoch": 1.5498476273400086,
      "grad_norm": 13.923538398639188,
      "learning_rate": 2.4858911258517953e-07,
      "logits/chosen": 0.2275421917438507,
      "logits/rejected": 0.3342249095439911,
      "logps/chosen": -296.0014953613281,
      "logps/rejected": -338.13580322265625,
      "loss": 0.5327,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 1.5528757572174072,
      "rewards/margins": 5.536377906799316,
      "rewards/rejected": -3.983502149581909,
      "step": 890
    },
    {
      "epoch": 1.567261645624728,
      "grad_norm": 14.148141568298001,
      "learning_rate": 2.438867314863649e-07,
      "logits/chosen": 0.09563969075679779,
      "logits/rejected": 0.15113821625709534,
      "logps/chosen": -343.15765380859375,
      "logps/rejected": -385.72100830078125,
      "loss": 0.5168,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 2.650477886199951,
      "rewards/margins": 6.407762050628662,
      "rewards/rejected": -3.757284164428711,
      "step": 900
    },
    {
      "epoch": 1.567261645624728,
      "eval_logits/chosen": 0.22664774954319,
      "eval_logits/rejected": 0.28383785486221313,
      "eval_logps/chosen": -336.0577392578125,
      "eval_logps/rejected": -380.47088623046875,
      "eval_loss": 0.5929730534553528,
      "eval_rewards/accuracies": 0.7753164768218994,
      "eval_rewards/chosen": 1.993420958518982,
      "eval_rewards/margins": 5.391592025756836,
      "eval_rewards/rejected": -3.3981709480285645,
      "eval_runtime": 71.3572,
      "eval_samples_per_second": 35.035,
      "eval_steps_per_second": 1.107,
      "step": 900
    },
    {
      "epoch": 1.5846756639094473,
      "grad_norm": 13.105682838644256,
      "learning_rate": 2.391865137409335e-07,
      "logits/chosen": 0.22982971370220184,
      "logits/rejected": 0.374919593334198,
      "logps/chosen": -331.8807678222656,
      "logps/rejected": -353.5442810058594,
      "loss": 0.5451,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 1.277345895767212,
      "rewards/margins": 4.490405082702637,
      "rewards/rejected": -3.2130589485168457,
      "step": 910
    },
    {
      "epoch": 1.6020896821941664,
      "grad_norm": 12.840872961914933,
      "learning_rate": 2.3449012265419263e-07,
      "logits/chosen": 0.22155573964118958,
      "logits/rejected": 0.3400117754936218,
      "logps/chosen": -313.2369079589844,
      "logps/rejected": -364.26910400390625,
      "loss": 0.5456,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 1.4575579166412354,
      "rewards/margins": 5.934301853179932,
      "rewards/rejected": -4.476743221282959,
      "step": 920
    },
    {
      "epoch": 1.6195037004788855,
      "grad_norm": 12.101267909285811,
      "learning_rate": 2.297992201772775e-07,
      "logits/chosen": 0.17083992063999176,
      "logits/rejected": 0.2611861526966095,
      "logps/chosen": -355.92559814453125,
      "logps/rejected": -386.23529052734375,
      "loss": 0.5311,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": 1.9388973712921143,
      "rewards/margins": 6.043047904968262,
      "rewards/rejected": -4.104150295257568,
      "step": 930
    },
    {
      "epoch": 1.6369177187636046,
      "grad_norm": 15.414581786862673,
      "learning_rate": 2.2511546631902348e-07,
      "logits/chosen": 0.1613834798336029,
      "logits/rejected": 0.1402578502893448,
      "logps/chosen": -334.2019348144531,
      "logps/rejected": -398.59014892578125,
      "loss": 0.5263,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 2.1678028106689453,
      "rewards/margins": 6.486172676086426,
      "rewards/rejected": -4.318368911743164,
      "step": 940
    },
    {
      "epoch": 1.654331737048324,
      "grad_norm": 13.343592771065993,
      "learning_rate": 2.2044051855852414e-07,
      "logits/chosen": 0.23705515265464783,
      "logits/rejected": 0.37995973229408264,
      "logps/chosen": -293.1175231933594,
      "logps/rejected": -328.06488037109375,
      "loss": 0.5589,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": 2.2306838035583496,
      "rewards/margins": 6.733452796936035,
      "rewards/rejected": -4.502768516540527,
      "step": 950
    },
    {
      "epoch": 1.6717457553330433,
      "grad_norm": 12.544780148755235,
      "learning_rate": 2.15776031258586e-07,
      "logits/chosen": 0.16758579015731812,
      "logits/rejected": 0.2723161578178406,
      "logps/chosen": -332.1502685546875,
      "logps/rejected": -419.9957580566406,
      "loss": 0.5405,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": 2.5997824668884277,
      "rewards/margins": 7.705753326416016,
      "rewards/rejected": -5.105970859527588,
      "step": 960
    },
    {
      "epoch": 1.6891597736177624,
      "grad_norm": 14.398467775390248,
      "learning_rate": 2.1112365508028512e-07,
      "logits/chosen": 0.22164323925971985,
      "logits/rejected": 0.35882630944252014,
      "logps/chosen": -311.85748291015625,
      "logps/rejected": -345.79376220703125,
      "loss": 0.5509,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": 2.5555522441864014,
      "rewards/margins": 6.213547229766846,
      "rewards/rejected": -3.6579947471618652,
      "step": 970
    },
    {
      "epoch": 1.7065737919024815,
      "grad_norm": 10.483483913623502,
      "learning_rate": 2.064850363988335e-07,
      "logits/chosen": 0.24829120934009552,
      "logits/rejected": 0.26147156953811646,
      "logps/chosen": -323.1725158691406,
      "logps/rejected": -371.21734619140625,
      "loss": 0.5522,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": 1.836071252822876,
      "rewards/margins": 6.013415336608887,
      "rewards/rejected": -4.177343368530273,
      "step": 980
    },
    {
      "epoch": 1.7239878101872006,
      "grad_norm": 13.297935553611692,
      "learning_rate": 2.0186181672096303e-07,
      "logits/chosen": 0.22199273109436035,
      "logits/rejected": 0.34791290760040283,
      "logps/chosen": -365.37841796875,
      "logps/rejected": -357.319091796875,
      "loss": 0.5596,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": 1.8863897323608398,
      "rewards/margins": 5.01448917388916,
      "rewards/rejected": -3.1280996799468994,
      "step": 990
    },
    {
      "epoch": 1.74140182847192,
      "grad_norm": 11.129567589353632,
      "learning_rate": 1.9725563210403152e-07,
      "logits/chosen": 0.27127689123153687,
      "logits/rejected": 0.3801821172237396,
      "logps/chosen": -304.1812438964844,
      "logps/rejected": -333.21844482421875,
      "loss": 0.5232,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": 1.9027084112167358,
      "rewards/margins": 4.797359943389893,
      "rewards/rejected": -2.8946516513824463,
      "step": 1000
    },
    {
      "epoch": 1.74140182847192,
      "eval_logits/chosen": 0.22198127210140228,
      "eval_logits/rejected": 0.2787408232688904,
      "eval_logps/chosen": -338.6839294433594,
      "eval_logps/rejected": -386.5126953125,
      "eval_loss": 0.5883829593658447,
      "eval_rewards/accuracies": 0.7816455960273743,
      "eval_rewards/chosen": 1.7307991981506348,
      "eval_rewards/margins": 5.733153343200684,
      "eval_rewards/rejected": -4.002354621887207,
      "eval_runtime": 70.4432,
      "eval_samples_per_second": 35.49,
      "eval_steps_per_second": 1.121,
      "step": 1000
    },
    {
      "epoch": 1.758815846756639,
      "grad_norm": 14.281945700020602,
      "learning_rate": 1.9266811257705763e-07,
      "logits/chosen": 0.21578781306743622,
      "logits/rejected": 0.30397921800613403,
      "logps/chosen": -329.77044677734375,
      "logps/rejected": -368.17926025390625,
      "loss": 0.5424,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": 2.238178253173828,
      "rewards/margins": 6.452548980712891,
      "rewards/rejected": -4.214371204376221,
      "step": 1010
    },
    {
      "epoch": 1.7762298650413584,
      "grad_norm": 13.233745779571802,
      "learning_rate": 1.8810088156388848e-07,
      "logits/chosen": 0.2450670748949051,
      "logits/rejected": 0.3020363748073578,
      "logps/chosen": -298.3768615722656,
      "logps/rejected": -362.04327392578125,
      "loss": 0.5603,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.085683822631836,
      "rewards/margins": 5.677943229675293,
      "rewards/rejected": -3.592259645462036,
      "step": 1020
    },
    {
      "epoch": 1.7936438833260775,
      "grad_norm": 11.453241931075056,
      "learning_rate": 1.8355555530870564e-07,
      "logits/chosen": 0.338943213224411,
      "logits/rejected": 0.35035696625709534,
      "logps/chosen": -256.6951599121094,
      "logps/rejected": -336.91552734375,
      "loss": 0.5418,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": 2.064159870147705,
      "rewards/margins": 6.696907043457031,
      "rewards/rejected": -4.632746696472168,
      "step": 1030
    },
    {
      "epoch": 1.8110579016107966,
      "grad_norm": 12.077192617585345,
      "learning_rate": 1.7903374230407104e-07,
      "logits/chosen": 0.2699436843395233,
      "logits/rejected": 0.28396037220954895,
      "logps/chosen": -290.0744934082031,
      "logps/rejected": -392.7906494140625,
      "loss": 0.5544,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 2.268066167831421,
      "rewards/margins": 6.793740749359131,
      "rewards/rejected": -4.525674343109131,
      "step": 1040
    },
    {
      "epoch": 1.828471919895516,
      "grad_norm": 12.946293550899172,
      "learning_rate": 1.7453704272171603e-07,
      "logits/chosen": 0.1450766772031784,
      "logits/rejected": 0.24746115505695343,
      "logps/chosen": -318.6651611328125,
      "logps/rejected": -367.54510498046875,
      "loss": 0.5793,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.2983639240264893,
      "rewards/margins": 7.265191078186035,
      "rewards/rejected": -4.966826915740967,
      "step": 1050
    },
    {
      "epoch": 1.845885938180235,
      "grad_norm": 14.659109006168315,
      "learning_rate": 1.7006704784627575e-07,
      "logits/chosen": 0.21030068397521973,
      "logits/rejected": 0.31739652156829834,
      "logps/chosen": -316.27313232421875,
      "logps/rejected": -366.7966613769531,
      "loss": 0.5398,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 2.333252429962158,
      "rewards/margins": 6.3725409507751465,
      "rewards/rejected": -4.0392889976501465,
      "step": 1060
    },
    {
      "epoch": 1.8632999564649544,
      "grad_norm": 11.575655038254915,
      "learning_rate": 1.656253395121674e-07,
      "logits/chosen": 0.2197595089673996,
      "logits/rejected": 0.3256808817386627,
      "logps/chosen": -326.97381591796875,
      "logps/rejected": -351.43341064453125,
      "loss": 0.5353,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": 2.4176201820373535,
      "rewards/margins": 6.151999473571777,
      "rewards/rejected": -3.734379291534424,
      "step": 1070
    },
    {
      "epoch": 1.8807139747496735,
      "grad_norm": 16.012865418831396,
      "learning_rate": 1.6121348954381397e-07,
      "logits/chosen": 0.2955351173877716,
      "logits/rejected": 0.4262094497680664,
      "logps/chosen": -309.02435302734375,
      "logps/rejected": -357.2958068847656,
      "loss": 0.5462,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": 1.9097408056259155,
      "rewards/margins": 5.701088905334473,
      "rewards/rejected": -3.7913482189178467,
      "step": 1080
    },
    {
      "epoch": 1.8981279930343926,
      "grad_norm": 16.856514300669417,
      "learning_rate": 1.5683305919940872e-07,
      "logits/chosen": 0.28492286801338196,
      "logits/rejected": 0.370318740606308,
      "logps/chosen": -324.5318908691406,
      "logps/rejected": -355.6055908203125,
      "loss": 0.5671,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.586484670639038,
      "rewards/margins": 5.815837860107422,
      "rewards/rejected": -4.229353427886963,
      "step": 1090
    },
    {
      "epoch": 1.9155420113191117,
      "grad_norm": 11.897078714973112,
      "learning_rate": 1.5248559861842053e-07,
      "logits/chosen": 0.23165082931518555,
      "logits/rejected": 0.2759276032447815,
      "logps/chosen": -305.85174560546875,
      "logps/rejected": -377.9584045410156,
      "loss": 0.5574,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": 1.902627944946289,
      "rewards/margins": 6.008756160736084,
      "rewards/rejected": -4.106128692626953,
      "step": 1100
    },
    {
      "epoch": 1.9155420113191117,
      "eval_logits/chosen": 0.21340951323509216,
      "eval_logits/rejected": 0.2706116735935211,
      "eval_logps/chosen": -337.5714416503906,
      "eval_logps/rejected": -385.840087890625,
      "eval_loss": 0.5849184393882751,
      "eval_rewards/accuracies": 0.7911392450332642,
      "eval_rewards/chosen": 1.8420459032058716,
      "eval_rewards/margins": 5.777135372161865,
      "eval_rewards/rejected": -3.935089588165283,
      "eval_runtime": 70.3599,
      "eval_samples_per_second": 35.532,
      "eval_steps_per_second": 1.123,
      "step": 1100
    },
    {
      "epoch": 1.932956029603831,
      "grad_norm": 13.676521515428378,
      "learning_rate": 1.481726462730327e-07,
      "logits/chosen": 0.2556658387184143,
      "logits/rejected": 0.3973894417285919,
      "logps/chosen": -321.7857666015625,
      "logps/rejected": -359.394775390625,
      "loss": 0.5369,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 1.8009535074234009,
      "rewards/margins": 6.82882022857666,
      "rewards/rejected": -5.027866363525391,
      "step": 1110
    },
    {
      "epoch": 1.9503700478885504,
      "grad_norm": 14.221820537526096,
      "learning_rate": 1.438957284237109e-07,
      "logits/chosen": 0.16190308332443237,
      "logits/rejected": 0.28133997321128845,
      "logps/chosen": -370.27801513671875,
      "logps/rejected": -407.38848876953125,
      "loss": 0.5675,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.6363109350204468,
      "rewards/margins": 6.053797721862793,
      "rewards/rejected": -4.417486667633057,
      "step": 1120
    },
    {
      "epoch": 1.9677840661732695,
      "grad_norm": 12.575846049875057,
      "learning_rate": 1.396563585790927e-07,
      "logits/chosen": 0.23068062961101532,
      "logits/rejected": 0.28086987137794495,
      "logps/chosen": -291.02239990234375,
      "logps/rejected": -354.7329406738281,
      "loss": 0.5651,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": 1.4899548292160034,
      "rewards/margins": 5.830671787261963,
      "rewards/rejected": -4.340716361999512,
      "step": 1130
    },
    {
      "epoch": 1.9851980844579886,
      "grad_norm": 12.302064991278899,
      "learning_rate": 1.3545603696038987e-07,
      "logits/chosen": 0.2145300656557083,
      "logits/rejected": 0.2948189675807953,
      "logps/chosen": -322.04754638671875,
      "logps/rejected": -364.0850830078125,
      "loss": 0.5366,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 1.6283271312713623,
      "rewards/margins": 5.910183429718018,
      "rewards/rejected": -4.281857013702393,
      "step": 1140
    },
    {
      "epoch": 2.0026121027427077,
      "grad_norm": 13.71124690365027,
      "learning_rate": 1.3129624997049228e-07,
      "logits/chosen": 0.22102287411689758,
      "logits/rejected": 0.25346601009368896,
      "logps/chosen": -299.03997802734375,
      "logps/rejected": -374.4776916503906,
      "loss": 0.5346,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": 2.4188122749328613,
      "rewards/margins": 6.5757155418396,
      "rewards/rejected": -4.156903266906738,
      "step": 1150
    },
    {
      "epoch": 2.0200261210274273,
      "grad_norm": 15.920538460335408,
      "learning_rate": 1.2717846966796214e-07,
      "logits/chosen": 0.20584805309772491,
      "logits/rejected": 0.30077749490737915,
      "logps/chosen": -354.84576416015625,
      "logps/rejected": -403.926513671875,
      "loss": 0.5265,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.021745204925537,
      "rewards/margins": 6.796699523925781,
      "rewards/rejected": -4.774954319000244,
      "step": 1160
    },
    {
      "epoch": 2.0374401393121464,
      "grad_norm": 12.844613549152813,
      "learning_rate": 1.2310415324610435e-07,
      "logits/chosen": 0.17609162628650665,
      "logits/rejected": 0.24105580151081085,
      "logps/chosen": -340.9830017089844,
      "logps/rejected": -415.0347595214844,
      "loss": 0.5227,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": 2.298529863357544,
      "rewards/margins": 6.613799095153809,
      "rewards/rejected": -4.315269947052002,
      "step": 1170
    },
    {
      "epoch": 2.0548541575968655,
      "grad_norm": 11.610832192571714,
      "learning_rate": 1.1907474251729752e-07,
      "logits/chosen": 0.1663503348827362,
      "logits/rejected": 0.31382399797439575,
      "logps/chosen": -316.79449462890625,
      "logps/rejected": -344.66070556640625,
      "loss": 0.5237,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.4878411293029785,
      "rewards/margins": 6.203591346740723,
      "rewards/rejected": -3.715749740600586,
      "step": 1180
    },
    {
      "epoch": 2.0722681758815846,
      "grad_norm": 13.49440122728778,
      "learning_rate": 1.1509166340276739e-07,
      "logits/chosen": 0.08771935105323792,
      "logits/rejected": 0.24409714341163635,
      "logps/chosen": -349.5455017089844,
      "logps/rejected": -384.40618896484375,
      "loss": 0.5192,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.7756285667419434,
      "rewards/margins": 7.52169132232666,
      "rewards/rejected": -4.746062278747559,
      "step": 1190
    },
    {
      "epoch": 2.0896821941663037,
      "grad_norm": 13.523283111167856,
      "learning_rate": 1.1115632542798467e-07,
      "logits/chosen": 0.2006814181804657,
      "logits/rejected": 0.27478426694869995,
      "logps/chosen": -333.44720458984375,
      "logps/rejected": -384.8387145996094,
      "loss": 0.5077,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": 1.8601166009902954,
      "rewards/margins": 5.925691604614258,
      "rewards/rejected": -4.06557559967041,
      "step": 1200
    },
    {
      "epoch": 2.0896821941663037,
      "eval_logits/chosen": 0.2083120197057724,
      "eval_logits/rejected": 0.2656678259372711,
      "eval_logps/chosen": -339.80426025390625,
      "eval_logps/rejected": -388.9606628417969,
      "eval_loss": 0.5841708183288574,
      "eval_rewards/accuracies": 0.7879746556282043,
      "eval_rewards/chosen": 1.6187670230865479,
      "eval_rewards/margins": 5.865919589996338,
      "eval_rewards/rejected": -4.247152805328369,
      "eval_runtime": 72.6682,
      "eval_samples_per_second": 34.403,
      "eval_steps_per_second": 1.087,
      "step": 1200
    },
    {
      "epoch": 2.1070962124510233,
      "grad_norm": 15.694137545891444,
      "learning_rate": 1.0727012122386415e-07,
      "logits/chosen": 0.22686560451984406,
      "logits/rejected": 0.31223005056381226,
      "logps/chosen": -321.732177734375,
      "logps/rejected": -370.21453857421875,
      "loss": 0.5185,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 2.1967837810516357,
      "rewards/margins": 6.164200782775879,
      "rewards/rejected": -3.967416286468506,
      "step": 1210
    },
    {
      "epoch": 2.1245102307357424,
      "grad_norm": 9.832217655573857,
      "learning_rate": 1.0343442603394282e-07,
      "logits/chosen": 0.15876635909080505,
      "logits/rejected": 0.19615477323532104,
      "logps/chosen": -317.2491149902344,
      "logps/rejected": -393.39251708984375,
      "loss": 0.4803,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 2.478919267654419,
      "rewards/margins": 6.481255531311035,
      "rewards/rejected": -4.002336502075195,
      "step": 1220
    },
    {
      "epoch": 2.1419242490204615,
      "grad_norm": 13.519648851089453,
      "learning_rate": 9.965059722771113e-08,
      "logits/chosen": 0.14877358078956604,
      "logits/rejected": 0.22332414984703064,
      "logps/chosen": -342.74566650390625,
      "logps/rejected": -403.8954162597656,
      "loss": 0.4879,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.4974751472473145,
      "rewards/margins": 6.861520290374756,
      "rewards/rejected": -4.364045143127441,
      "step": 1230
    },
    {
      "epoch": 2.1593382673051806,
      "grad_norm": 11.584711006576716,
      "learning_rate": 9.59199738202689e-08,
      "logits/chosen": 0.2872892916202545,
      "logits/rejected": 0.3496358394622803,
      "logps/chosen": -306.6114196777344,
      "logps/rejected": -357.1216125488281,
      "loss": 0.5042,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": 2.1603891849517822,
      "rewards/margins": 7.638640403747559,
      "rewards/rejected": -5.478252410888672,
      "step": 1240
    },
    {
      "epoch": 2.1767522855898997,
      "grad_norm": 13.250705440919303,
      "learning_rate": 9.22438759984775e-08,
      "logits/chosen": 0.23227651417255402,
      "logits/rejected": 0.2910657525062561,
      "logps/chosen": -294.86236572265625,
      "logps/rejected": -350.4806213378906,
      "loss": 0.4824,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 2.0290393829345703,
      "rewards/margins": 6.93194580078125,
      "rewards/rejected": -4.9029059410095215,
      "step": 1250
    },
    {
      "epoch": 2.194166303874619,
      "grad_norm": 10.291356601407243,
      "learning_rate": 8.862360465377356e-08,
      "logits/chosen": 0.17067928612232208,
      "logits/rejected": 0.24521295726299286,
      "logps/chosen": -333.3260803222656,
      "logps/rejected": -365.29766845703125,
      "loss": 0.5221,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": 1.595489263534546,
      "rewards/margins": 5.210162162780762,
      "rewards/rejected": -3.6146724224090576,
      "step": 1260
    },
    {
      "epoch": 2.2115803221593384,
      "grad_norm": 13.900410810772762,
      "learning_rate": 8.506044092181233e-08,
      "logits/chosen": 0.20954521000385284,
      "logits/rejected": 0.259429007768631,
      "logps/chosen": -309.11968994140625,
      "logps/rejected": -383.56964111328125,
      "loss": 0.5201,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": 2.049609422683716,
      "rewards/margins": 6.555842399597168,
      "rewards/rejected": -4.506233215332031,
      "step": 1270
    },
    {
      "epoch": 2.2289943404440575,
      "grad_norm": 12.91211847131762,
      "learning_rate": 8.15556457291011e-08,
      "logits/chosen": 0.16506996750831604,
      "logits/rejected": 0.3168487846851349,
      "logps/chosen": -314.2242431640625,
      "logps/rejected": -387.1705017089844,
      "loss": 0.4894,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.2459182739257812,
      "rewards/margins": 6.696404933929443,
      "rewards/rejected": -4.45048713684082,
      "step": 1280
    },
    {
      "epoch": 2.2464083587287766,
      "grad_norm": 14.30007682532237,
      "learning_rate": 7.811045934678467e-08,
      "logits/chosen": 0.20867156982421875,
      "logits/rejected": 0.3092384338378906,
      "logps/chosen": -326.1697692871094,
      "logps/rejected": -394.4437561035156,
      "loss": 0.5004,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": 2.4621286392211914,
      "rewards/margins": 6.926805019378662,
      "rewards/rejected": -4.464676856994629,
      "step": 1290
    },
    {
      "epoch": 2.2638223770134958,
      "grad_norm": 18.216105796084022,
      "learning_rate": 7.472610095174051e-08,
      "logits/chosen": 0.1239229068160057,
      "logits/rejected": 0.2724478840827942,
      "logps/chosen": -338.5323791503906,
      "logps/rejected": -356.60479736328125,
      "loss": 0.4952,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": 1.9667659997940063,
      "rewards/margins": 5.838881492614746,
      "rewards/rejected": -3.87211537361145,
      "step": 1300
    },
    {
      "epoch": 2.2638223770134958,
      "eval_logits/chosen": 0.21147681772708893,
      "eval_logits/rejected": 0.26940545439720154,
      "eval_logps/chosen": -336.6758728027344,
      "eval_logps/rejected": -385.40179443359375,
      "eval_loss": 0.5836542844772339,
      "eval_rewards/accuracies": 0.7816455960273743,
      "eval_rewards/chosen": 1.9316082000732422,
      "eval_rewards/margins": 5.822873115539551,
      "eval_rewards/rejected": -3.8912646770477295,
      "eval_runtime": 70.4626,
      "eval_samples_per_second": 35.48,
      "eval_steps_per_second": 1.121,
      "step": 1300
    },
    {
      "epoch": 2.281236395298215,
      "grad_norm": 12.712872435452034,
      "learning_rate": 7.140376819513782e-08,
      "logits/chosen": 0.17473125457763672,
      "logits/rejected": 0.24829836189746857,
      "logps/chosen": -291.07281494140625,
      "logps/rejected": -378.64520263671875,
      "loss": 0.5163,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.978382110595703,
      "rewards/margins": 7.624729156494141,
      "rewards/rejected": -4.646347522735596,
      "step": 1310
    },
    {
      "epoch": 2.2986504135829344,
      "grad_norm": 12.944938100249528,
      "learning_rate": 6.814463677861556e-08,
      "logits/chosen": 0.20871727168560028,
      "logits/rejected": 0.31438320875167847,
      "logps/chosen": -343.16485595703125,
      "logps/rejected": -391.0745544433594,
      "loss": 0.5179,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.9679534435272217,
      "rewards/margins": 5.962490081787109,
      "rewards/rejected": -3.9945366382598877,
      "step": 1320
    },
    {
      "epoch": 2.3160644318676535,
      "grad_norm": 11.974646389262404,
      "learning_rate": 6.494986003822631e-08,
      "logits/chosen": 0.2468450963497162,
      "logits/rejected": 0.37068819999694824,
      "logps/chosen": -307.6761779785156,
      "logps/rejected": -366.2533874511719,
      "loss": 0.4976,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 2.4139225482940674,
      "rewards/margins": 6.581143379211426,
      "rewards/rejected": -4.167220592498779,
      "step": 1330
    },
    {
      "epoch": 2.3334784501523727,
      "grad_norm": 13.867952710064575,
      "learning_rate": 6.182056853629625e-08,
      "logits/chosen": 0.2119470089673996,
      "logits/rejected": 0.32034552097320557,
      "logps/chosen": -315.0056457519531,
      "logps/rejected": -343.30120849609375,
      "loss": 0.4945,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.3241541385650635,
      "rewards/margins": 6.567809104919434,
      "rewards/rejected": -4.243655204772949,
      "step": 1340
    },
    {
      "epoch": 2.3508924684370918,
      "grad_norm": 12.973425439181085,
      "learning_rate": 5.875786966134294e-08,
      "logits/chosen": 0.3225061893463135,
      "logits/rejected": 0.2947853207588196,
      "logps/chosen": -304.63616943359375,
      "logps/rejected": -394.45196533203125,
      "loss": 0.5386,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 1.3494211435317993,
      "rewards/margins": 5.996374130249023,
      "rewards/rejected": -4.6469526290893555,
      "step": 1350
    },
    {
      "epoch": 2.368306486721811,
      "grad_norm": 15.471139649509851,
      "learning_rate": 5.576284723619537e-08,
      "logits/chosen": 0.15628595650196075,
      "logits/rejected": 0.2223624885082245,
      "logps/chosen": -351.3752136230469,
      "logps/rejected": -382.9642333984375,
      "loss": 0.5025,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": 1.9524848461151123,
      "rewards/margins": 6.2653093338012695,
      "rewards/rejected": -4.312824249267578,
      "step": 1360
    },
    {
      "epoch": 2.3857205050065304,
      "grad_norm": 12.394641149417646,
      "learning_rate": 5.283656113445181e-08,
      "logits/chosen": 0.20236392319202423,
      "logits/rejected": 0.2501729130744934,
      "logps/chosen": -332.77020263671875,
      "logps/rejected": -397.8937683105469,
      "loss": 0.4842,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 2.4705593585968018,
      "rewards/margins": 6.949694633483887,
      "rewards/rejected": -4.479134559631348,
      "step": 1370
    },
    {
      "epoch": 2.4031345232912495,
      "grad_norm": 11.158717402277203,
      "learning_rate": 4.998004690541358e-08,
      "logits/chosen": 0.18652646243572235,
      "logits/rejected": 0.2707822322845459,
      "logps/chosen": -306.2278747558594,
      "logps/rejected": -334.9998474121094,
      "loss": 0.5261,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": 2.4948792457580566,
      "rewards/margins": 6.4533843994140625,
      "rewards/rejected": -3.9585044384002686,
      "step": 1380
    },
    {
      "epoch": 2.4205485415759687,
      "grad_norm": 9.57471341907718,
      "learning_rate": 4.719431540762697e-08,
      "logits/chosen": 0.3041616976261139,
      "logits/rejected": 0.37496739625930786,
      "logps/chosen": -292.21923828125,
      "logps/rejected": -355.483154296875,
      "loss": 0.5055,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.382023572921753,
      "rewards/margins": 6.094801902770996,
      "rewards/rejected": -3.7127785682678223,
      "step": 1390
    },
    {
      "epoch": 2.4379625598606878,
      "grad_norm": 13.814429830375937,
      "learning_rate": 4.44803524511618e-08,
      "logits/chosen": 0.2870798110961914,
      "logits/rejected": 0.2993796169757843,
      "logps/chosen": -281.04522705078125,
      "logps/rejected": -349.2628173828125,
      "loss": 0.5236,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 2.1859898567199707,
      "rewards/margins": 6.1772050857543945,
      "rewards/rejected": -3.991215467453003,
      "step": 1400
    },
    {
      "epoch": 2.4379625598606878,
      "eval_logits/chosen": 0.18954265117645264,
      "eval_logits/rejected": 0.24652473628520966,
      "eval_logps/chosen": -337.7025451660156,
      "eval_logps/rejected": -387.1252746582031,
      "eval_loss": 0.5812255144119263,
      "eval_rewards/accuracies": 0.7879746556282043,
      "eval_rewards/chosen": 1.8289377689361572,
      "eval_rewards/margins": 5.892548084259033,
      "eval_rewards/rejected": -4.063610076904297,
      "eval_runtime": 70.4535,
      "eval_samples_per_second": 35.484,
      "eval_steps_per_second": 1.121,
      "step": 1400
    },
    {
      "epoch": 2.455376578145407,
      "grad_norm": 16.245030931518695,
      "learning_rate": 4.183911844875504e-08,
      "logits/chosen": 0.32038670778274536,
      "logits/rejected": 0.38889437913894653,
      "logps/chosen": -293.1011657714844,
      "logps/rejected": -366.3570556640625,
      "loss": 0.5035,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": 2.785410165786743,
      "rewards/margins": 7.212292671203613,
      "rewards/rejected": -4.426882743835449,
      "step": 1410
    },
    {
      "epoch": 2.4727905964301264,
      "grad_norm": 15.3110536389278,
      "learning_rate": 3.9271548075940994e-08,
      "logits/chosen": 0.22183004021644592,
      "logits/rejected": 0.20372053980827332,
      "logps/chosen": -289.2795104980469,
      "logps/rejected": -411.65118408203125,
      "loss": 0.5158,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": 2.527660608291626,
      "rewards/margins": 7.5610809326171875,
      "rewards/rejected": -5.033421039581299,
      "step": 1420
    },
    {
      "epoch": 2.4902046147148456,
      "grad_norm": 12.657778002838807,
      "learning_rate": 3.677854994029045e-08,
      "logits/chosen": 0.16141930222511292,
      "logits/rejected": 0.28222227096557617,
      "logps/chosen": -329.6971740722656,
      "logps/rejected": -367.878662109375,
      "loss": 0.5147,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": 2.3014779090881348,
      "rewards/margins": 6.771458625793457,
      "rewards/rejected": -4.469980716705322,
      "step": 1430
    },
    {
      "epoch": 2.5076186329995647,
      "grad_norm": 15.358056422127024,
      "learning_rate": 3.436100625987373e-08,
      "logits/chosen": 0.16851040720939636,
      "logits/rejected": 0.21520404517650604,
      "logps/chosen": -313.4747619628906,
      "logps/rejected": -369.7133483886719,
      "loss": 0.5227,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.977530837059021,
      "rewards/margins": 5.828042507171631,
      "rewards/rejected": -3.8505122661590576,
      "step": 1440
    },
    {
      "epoch": 2.525032651284284,
      "grad_norm": 14.637744065096362,
      "learning_rate": 3.201977255106292e-08,
      "logits/chosen": 0.19486942887306213,
      "logits/rejected": 0.33268415927886963,
      "logps/chosen": -309.989990234375,
      "logps/rejected": -368.7453308105469,
      "loss": 0.5085,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 1.9779468774795532,
      "rewards/margins": 7.094501495361328,
      "rewards/rejected": -5.1165547370910645,
      "step": 1450
    },
    {
      "epoch": 2.542446669569003,
      "grad_norm": 14.541660553593134,
      "learning_rate": 2.975567732578324e-08,
      "logits/chosen": 0.23976822197437286,
      "logits/rejected": 0.220484659075737,
      "logps/chosen": -272.70562744140625,
      "logps/rejected": -348.28485107421875,
      "loss": 0.5083,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.1403632164001465,
      "rewards/margins": 6.054244518280029,
      "rewards/rejected": -3.91388201713562,
      "step": 1460
    },
    {
      "epoch": 2.559860687853722,
      "grad_norm": 15.96580948389595,
      "learning_rate": 2.7569521798320368e-08,
      "logits/chosen": 0.073675736784935,
      "logits/rejected": 0.2692302465438843,
      "logps/chosen": -336.4574279785156,
      "logps/rejected": -362.8096618652344,
      "loss": 0.49,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.4895553588867188,
      "rewards/margins": 6.369107723236084,
      "rewards/rejected": -3.8795523643493652,
      "step": 1470
    },
    {
      "epoch": 2.5772747061384416,
      "grad_norm": 12.263431349665568,
      "learning_rate": 2.5462079601788367e-08,
      "logits/chosen": 0.2085539847612381,
      "logits/rejected": 0.31902140378952026,
      "logps/chosen": -305.46807861328125,
      "logps/rejected": -364.35003662109375,
      "loss": 0.5051,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": 1.7462059259414673,
      "rewards/margins": 5.998797416687012,
      "rewards/rejected": -4.252591609954834,
      "step": 1480
    },
    {
      "epoch": 2.5946887244231607,
      "grad_norm": 14.474568168334054,
      "learning_rate": 2.3434096514357403e-08,
      "logits/chosen": 0.17831678688526154,
      "logits/rejected": 0.23579061031341553,
      "logps/chosen": -312.26025390625,
      "logps/rejected": -350.52056884765625,
      "loss": 0.5033,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 2.2567784786224365,
      "rewards/margins": 6.1402106285095215,
      "rewards/rejected": -3.883431911468506,
      "step": 1490
    },
    {
      "epoch": 2.61210274270788,
      "grad_norm": 11.85075934174815,
      "learning_rate": 2.1486290195339153e-08,
      "logits/chosen": 0.19868260622024536,
      "logits/rejected": 0.25412097573280334,
      "logps/chosen": -341.95849609375,
      "logps/rejected": -417.5733337402344,
      "loss": 0.5001,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 1.9825395345687866,
      "rewards/margins": 6.966047763824463,
      "rewards/rejected": -4.983508110046387,
      "step": 1500
    },
    {
      "epoch": 2.61210274270788,
      "eval_logits/chosen": 0.182608962059021,
      "eval_logits/rejected": 0.23953425884246826,
      "eval_logps/chosen": -338.5595703125,
      "eval_logps/rejected": -388.22418212890625,
      "eval_loss": 0.5813568234443665,
      "eval_rewards/accuracies": 0.7848101258277893,
      "eval_rewards/chosen": 1.7432384490966797,
      "eval_rewards/margins": 5.916741847991943,
      "eval_rewards/rejected": -4.173503398895264,
      "eval_runtime": 70.4593,
      "eval_samples_per_second": 35.481,
      "eval_steps_per_second": 1.121,
      "step": 1500
    },
    {
      "epoch": 2.629516760992599,
      "grad_norm": 14.554814181690626,
      "learning_rate": 1.9619349931222745e-08,
      "logits/chosen": 0.1401619017124176,
      "logits/rejected": 0.16610558331012726,
      "logps/chosen": -323.71453857421875,
      "logps/rejected": -358.86737060546875,
      "loss": 0.5201,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": 1.5991357564926147,
      "rewards/margins": 5.076028347015381,
      "rewards/rejected": -3.4768924713134766,
      "step": 1510
    },
    {
      "epoch": 2.6469307792773185,
      "grad_norm": 14.072160649981779,
      "learning_rate": 1.783393639175096e-08,
      "logits/chosen": 0.2151796817779541,
      "logits/rejected": 0.20476499199867249,
      "logps/chosen": -325.95294189453125,
      "logps/rejected": -387.7116394042969,
      "loss": 0.5068,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.143306016921997,
      "rewards/margins": 6.754826545715332,
      "rewards/rejected": -4.611520290374756,
      "step": 1520
    },
    {
      "epoch": 2.6643447975620376,
      "grad_norm": 13.955290549780965,
      "learning_rate": 1.6130681396123807e-08,
      "logits/chosen": 0.21071608364582062,
      "logits/rejected": 0.3641330599784851,
      "logps/chosen": -315.24664306640625,
      "logps/rejected": -334.0530090332031,
      "loss": 0.5079,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": 2.3173837661743164,
      "rewards/margins": 7.015236854553223,
      "rewards/rejected": -4.69785213470459,
      "step": 1530
    },
    {
      "epoch": 2.6817588158467567,
      "grad_norm": 11.853297635416354,
      "learning_rate": 1.4510187689411146e-08,
      "logits/chosen": 0.2876327931880951,
      "logits/rejected": 0.2770300507545471,
      "logps/chosen": -317.94451904296875,
      "logps/rejected": -363.93353271484375,
      "loss": 0.5135,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 2.1870007514953613,
      "rewards/margins": 6.509056091308594,
      "rewards/rejected": -4.322054386138916,
      "step": 1540
    },
    {
      "epoch": 2.699172834131476,
      "grad_norm": 12.475974805623585,
      "learning_rate": 1.2973028729254376e-08,
      "logits/chosen": 0.12307579815387726,
      "logits/rejected": 0.2281501293182373,
      "logps/chosen": -334.509033203125,
      "logps/rejected": -369.52850341796875,
      "loss": 0.5532,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 1.6136415004730225,
      "rewards/margins": 5.368951320648193,
      "rewards/rejected": -3.75531005859375,
      "step": 1550
    },
    {
      "epoch": 2.716586852416195,
      "grad_norm": 12.22576416710605,
      "learning_rate": 1.1519748482932217e-08,
      "logits/chosen": 0.1852325052022934,
      "logits/rejected": 0.23839490115642548,
      "logps/chosen": -318.0132751464844,
      "logps/rejected": -392.6969299316406,
      "loss": 0.5038,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": 2.2853362560272217,
      "rewards/margins": 7.032902717590332,
      "rewards/rejected": -4.747566223144531,
      "step": 1560
    },
    {
      "epoch": 2.734000870700914,
      "grad_norm": 10.409074294198579,
      "learning_rate": 1.0150861234862362e-08,
      "logits/chosen": 0.207319974899292,
      "logits/rejected": 0.3117375075817108,
      "logps/chosen": -320.4088439941406,
      "logps/rejected": -359.54144287109375,
      "loss": 0.5016,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 1.6186834573745728,
      "rewards/margins": 6.262956142425537,
      "rewards/rejected": -4.644272804260254,
      "step": 1570
    },
    {
      "epoch": 2.7514148889856336,
      "grad_norm": 13.421082903045352,
      "learning_rate": 8.866851404607412e-09,
      "logits/chosen": 0.19717201590538025,
      "logits/rejected": 0.323054701089859,
      "logps/chosen": -331.4278259277344,
      "logps/rejected": -357.0315856933594,
      "loss": 0.5102,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 1.7053760290145874,
      "rewards/margins": 6.334009647369385,
      "rewards/rejected": -4.628633499145508,
      "step": 1580
    },
    {
      "epoch": 2.7688289072703527,
      "grad_norm": 11.2379650890193,
      "learning_rate": 7.668173375449338e-09,
      "logits/chosen": 0.2212512046098709,
      "logits/rejected": 0.34725892543792725,
      "logps/chosen": -327.44915771484375,
      "logps/rejected": -375.58099365234375,
      "loss": 0.5008,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": 2.510315418243408,
      "rewards/margins": 6.8775482177734375,
      "rewards/rejected": -4.3672332763671875,
      "step": 1590
    },
    {
      "epoch": 2.786242925555072,
      "grad_norm": 13.18885343490197,
      "learning_rate": 6.555251333592998e-09,
      "logits/chosen": 0.17324404418468475,
      "logits/rejected": 0.2584208548069,
      "logps/chosen": -314.3653259277344,
      "logps/rejected": -364.072021484375,
      "loss": 0.5246,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 2.691312074661255,
      "rewards/margins": 6.843153953552246,
      "rewards/rejected": -4.15184211730957,
      "step": 1600
    },
    {
      "epoch": 2.786242925555072,
      "eval_logits/chosen": 0.18249349296092987,
      "eval_logits/rejected": 0.2394995093345642,
      "eval_logps/chosen": -337.3701477050781,
      "eval_logps/rejected": -386.60931396484375,
      "eval_loss": 0.5809482336044312,
      "eval_rewards/accuracies": 0.7879746556282043,
      "eval_rewards/chosen": 1.8621777296066284,
      "eval_rewards/margins": 5.874190807342529,
      "eval_rewards/rejected": -4.012012958526611,
      "eval_runtime": 70.7757,
      "eval_samples_per_second": 35.323,
      "eval_steps_per_second": 1.116,
      "step": 1600
    },
    {
      "epoch": 2.803656943839791,
      "grad_norm": 11.423877289010216,
      "learning_rate": 5.5284791180557626e-09,
      "logits/chosen": 0.13084809482097626,
      "logits/rejected": 0.2546931207180023,
      "logps/chosen": -317.41326904296875,
      "logps/rejected": -364.1874084472656,
      "loss": 0.4989,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": 1.877049207687378,
      "rewards/margins": 6.358148097991943,
      "rewards/rejected": -4.481098175048828,
      "step": 1610
    },
    {
      "epoch": 2.8210709621245105,
      "grad_norm": 12.979992159418385,
      "learning_rate": 4.5882200812965846e-09,
      "logits/chosen": 0.05934108421206474,
      "logits/rejected": 0.15001478791236877,
      "logps/chosen": -352.37542724609375,
      "logps/rejected": -369.80731201171875,
      "loss": 0.5187,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 2.666846752166748,
      "rewards/margins": 6.001718997955322,
      "rewards/rejected": -3.3348724842071533,
      "step": 1620
    },
    {
      "epoch": 2.8384849804092296,
      "grad_norm": 10.84919385210753,
      "learning_rate": 3.734806960633274e-09,
      "logits/chosen": 0.2253892421722412,
      "logits/rejected": 0.22172415256500244,
      "logps/chosen": -297.4983825683594,
      "logps/rejected": -369.169189453125,
      "loss": 0.5103,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": 3.0143134593963623,
      "rewards/margins": 7.2586469650268555,
      "rewards/rejected": -4.2443342208862305,
      "step": 1630
    },
    {
      "epoch": 2.8558989986939487,
      "grad_norm": 12.859015837238639,
      "learning_rate": 2.9685417604940755e-09,
      "logits/chosen": 0.07296539843082428,
      "logits/rejected": 0.22939100861549377,
      "logps/chosen": -346.80975341796875,
      "logps/rejected": -376.8129577636719,
      "loss": 0.5063,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": 2.122498035430908,
      "rewards/margins": 5.796047210693359,
      "rewards/rejected": -3.6735496520996094,
      "step": 1640
    },
    {
      "epoch": 2.873313016978668,
      "grad_norm": 12.979116784826523,
      "learning_rate": 2.289695645544709e-09,
      "logits/chosen": 0.16871683299541473,
      "logits/rejected": 0.23157720267772675,
      "logps/chosen": -316.1363830566406,
      "logps/rejected": -375.68621826171875,
      "loss": 0.511,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 2.305443525314331,
      "rewards/margins": 6.389838218688965,
      "rewards/rejected": -4.0843939781188965,
      "step": 1650
    },
    {
      "epoch": 2.890727035263387,
      "grad_norm": 11.980615628068511,
      "learning_rate": 1.698508844729274e-09,
      "logits/chosen": 0.18221810460090637,
      "logits/rejected": 0.2593774199485779,
      "logps/chosen": -307.32135009765625,
      "logps/rejected": -360.2170104980469,
      "loss": 0.4977,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 2.1851983070373535,
      "rewards/margins": 6.605923652648926,
      "rewards/rejected": -4.4207258224487305,
      "step": 1660
    },
    {
      "epoch": 2.908141053548106,
      "grad_norm": 14.609213847266954,
      "learning_rate": 1.195190566258164e-09,
      "logits/chosen": 0.17112967371940613,
      "logits/rejected": 0.22373953461647034,
      "logps/chosen": -311.63116455078125,
      "logps/rejected": -366.12811279296875,
      "loss": 0.5065,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": 2.0072693824768066,
      "rewards/margins": 6.717558860778809,
      "rewards/rejected": -4.71028995513916,
      "step": 1670
    },
    {
      "epoch": 2.925555071832825,
      "grad_norm": 14.416615274869756,
      "learning_rate": 7.799189235740133e-10,
      "logits/chosen": 0.292160302400589,
      "logits/rejected": 0.35216864943504333,
      "logps/chosen": -307.4648132324219,
      "logps/rejected": -359.35198974609375,
      "loss": 0.4928,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": 2.0985865592956543,
      "rewards/margins": 6.522464752197266,
      "rewards/rejected": -4.423877716064453,
      "step": 1680
    },
    {
      "epoch": 2.9429690901175447,
      "grad_norm": 11.727002920198004,
      "learning_rate": 4.5284087232114186e-10,
      "logits/chosen": 0.14393924176692963,
      "logits/rejected": 0.29668131470680237,
      "logps/chosen": -344.20501708984375,
      "logps/rejected": -383.1951599121094,
      "loss": 0.51,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": 1.8797314167022705,
      "rewards/margins": 6.819934844970703,
      "rewards/rejected": -4.940203666687012,
      "step": 1690
    },
    {
      "epoch": 2.960383108402264,
      "grad_norm": 12.672419053954089,
      "learning_rate": 2.1407215834112756e-10,
      "logits/chosen": 0.2254418432712555,
      "logits/rejected": 0.3040105700492859,
      "logps/chosen": -312.3700256347656,
      "logps/rejected": -358.2984619140625,
      "loss": 0.5042,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": 2.4021451473236084,
      "rewards/margins": 6.556065559387207,
      "rewards/rejected": -4.153920650482178,
      "step": 1700
    },
    {
      "epoch": 2.960383108402264,
      "eval_logits/chosen": 0.1785387396812439,
      "eval_logits/rejected": 0.23552976548671722,
      "eval_logps/chosen": -337.86688232421875,
      "eval_logps/rejected": -387.3112487792969,
      "eval_loss": 0.5807977914810181,
      "eval_rewards/accuracies": 0.7879746556282043,
      "eval_rewards/chosen": 1.8125056028366089,
      "eval_rewards/margins": 5.89471435546875,
      "eval_rewards/rejected": -4.08220911026001,
      "eval_runtime": 70.4219,
      "eval_samples_per_second": 35.5,
      "eval_steps_per_second": 1.122,
      "step": 1700
    },
    {
      "epoch": 2.977797126686983,
      "grad_norm": 15.591584256119575,
      "learning_rate": 6.369727671279301e-11,
      "logits/chosen": 0.2616572380065918,
      "logits/rejected": 0.33414459228515625,
      "logps/chosen": -307.9795837402344,
      "logps/rejected": -374.9503479003906,
      "loss": 0.4879,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 2.143859386444092,
      "rewards/margins": 6.652590274810791,
      "rewards/rejected": -4.508730888366699,
      "step": 1710
    },
    {
      "epoch": 2.9952111449717025,
      "grad_norm": 11.823556322241616,
      "learning_rate": 1.7694418513736209e-12,
      "logits/chosen": 0.21972191333770752,
      "logits/rejected": 0.25740760564804077,
      "logps/chosen": -297.4487609863281,
      "logps/rejected": -355.6648864746094,
      "loss": 0.4947,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": 2.5511362552642822,
      "rewards/margins": 6.9618659019470215,
      "rewards/rejected": -4.410729885101318,
      "step": 1720
    },
    {
      "epoch": 2.998693948628646,
      "step": 1722,
      "total_flos": 0.0,
      "train_loss": 0.5855818307081304,
      "train_runtime": 16735.6732,
      "train_samples_per_second": 13.174,
      "train_steps_per_second": 0.103
    }
  ],
  "logging_steps": 10,
  "max_steps": 1722,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 300,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}