{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9994767137624281,
  "eval_steps": 100,
  "global_step": 955,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "grad_norm": 56.53550588900101,
      "learning_rate": 5.208333333333333e-09,
      "logits/chosen": -1.4956517219543457,
      "logits/rejected": -1.5874533653259277,
      "logps/chosen": -261.6839294433594,
      "logps/rejected": -67.61537170410156,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "grad_norm": 58.535794970458376,
      "learning_rate": 5.208333333333333e-08,
      "logits/chosen": -1.8972623348236084,
      "logits/rejected": -1.7543258666992188,
      "logps/chosen": -322.7024230957031,
      "logps/rejected": -84.92481994628906,
      "loss": 0.6929,
      "rewards/accuracies": 0.5347222089767456,
      "rewards/chosen": 0.0009713808540254831,
      "rewards/margins": 0.0015221635112538934,
      "rewards/rejected": -0.0005507825990207493,
      "step": 10
    },
    {
      "epoch": 0.02,
      "grad_norm": 55.33889749767693,
      "learning_rate": 1.0416666666666667e-07,
      "logits/chosen": -1.9332126379013062,
      "logits/rejected": -1.7341406345367432,
      "logps/chosen": -321.18292236328125,
      "logps/rejected": -77.37178802490234,
      "loss": 0.6829,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": 0.013588125817477703,
      "rewards/margins": 0.020762193948030472,
      "rewards/rejected": -0.007174068596214056,
      "step": 20
    },
    {
      "epoch": 0.03,
      "grad_norm": 44.16167777782168,
      "learning_rate": 1.5624999999999999e-07,
      "logits/chosen": -1.9293384552001953,
      "logits/rejected": -1.7555296421051025,
      "logps/chosen": -310.2709045410156,
      "logps/rejected": -78.48434448242188,
      "loss": 0.6291,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.08381162583827972,
      "rewards/margins": 0.13813219964504242,
      "rewards/rejected": -0.05432058125734329,
      "step": 30
    },
    {
      "epoch": 0.04,
      "grad_norm": 24.18459770303762,
      "learning_rate": 2.0833333333333333e-07,
      "logits/chosen": -2.0400230884552,
      "logits/rejected": -1.9282751083374023,
      "logps/chosen": -271.0721740722656,
      "logps/rejected": -90.02369689941406,
      "loss": 0.5386,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": 0.12494243681430817,
      "rewards/margins": 0.3297843337059021,
      "rewards/rejected": -0.20484189689159393,
      "step": 40
    },
    {
      "epoch": 0.05,
      "grad_norm": 12.866003675141569,
      "learning_rate": 2.604166666666667e-07,
      "logits/chosen": -2.2248549461364746,
      "logits/rejected": -2.144918918609619,
      "logps/chosen": -286.402099609375,
      "logps/rejected": -132.4581298828125,
      "loss": 0.4557,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.10796310007572174,
      "rewards/margins": 0.6674192547798157,
      "rewards/rejected": -0.5594561100006104,
      "step": 50
    },
    {
      "epoch": 0.06,
      "grad_norm": 9.99207087250347,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": -2.3598127365112305,
      "logits/rejected": -2.3264596462249756,
      "logps/chosen": -266.46722412109375,
      "logps/rejected": -148.8068084716797,
      "loss": 0.4259,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.10700075328350067,
      "rewards/margins": 0.7022608518600464,
      "rewards/rejected": -0.8092615008354187,
      "step": 60
    },
    {
      "epoch": 0.07,
      "grad_norm": 10.748569520410587,
      "learning_rate": 3.645833333333333e-07,
      "logits/chosen": -2.430502414703369,
      "logits/rejected": -2.365962266921997,
      "logps/chosen": -304.8414611816406,
      "logps/rejected": -193.85157775878906,
      "loss": 0.3815,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.26300927996635437,
      "rewards/margins": 0.9299274682998657,
      "rewards/rejected": -1.192936897277832,
      "step": 70
    },
    {
      "epoch": 0.08,
      "grad_norm": 17.211902503001124,
      "learning_rate": 4.1666666666666667e-07,
      "logits/chosen": -2.4501707553863525,
      "logits/rejected": -2.4652693271636963,
      "logps/chosen": -322.5343933105469,
      "logps/rejected": -228.8905792236328,
      "loss": 0.3641,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.4048892557621002,
      "rewards/margins": 1.1651568412780762,
      "rewards/rejected": -1.570046067237854,
      "step": 80
    },
    {
      "epoch": 0.09,
      "grad_norm": 31.451743856711428,
      "learning_rate": 4.6874999999999996e-07,
      "logits/chosen": -2.4756340980529785,
      "logits/rejected": -2.466632843017578,
      "logps/chosen": -361.0640563964844,
      "logps/rejected": -314.8011169433594,
      "loss": 0.307,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.6497557759284973,
      "rewards/margins": 1.716461181640625,
      "rewards/rejected": -2.3662171363830566,
      "step": 90
    },
    {
      "epoch": 0.1,
      "grad_norm": 35.636553442388696,
      "learning_rate": 4.999732492681437e-07,
      "logits/chosen": -2.333749294281006,
      "logits/rejected": -2.305887222290039,
      "logps/chosen": -341.64007568359375,
      "logps/rejected": -323.59344482421875,
      "loss": 0.2876,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8234262466430664,
      "rewards/margins": 1.7204338312149048,
      "rewards/rejected": -2.5438601970672607,
      "step": 100
    },
    {
      "epoch": 0.1,
      "eval_logits/chosen": -2.396522045135498,
      "eval_logits/rejected": -2.355853796005249,
      "eval_logps/chosen": -391.6133728027344,
      "eval_logps/rejected": -394.628662109375,
      "eval_loss": 0.8317008018493652,
      "eval_rewards/accuracies": 0.48828125,
      "eval_rewards/chosen": -1.099798321723938,
      "eval_rewards/margins": -0.031127024441957474,
      "eval_rewards/rejected": -1.0686713457107544,
      "eval_runtime": 98.2432,
      "eval_samples_per_second": 20.358,
      "eval_steps_per_second": 0.326,
      "step": 100
    },
    {
      "epoch": 0.12,
      "grad_norm": 23.831429692864965,
      "learning_rate": 4.996723692767926e-07,
      "logits/chosen": -2.3073012828826904,
      "logits/rejected": -2.196600914001465,
      "logps/chosen": -412.70318603515625,
      "logps/rejected": -392.77587890625,
      "loss": 0.2541,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.8505666851997375,
      "rewards/margins": 2.337313175201416,
      "rewards/rejected": -3.1878795623779297,
      "step": 110
    },
    {
      "epoch": 0.13,
      "grad_norm": 43.23249988279924,
      "learning_rate": 4.990375746213598e-07,
      "logits/chosen": -1.9642112255096436,
      "logits/rejected": -1.8769893646240234,
      "logps/chosen": -414.23870849609375,
      "logps/rejected": -447.10711669921875,
      "loss": 0.2446,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.2714389562606812,
      "rewards/margins": 2.4800686836242676,
      "rewards/rejected": -3.75150728225708,
      "step": 120
    },
    {
      "epoch": 0.14,
      "grad_norm": 23.683233547305676,
      "learning_rate": 4.980697142834314e-07,
      "logits/chosen": -1.82988703250885,
      "logits/rejected": -1.7500931024551392,
      "logps/chosen": -408.89654541015625,
      "logps/rejected": -423.0914611816406,
      "loss": 0.221,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.1055103540420532,
      "rewards/margins": 2.3427820205688477,
      "rewards/rejected": -3.4482922554016113,
      "step": 130
    },
    {
      "epoch": 0.15,
      "grad_norm": 34.7318142014321,
      "learning_rate": 4.967700826904229e-07,
      "logits/chosen": -1.6275895833969116,
      "logits/rejected": -1.5105640888214111,
      "logps/chosen": -413.52874755859375,
      "logps/rejected": -470.81646728515625,
      "loss": 0.213,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.2829525470733643,
      "rewards/margins": 2.6193902492523193,
      "rewards/rejected": -3.9023425579071045,
      "step": 140
    },
    {
      "epoch": 0.16,
      "grad_norm": 35.47160662372655,
      "learning_rate": 4.951404179843962e-07,
      "logits/chosen": -1.420513391494751,
      "logits/rejected": -1.204770565032959,
      "logps/chosen": -490.7432556152344,
      "logps/rejected": -578.1682739257812,
      "loss": 0.1881,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.6627490520477295,
      "rewards/margins": 3.3605008125305176,
      "rewards/rejected": -5.023249626159668,
      "step": 150
    },
    {
      "epoch": 0.17,
      "grad_norm": 24.235384935764635,
      "learning_rate": 4.931828996974498e-07,
      "logits/chosen": -1.1433038711547852,
      "logits/rejected": -0.8378010988235474,
      "logps/chosen": -403.712646484375,
      "logps/rejected": -517.12646484375,
      "loss": 0.1985,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.234602689743042,
      "rewards/margins": 3.2191390991210938,
      "rewards/rejected": -4.453742027282715,
      "step": 160
    },
    {
      "epoch": 0.18,
      "grad_norm": 32.26038885090466,
      "learning_rate": 4.909001458367866e-07,
      "logits/chosen": -0.7779145240783691,
      "logits/rejected": -0.8182721138000488,
      "logps/chosen": -424.05731201171875,
      "logps/rejected": -523.2410278320312,
      "loss": 0.1927,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.3222427368164062,
      "rewards/margins": 3.1079933643341064,
      "rewards/rejected": -4.430236339569092,
      "step": 170
    },
    {
      "epoch": 0.19,
      "grad_norm": 21.698446424421586,
      "learning_rate": 4.882952093833627e-07,
      "logits/chosen": -0.5563231706619263,
      "logits/rejected": -0.019981075078248978,
      "logps/chosen": -553.7864990234375,
      "logps/rejected": -680.9860229492188,
      "loss": 0.1779,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.200310230255127,
      "rewards/margins": 3.8623929023742676,
      "rewards/rejected": -6.0627031326293945,
      "step": 180
    },
    {
      "epoch": 0.2,
      "grad_norm": 25.057103619592738,
      "learning_rate": 4.853715742087946e-07,
      "logits/chosen": -0.2753412425518036,
      "logits/rejected": 0.2752535939216614,
      "logps/chosen": -462.932861328125,
      "logps/rejected": -551.9318237304688,
      "loss": 0.1827,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.5860226154327393,
      "rewards/margins": 3.1569104194641113,
      "rewards/rejected": -4.7429327964782715,
      "step": 190
    },
    {
      "epoch": 0.21,
      "grad_norm": 23.58218236831597,
      "learning_rate": 4.821331504159906e-07,
      "logits/chosen": -0.08929546922445297,
      "logits/rejected": 0.7550326585769653,
      "logps/chosen": -471.25146484375,
      "logps/rejected": -613.5535278320312,
      "loss": 0.1728,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8531732559204102,
      "rewards/margins": 3.5373101234436035,
      "rewards/rejected": -5.3904829025268555,
      "step": 200
    },
    {
      "epoch": 0.21,
      "eval_logits/chosen": -0.2344323694705963,
      "eval_logits/rejected": -0.12693485617637634,
      "eval_logps/chosen": -464.6779479980469,
      "eval_logps/rejected": -471.8403015136719,
      "eval_loss": 1.0232110023498535,
      "eval_rewards/accuracies": 0.4765625,
      "eval_rewards/chosen": -1.830443263053894,
      "eval_rewards/margins": 0.010343983769416809,
      "eval_rewards/rejected": -1.8407872915267944,
      "eval_runtime": 98.156,
      "eval_samples_per_second": 20.376,
      "eval_steps_per_second": 0.326,
      "step": 200
    },
    {
      "epoch": 0.22,
      "grad_norm": 16.601378177571135,
      "learning_rate": 4.785842691097342e-07,
      "logits/chosen": 0.09680459648370743,
      "logits/rejected": 0.48119276762008667,
      "logps/chosen": -502.2862243652344,
      "logps/rejected": -638.9625244140625,
      "loss": 0.1815,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.149169445037842,
      "rewards/margins": 3.4517273902893066,
      "rewards/rejected": -5.600897312164307,
      "step": 210
    },
    {
      "epoch": 0.23,
      "grad_norm": 21.947482623546673,
      "learning_rate": 4.7472967660421603e-07,
      "logits/chosen": -0.02034606970846653,
      "logits/rejected": 0.8195778131484985,
      "logps/chosen": -490.51446533203125,
      "logps/rejected": -611.6964111328125,
      "loss": 0.1811,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.6580333709716797,
      "rewards/margins": 3.685889482498169,
      "rewards/rejected": -5.343923091888428,
      "step": 220
    },
    {
      "epoch": 0.24,
      "grad_norm": 16.016947563393067,
      "learning_rate": 4.705745280752585e-07,
      "logits/chosen": -0.4292992651462555,
      "logits/rejected": 0.49736231565475464,
      "logps/chosen": -421.0140075683594,
      "logps/rejected": -553.5887451171875,
      "loss": 0.1643,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2240885496139526,
      "rewards/margins": 3.5786375999450684,
      "rewards/rejected": -4.802725791931152,
      "step": 230
    },
    {
      "epoch": 0.25,
      "grad_norm": 30.96422971774161,
      "learning_rate": 4.6612438066572555e-07,
      "logits/chosen": -0.5708068609237671,
      "logits/rejected": -0.08234528452157974,
      "logps/chosen": -484.21844482421875,
      "logps/rejected": -636.4466552734375,
      "loss": 0.166,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.6084845066070557,
      "rewards/margins": 3.941319227218628,
      "rewards/rejected": -5.549803733825684,
      "step": 240
    },
    {
      "epoch": 0.26,
      "grad_norm": 20.36609072126483,
      "learning_rate": 4.6138518605333664e-07,
      "logits/chosen": -0.4158596992492676,
      "logits/rejected": 0.18731757998466492,
      "logps/chosen": -477.925537109375,
      "logps/rejected": -642.4118041992188,
      "loss": 0.1704,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.7353404760360718,
      "rewards/margins": 3.9690043926239014,
      "rewards/rejected": -5.704344749450684,
      "step": 250
    },
    {
      "epoch": 0.27,
      "grad_norm": 22.407290348323052,
      "learning_rate": 4.5636328249082514e-07,
      "logits/chosen": -0.11003784090280533,
      "logits/rejected": 0.4759120047092438,
      "logps/chosen": -487.62554931640625,
      "logps/rejected": -681.8943481445312,
      "loss": 0.1625,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.7713251113891602,
      "rewards/margins": 4.262612819671631,
      "rewards/rejected": -6.033938407897949,
      "step": 260
    },
    {
      "epoch": 0.28,
      "grad_norm": 22.861972667003773,
      "learning_rate": 4.510653863290871e-07,
      "logits/chosen": -0.15707261860370636,
      "logits/rejected": 0.6316308379173279,
      "logps/chosen": -509.5392150878906,
      "logps/rejected": -689.3701171875,
      "loss": 0.1488,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.851678490638733,
      "rewards/margins": 4.221504211425781,
      "rewards/rejected": -6.073182106018066,
      "step": 270
    },
    {
      "epoch": 0.29,
      "grad_norm": 28.362740265104886,
      "learning_rate": 4.4549858303465737e-07,
      "logits/chosen": -0.3572691082954407,
      "logits/rejected": 0.02738502062857151,
      "logps/chosen": -535.75244140625,
      "logps/rejected": -780.0845947265625,
      "loss": 0.1544,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.334204912185669,
      "rewards/margins": 4.5474934577941895,
      "rewards/rejected": -6.8816986083984375,
      "step": 280
    },
    {
      "epoch": 0.3,
      "grad_norm": 19.170163185011805,
      "learning_rate": 4.396703177135261e-07,
      "logits/chosen": -0.4469999372959137,
      "logits/rejected": 0.3809048533439636,
      "logps/chosen": -505.42718505859375,
      "logps/rejected": -677.1552734375,
      "loss": 0.148,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.815896987915039,
      "rewards/margins": 4.110238075256348,
      "rewards/rejected": -5.926135063171387,
      "step": 290
    },
    {
      "epoch": 0.31,
      "grad_norm": 25.4750421565511,
      "learning_rate": 4.335883851539693e-07,
      "logits/chosen": -0.324177086353302,
      "logits/rejected": 0.4039342403411865,
      "logps/chosen": -515.0848388671875,
      "logps/rejected": -673.1795043945312,
      "loss": 0.1485,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.131357431411743,
      "rewards/margins": 3.8231749534606934,
      "rewards/rejected": -5.954532146453857,
      "step": 300
    },
    {
      "epoch": 0.31,
      "eval_logits/chosen": -0.33198875188827515,
      "eval_logits/rejected": -0.2138945460319519,
      "eval_logps/chosen": -506.083984375,
      "eval_logps/rejected": -508.1474609375,
      "eval_loss": 1.1085234880447388,
      "eval_rewards/accuracies": 0.48828125,
      "eval_rewards/chosen": -2.244504451751709,
      "eval_rewards/margins": -0.0406457856297493,
      "eval_rewards/rejected": -2.2038588523864746,
      "eval_runtime": 98.205,
      "eval_samples_per_second": 20.366,
      "eval_steps_per_second": 0.326,
      "step": 300
    },
    {
      "epoch": 0.32,
      "grad_norm": 16.440227559566175,
      "learning_rate": 4.272609194017105e-07,
      "logits/chosen": -0.4933062493801117,
      "logits/rejected": 0.30621373653411865,
      "logps/chosen": -519.3504638671875,
      "logps/rejected": -700.8064575195312,
      "loss": 0.1381,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.7165651321411133,
      "rewards/margins": 4.468541145324707,
      "rewards/rejected": -6.185105323791504,
      "step": 310
    },
    {
      "epoch": 0.33,
      "grad_norm": 19.302709909633332,
      "learning_rate": 4.2069638288135547e-07,
      "logits/chosen": -0.23755446076393127,
      "logits/rejected": 0.3369477391242981,
      "logps/chosen": -563.7846069335938,
      "logps/rejected": -759.4010009765625,
      "loss": 0.137,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.260817050933838,
      "rewards/margins": 4.473932266235352,
      "rewards/rejected": -6.734748840332031,
      "step": 320
    },
    {
      "epoch": 0.35,
      "grad_norm": 24.014682889119992,
      "learning_rate": 4.139035550786494e-07,
      "logits/chosen": 0.16438080370426178,
      "logits/rejected": 0.933226466178894,
      "logps/chosen": -518.5066528320312,
      "logps/rejected": -685.0315551757812,
      "loss": 0.1353,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -2.001511573791504,
      "rewards/margins": 4.065824508666992,
      "rewards/rejected": -6.067336559295654,
      "step": 330
    },
    {
      "epoch": 0.36,
      "grad_norm": 28.562636603741762,
      "learning_rate": 4.0689152079869306e-07,
      "logits/chosen": -0.4411384165287018,
      "logits/rejected": 0.541918933391571,
      "logps/chosen": -530.517333984375,
      "logps/rejected": -772.8568725585938,
      "loss": 0.1297,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.727979302406311,
      "rewards/margins": 5.220716953277588,
      "rewards/rejected": -6.948697090148926,
      "step": 340
    },
    {
      "epoch": 0.37,
      "grad_norm": 27.161000273751988,
      "learning_rate": 3.99669658015821e-07,
      "logits/chosen": -0.5287201404571533,
      "logits/rejected": 0.0556589737534523,
      "logps/chosen": -524.2675170898438,
      "logps/rejected": -735.3182373046875,
      "loss": 0.1433,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.134932041168213,
      "rewards/margins": 4.41909646987915,
      "rewards/rejected": -6.554028511047363,
      "step": 350
    },
    {
      "epoch": 0.38,
      "grad_norm": 17.23934546251037,
      "learning_rate": 3.92247625331392e-07,
      "logits/chosen": -0.21263980865478516,
      "logits/rejected": 0.10697118192911148,
      "logps/chosen": -474.26617431640625,
      "logps/rejected": -681.5820922851562,
      "loss": 0.1267,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.0363478660583496,
      "rewards/margins": 4.044428825378418,
      "rewards/rejected": -6.080777168273926,
      "step": 360
    },
    {
      "epoch": 0.39,
      "grad_norm": 19.030144049954593,
      "learning_rate": 3.846353490562664e-07,
      "logits/chosen": -0.14467313885688782,
      "logits/rejected": 0.4623333513736725,
      "logps/chosen": -475.9998474121094,
      "logps/rejected": -659.7310791015625,
      "loss": 0.1378,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.9292904138565063,
      "rewards/margins": 3.9880707263946533,
      "rewards/rejected": -5.917361259460449,
      "step": 370
    },
    {
      "epoch": 0.4,
      "grad_norm": 15.941291713197039,
      "learning_rate": 3.768430099352445e-07,
      "logits/chosen": -0.30350250005722046,
      "logits/rejected": 0.09275643527507782,
      "logps/chosen": -473.51934814453125,
      "logps/rejected": -694.0122680664062,
      "loss": 0.1323,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.9375219345092773,
      "rewards/margins": 4.174939155578613,
      "rewards/rejected": -6.112461090087891,
      "step": 380
    },
    {
      "epoch": 0.41,
      "grad_norm": 20.594348598442483,
      "learning_rate": 3.6888102953122304e-07,
      "logits/chosen": -0.44062256813049316,
      "logits/rejected": 0.3265061378479004,
      "logps/chosen": -475.5601501464844,
      "logps/rejected": -694.0905151367188,
      "loss": 0.132,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.7453269958496094,
      "rewards/margins": 4.402843475341797,
      "rewards/rejected": -6.148170471191406,
      "step": 390
    },
    {
      "epoch": 0.42,
      "grad_norm": 26.139898527689652,
      "learning_rate": 3.607600562872785e-07,
      "logits/chosen": -0.23806443810462952,
      "logits/rejected": 0.7257732152938843,
      "logps/chosen": -520.6710205078125,
      "logps/rejected": -760.2214965820312,
      "loss": 0.1363,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.165508985519409,
      "rewards/margins": 4.602757453918457,
      "rewards/rejected": -6.768266201019287,
      "step": 400
    },
    {
      "epoch": 0.42,
      "eval_logits/chosen": -0.29007014632225037,
      "eval_logits/rejected": -0.17282728850841522,
      "eval_logps/chosen": -477.35302734375,
      "eval_logps/rejected": -486.5422058105469,
      "eval_loss": 1.161571979522705,
      "eval_rewards/accuracies": 0.49609375,
      "eval_rewards/chosen": -1.9571943283081055,
      "eval_rewards/margins": 0.030611902475357056,
      "eval_rewards/rejected": -1.9878063201904297,
      "eval_runtime": 98.1714,
      "eval_samples_per_second": 20.373,
      "eval_steps_per_second": 0.326,
      "step": 400
    },
    {
      "epoch": 0.43,
      "grad_norm": 27.719873706295672,
      "learning_rate": 3.5249095128531856e-07,
      "logits/chosen": -0.22032999992370605,
      "logits/rejected": 0.5274828672409058,
      "logps/chosen": -466.26702880859375,
      "logps/rejected": -728.7267456054688,
      "loss": 0.1355,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.8140884637832642,
      "rewards/margins": 4.6839799880981445,
      "rewards/rejected": -6.498068332672119,
      "step": 410
    },
    {
      "epoch": 0.44,
      "grad_norm": 21.173795643942725,
      "learning_rate": 3.4408477372034736e-07,
      "logits/chosen": -0.1383340060710907,
      "logits/rejected": 0.8013898730278015,
      "logps/chosen": -500.862548828125,
      "logps/rejected": -744.8663330078125,
      "loss": 0.1315,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -2.3077805042266846,
      "rewards/margins": 4.435772895812988,
      "rewards/rejected": -6.743553161621094,
      "step": 420
    },
    {
      "epoch": 0.45,
      "grad_norm": 15.065368811496283,
      "learning_rate": 3.3555276610977276e-07,
      "logits/chosen": 0.07787096500396729,
      "logits/rejected": 1.0413458347320557,
      "logps/chosen": -519.1903686523438,
      "logps/rejected": -721.7469482421875,
      "loss": 0.1303,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.1075587272644043,
      "rewards/margins": 4.277621269226074,
      "rewards/rejected": -6.385180473327637,
      "step": 430
    },
    {
      "epoch": 0.46,
      "grad_norm": 21.359284608507693,
      "learning_rate": 3.269063392575352e-07,
      "logits/chosen": 0.20308120548725128,
      "logits/rejected": 0.9388936161994934,
      "logps/chosen": -507.998046875,
      "logps/rejected": -747.5,
      "loss": 0.1341,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.9351876974105835,
      "rewards/margins": 4.7113494873046875,
      "rewards/rejected": -6.646537780761719,
      "step": 440
    },
    {
      "epoch": 0.47,
      "grad_norm": 27.318228969564636,
      "learning_rate": 3.1815705699316964e-07,
      "logits/chosen": 0.6579626202583313,
      "logits/rejected": 1.765021562576294,
      "logps/chosen": -553.9263305664062,
      "logps/rejected": -844.2288208007812,
      "loss": 0.1277,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.427884578704834,
      "rewards/margins": 5.219451904296875,
      "rewards/rejected": -7.647336483001709,
      "step": 450
    },
    {
      "epoch": 0.48,
      "grad_norm": 27.95431686036955,
      "learning_rate": 3.0931662070620794e-07,
      "logits/chosen": -0.021429266780614853,
      "logits/rejected": 0.7255157232284546,
      "logps/chosen": -526.6499633789062,
      "logps/rejected": -762.1712646484375,
      "loss": 0.1379,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.8736591339111328,
      "rewards/margins": 4.832096576690674,
      "rewards/rejected": -6.705755710601807,
      "step": 460
    },
    {
      "epoch": 0.49,
      "grad_norm": 21.248985584336232,
      "learning_rate": 3.003968536966078e-07,
      "logits/chosen": 0.6433358788490295,
      "logits/rejected": 1.6070924997329712,
      "logps/chosen": -555.550048828125,
      "logps/rejected": -811.1224365234375,
      "loss": 0.1362,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.4847140312194824,
      "rewards/margins": 4.877633571624756,
      "rewards/rejected": -7.3623480796813965,
      "step": 470
    },
    {
      "epoch": 0.5,
      "grad_norm": 27.164827660138915,
      "learning_rate": 2.9140968536213693e-07,
      "logits/chosen": 0.8196287155151367,
      "logits/rejected": 1.9903045892715454,
      "logps/chosen": -546.5592651367188,
      "logps/rejected": -858.4177856445312,
      "loss": 0.126,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -2.6066067218780518,
      "rewards/margins": 5.207320213317871,
      "rewards/rejected": -7.813926696777344,
      "step": 480
    },
    {
      "epoch": 0.51,
      "grad_norm": 32.32085610980076,
      "learning_rate": 2.823671352438608e-07,
      "logits/chosen": 0.9727161526679993,
      "logits/rejected": 2.0250372886657715,
      "logps/chosen": -659.5581665039062,
      "logps/rejected": -1049.233154296875,
      "loss": 0.1095,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.477785110473633,
      "rewards/margins": 6.155587673187256,
      "rewards/rejected": -9.63337230682373,
      "step": 490
    },
    {
      "epoch": 0.52,
      "grad_norm": 20.87674341195024,
      "learning_rate": 2.73281296951072e-07,
      "logits/chosen": 0.9520805478096008,
      "logits/rejected": 2.0586464405059814,
      "logps/chosen": -609.9530029296875,
      "logps/rejected": -901.1349487304688,
      "loss": 0.1192,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.0585007667541504,
      "rewards/margins": 5.11794376373291,
      "rewards/rejected": -8.176445007324219,
      "step": 500
    },
    {
      "epoch": 0.52,
      "eval_logits/chosen": 0.8076502680778503,
      "eval_logits/rejected": 0.882068932056427,
      "eval_logps/chosen": -553.1240234375,
      "eval_logps/rejected": -562.3369750976562,
      "eval_loss": 1.260213851928711,
      "eval_rewards/accuracies": 0.49609375,
      "eval_rewards/chosen": -2.714904308319092,
      "eval_rewards/margins": 0.0308496356010437,
      "eval_rewards/rejected": -2.7457542419433594,
      "eval_runtime": 98.1675,
      "eval_samples_per_second": 20.373,
      "eval_steps_per_second": 0.326,
      "step": 500
    },
    {
      "epoch": 0.53,
      "grad_norm": 18.767151683387205,
      "learning_rate": 2.641643219871597e-07,
      "logits/chosen": 0.7849302887916565,
      "logits/rejected": 2.1215097904205322,
      "logps/chosen": -577.2518920898438,
      "logps/rejected": -889.7449340820312,
      "loss": 0.1317,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.842534303665161,
      "rewards/margins": 5.340787887573242,
      "rewards/rejected": -8.183321952819824,
      "step": 510
    },
    {
      "epoch": 0.54,
      "grad_norm": 29.448844346722687,
      "learning_rate": 2.550284034980507e-07,
      "logits/chosen": 1.155196189880371,
      "logits/rejected": 1.8559916019439697,
      "logps/chosen": -594.45556640625,
      "logps/rejected": -878.7010498046875,
      "loss": 0.1109,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.863124370574951,
      "rewards/margins": 5.095701217651367,
      "rewards/rejected": -7.95882511138916,
      "step": 520
    },
    {
      "epoch": 0.55,
      "grad_norm": 27.252220894010925,
      "learning_rate": 2.4588575996495794e-07,
      "logits/chosen": 1.095752239227295,
      "logits/rejected": 1.7183897495269775,
      "logps/chosen": -542.5738525390625,
      "logps/rejected": -803.3109130859375,
      "loss": 0.1195,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.78588604927063,
      "rewards/margins": 4.481093883514404,
      "rewards/rejected": -7.266980171203613,
      "step": 530
    },
    {
      "epoch": 0.57,
      "grad_norm": 22.12136172682442,
      "learning_rate": 2.367486188632446e-07,
      "logits/chosen": 0.4685417711734772,
      "logits/rejected": 2.220308542251587,
      "logps/chosen": -570.0831298828125,
      "logps/rejected": -881.7498779296875,
      "loss": 0.1043,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.479081630706787,
      "rewards/margins": 5.567392349243164,
      "rewards/rejected": -8.046473503112793,
      "step": 540
    },
    {
      "epoch": 0.58,
      "grad_norm": 19.108838505785222,
      "learning_rate": 2.276292003092593e-07,
      "logits/chosen": 1.0669015645980835,
      "logits/rejected": 2.3918092250823975,
      "logps/chosen": -626.561279296875,
      "logps/rejected": -949.5797729492188,
      "loss": 0.103,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.1381304264068604,
      "rewards/margins": 5.550665855407715,
      "rewards/rejected": -8.688796043395996,
      "step": 550
    },
    {
      "epoch": 0.59,
      "grad_norm": 32.68766498683882,
      "learning_rate": 2.185397007170141e-07,
      "logits/chosen": 1.3143640756607056,
      "logits/rejected": 2.8337502479553223,
      "logps/chosen": -665.8894653320312,
      "logps/rejected": -1035.4178466796875,
      "loss": 0.0986,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -3.870831251144409,
      "rewards/margins": 5.7238664627075195,
      "rewards/rejected": -9.594697952270508,
      "step": 560
    },
    {
      "epoch": 0.6,
      "grad_norm": 16.198596144621373,
      "learning_rate": 2.094922764865619e-07,
      "logits/chosen": 1.3269495964050293,
      "logits/rejected": 2.8318161964416504,
      "logps/chosen": -717.6715087890625,
      "logps/rejected": -1159.4893798828125,
      "loss": 0.1079,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -4.280415058135986,
      "rewards/margins": 6.474959373474121,
      "rewards/rejected": -10.755374908447266,
      "step": 570
    },
    {
      "epoch": 0.61,
      "grad_norm": 20.71654704841871,
      "learning_rate": 2.0049902774588797e-07,
      "logits/chosen": 0.6866815090179443,
      "logits/rejected": 2.683020830154419,
      "logps/chosen": -692.4168701171875,
      "logps/rejected": -1045.0552978515625,
      "loss": 0.1081,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.6754519939422607,
      "rewards/margins": 6.0012431144714355,
      "rewards/rejected": -9.676694869995117,
      "step": 580
    },
    {
      "epoch": 0.62,
      "grad_norm": 23.433263062431486,
      "learning_rate": 1.9157198216806238e-07,
      "logits/chosen": 1.8187183141708374,
      "logits/rejected": 2.8524560928344727,
      "logps/chosen": -665.5374755859375,
      "logps/rejected": -994.6904296875,
      "loss": 0.1248,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -4.142047882080078,
      "rewards/margins": 5.089578151702881,
      "rewards/rejected": -9.231626510620117,
      "step": 590
    },
    {
      "epoch": 0.63,
      "grad_norm": 25.576809503577486,
      "learning_rate": 1.8272307888529274e-07,
      "logits/chosen": 1.7792863845825195,
      "logits/rejected": 3.1064600944519043,
      "logps/chosen": -614.6641235351562,
      "logps/rejected": -954.5841064453125,
      "loss": 0.1061,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.437767744064331,
      "rewards/margins": 5.413891315460205,
      "rewards/rejected": -8.851658821105957,
      "step": 600
    },
    {
      "epoch": 0.63,
      "eval_logits/chosen": 1.3499289751052856,
      "eval_logits/rejected": 1.4624172449111938,
      "eval_logps/chosen": -636.7334594726562,
      "eval_logps/rejected": -655.7739868164062,
      "eval_loss": 1.3569632768630981,
      "eval_rewards/accuracies": 0.5078125,
      "eval_rewards/chosen": -3.5509989261627197,
      "eval_rewards/margins": 0.12912467122077942,
      "eval_rewards/rejected": -3.6801235675811768,
      "eval_runtime": 98.2105,
      "eval_samples_per_second": 20.364,
      "eval_steps_per_second": 0.326,
      "step": 600
    },
    {
      "epoch": 0.64,
      "grad_norm": 24.852031957562648,
      "learning_rate": 1.7396415252139288e-07,
      "logits/chosen": 2.000277042388916,
      "logits/rejected": 3.6395816802978516,
      "logps/chosen": -704.3046875,
      "logps/rejected": -1072.0576171875,
      "loss": 0.1011,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -4.077065467834473,
      "rewards/margins": 5.8812713623046875,
      "rewards/rejected": -9.958337783813477,
      "step": 610
    },
    {
      "epoch": 0.65,
      "grad_norm": 28.21709504471574,
      "learning_rate": 1.6530691736402316e-07,
      "logits/chosen": 1.2065070867538452,
      "logits/rejected": 2.8331146240234375,
      "logps/chosen": -668.9984130859375,
      "logps/rejected": -1089.423583984375,
      "loss": 0.1278,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.658174991607666,
      "rewards/margins": 6.437078952789307,
      "rewards/rejected": -10.095253944396973,
      "step": 620
    },
    {
      "epoch": 0.66,
      "grad_norm": 23.63694930913145,
      "learning_rate": 1.5676295169786864e-07,
      "logits/chosen": 0.5527520775794983,
      "logits/rejected": 2.127023696899414,
      "logps/chosen": -621.4510498046875,
      "logps/rejected": -956.1905517578125,
      "loss": 0.0982,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.0273334980010986,
      "rewards/margins": 5.792135715484619,
      "rewards/rejected": -8.819470405578613,
      "step": 630
    },
    {
      "epoch": 0.67,
      "grad_norm": 30.792947251941303,
      "learning_rate": 1.483436823197092e-07,
      "logits/chosen": 1.4910873174667358,
      "logits/rejected": 2.7201919555664062,
      "logps/chosen": -658.2792358398438,
      "logps/rejected": -1054.40625,
      "loss": 0.0943,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.9800808429718018,
      "rewards/margins": 5.860440731048584,
      "rewards/rejected": -9.840522766113281,
      "step": 640
    },
    {
      "epoch": 0.68,
      "grad_norm": 30.835634235164253,
      "learning_rate": 1.4006036925609243e-07,
      "logits/chosen": 1.4370182752609253,
      "logits/rejected": 2.603792428970337,
      "logps/chosen": -682.9066772460938,
      "logps/rejected": -1069.502685546875,
      "loss": 0.1096,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -3.886139392852783,
      "rewards/margins": 6.004186630249023,
      "rewards/rejected": -9.890324592590332,
      "step": 650
    },
    {
      "epoch": 0.69,
      "grad_norm": 24.03119617965728,
      "learning_rate": 1.319240907040458e-07,
      "logits/chosen": 1.332709550857544,
      "logits/rejected": 2.975593090057373,
      "logps/chosen": -689.5634155273438,
      "logps/rejected": -1074.7861328125,
      "loss": 0.1036,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.8198745250701904,
      "rewards/margins": 6.1887993812561035,
      "rewards/rejected": -10.008673667907715,
      "step": 660
    },
    {
      "epoch": 0.7,
      "grad_norm": 23.254329026411572,
      "learning_rate": 1.239457282149695e-07,
      "logits/chosen": 1.9267222881317139,
      "logits/rejected": 3.357774257659912,
      "logps/chosen": -649.8111572265625,
      "logps/rejected": -1027.4486083984375,
      "loss": 0.1018,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.9152133464813232,
      "rewards/margins": 5.612493991851807,
      "rewards/rejected": -9.527708053588867,
      "step": 670
    },
    {
      "epoch": 0.71,
      "grad_norm": 24.156755650166915,
      "learning_rate": 1.1613595214152711e-07,
      "logits/chosen": 1.7098283767700195,
      "logits/rejected": 2.766923189163208,
      "logps/chosen": -640.9730224609375,
      "logps/rejected": -1028.6197509765625,
      "loss": 0.1117,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -3.781348466873169,
      "rewards/margins": 5.773517608642578,
      "rewards/rejected": -9.554865837097168,
      "step": 680
    },
    {
      "epoch": 0.72,
      "grad_norm": 24.122068893750104,
      "learning_rate": 1.0850520736699362e-07,
      "logits/chosen": 1.7467336654663086,
      "logits/rejected": 2.8030543327331543,
      "logps/chosen": -693.4568481445312,
      "logps/rejected": -1148.7037353515625,
      "loss": 0.0942,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -4.0760016441345215,
      "rewards/margins": 6.649557590484619,
      "rewards/rejected": -10.72555923461914,
      "step": 690
    },
    {
      "epoch": 0.73,
      "grad_norm": 29.944632705362917,
      "learning_rate": 1.0106369933615042e-07,
      "logits/chosen": 1.8298194408416748,
      "logits/rejected": 2.8535187244415283,
      "logps/chosen": -790.9088134765625,
      "logps/rejected": -1210.2066650390625,
      "loss": 0.0916,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -4.785300254821777,
      "rewards/margins": 6.468996524810791,
      "rewards/rejected": -11.254297256469727,
      "step": 700
    },
    {
      "epoch": 0.73,
      "eval_logits/chosen": 1.724426507949829,
      "eval_logits/rejected": 1.836971640586853,
      "eval_logps/chosen": -780.9122314453125,
      "eval_logps/rejected": -803.1144409179688,
      "eval_loss": 1.5923107862472534,
      "eval_rewards/accuracies": 0.51953125,
      "eval_rewards/chosen": -4.992786407470703,
      "eval_rewards/margins": 0.16074250638484955,
      "eval_rewards/rejected": -5.153529167175293,
      "eval_runtime": 98.1228,
      "eval_samples_per_second": 20.383,
      "eval_steps_per_second": 0.326,
      "step": 700
    },
    {
      "epoch": 0.74,
      "grad_norm": 20.908723209277646,
      "learning_rate": 9.382138040640714e-08,
      "logits/chosen": 1.1189043521881104,
      "logits/rejected": 2.7733755111694336,
      "logps/chosen": -770.1724853515625,
      "logps/rejected": -1226.529052734375,
      "loss": 0.0963,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -4.310009956359863,
      "rewards/margins": 7.130062103271484,
      "rewards/rejected": -11.440072059631348,
      "step": 710
    },
    {
      "epoch": 0.75,
      "grad_norm": 36.31436756476285,
      "learning_rate": 8.678793653740632e-08,
      "logits/chosen": 0.8306254148483276,
      "logits/rejected": 2.2698352336883545,
      "logps/chosen": -698.108642578125,
      "logps/rejected": -1135.697509765625,
      "loss": 0.098,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.6820595264434814,
      "rewards/margins": 6.853481292724609,
      "rewards/rejected": -10.535540580749512,
      "step": 720
    },
    {
      "epoch": 0.76,
      "grad_norm": 31.693149214366326,
      "learning_rate": 7.997277433690983e-08,
      "logits/chosen": 1.5684382915496826,
      "logits/rejected": 2.5911812782287598,
      "logps/chosen": -716.12255859375,
      "logps/rejected": -1081.2698974609375,
      "loss": 0.1075,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.1696271896362305,
      "rewards/margins": 5.879700660705566,
      "rewards/rejected": -10.049327850341797,
      "step": 730
    },
    {
      "epoch": 0.77,
      "grad_norm": 22.44050706466778,
      "learning_rate": 7.338500848029602e-08,
      "logits/chosen": 1.5790059566497803,
      "logits/rejected": 2.5449037551879883,
      "logps/chosen": -735.7557373046875,
      "logps/rejected": -1158.8265380859375,
      "loss": 0.0943,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -4.639529705047607,
      "rewards/margins": 6.151622295379639,
      "rewards/rejected": -10.791152954101562,
      "step": 740
    },
    {
      "epoch": 0.78,
      "grad_norm": 39.0366898564141,
      "learning_rate": 6.70334495204884e-08,
      "logits/chosen": 1.2770452499389648,
      "logits/rejected": 3.155823230743408,
      "logps/chosen": -781.6085205078125,
      "logps/rejected": -1216.711181640625,
      "loss": 0.1008,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -4.443601131439209,
      "rewards/margins": 6.983972072601318,
      "rewards/rejected": -11.427572250366211,
      "step": 750
    },
    {
      "epoch": 0.8,
      "grad_norm": 15.124341176010748,
      "learning_rate": 6.092659210462231e-08,
      "logits/chosen": 2.077390670776367,
      "logits/rejected": 3.1247870922088623,
      "logps/chosen": -768.5572509765625,
      "logps/rejected": -1171.698974609375,
      "loss": 0.1056,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -4.7178215980529785,
      "rewards/margins": 6.16521692276001,
      "rewards/rejected": -10.883038520812988,
      "step": 760
    },
    {
      "epoch": 0.81,
      "grad_norm": 17.976947974252425,
      "learning_rate": 5.507260361320737e-08,
      "logits/chosen": 1.3104530572891235,
      "logits/rejected": 2.271193027496338,
      "logps/chosen": -656.1422119140625,
      "logps/rejected": -1040.9273681640625,
      "loss": 0.0942,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.6499016284942627,
      "rewards/margins": 5.8217973709106445,
      "rewards/rejected": -9.471699714660645,
      "step": 770
    },
    {
      "epoch": 0.82,
      "grad_norm": 22.98428455590702,
      "learning_rate": 4.947931323697982e-08,
      "logits/chosen": 1.5574134588241577,
      "logits/rejected": 2.6767842769622803,
      "logps/chosen": -622.7532348632812,
      "logps/rejected": -987.5330200195312,
      "loss": 0.0929,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.838610887527466,
      "rewards/margins": 5.3154802322387695,
      "rewards/rejected": -9.154090881347656,
      "step": 780
    },
    {
      "epoch": 0.83,
      "grad_norm": 18.290673124323764,
      "learning_rate": 4.415420150605398e-08,
      "logits/chosen": 1.273377776145935,
      "logits/rejected": 2.5009658336639404,
      "logps/chosen": -672.8284301757812,
      "logps/rejected": -1044.4967041015625,
      "loss": 0.0833,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.731667995452881,
      "rewards/margins": 5.8987932205200195,
      "rewards/rejected": -9.630460739135742,
      "step": 790
    },
    {
      "epoch": 0.84,
      "grad_norm": 21.047297751815922,
      "learning_rate": 3.9104390285376374e-08,
      "logits/chosen": 1.6440566778182983,
      "logits/rejected": 2.9946322441101074,
      "logps/chosen": -687.345703125,
      "logps/rejected": -1127.282958984375,
      "loss": 0.0982,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.8111820220947266,
      "rewards/margins": 6.623629570007324,
      "rewards/rejected": -10.43481159210205,
      "step": 800
    },
    {
      "epoch": 0.84,
      "eval_logits/chosen": 1.495996356010437,
      "eval_logits/rejected": 1.588535189628601,
      "eval_logps/chosen": -701.4559936523438,
      "eval_logps/rejected": -722.2228393554688,
      "eval_loss": 1.4366517066955566,
      "eval_rewards/accuracies": 0.51171875,
      "eval_rewards/chosen": -4.1982245445251465,
      "eval_rewards/margins": 0.14638793468475342,
      "eval_rewards/rejected": -4.344612121582031,
      "eval_runtime": 98.1092,
      "eval_samples_per_second": 20.385,
      "eval_steps_per_second": 0.326,
      "step": 800
    },
    {
      "epoch": 0.85,
      "grad_norm": 19.966317538177393,
      "learning_rate": 3.433663324986208e-08,
      "logits/chosen": 1.5778945684432983,
      "logits/rejected": 2.4630589485168457,
      "logps/chosen": -719.8331298828125,
      "logps/rejected": -1128.375244140625,
      "loss": 0.1004,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -4.297072410583496,
      "rewards/margins": 6.129397392272949,
      "rewards/rejected": -10.426468849182129,
      "step": 810
    },
    {
      "epoch": 0.86,
      "grad_norm": 46.332744181157814,
      "learning_rate": 2.9857306851953897e-08,
      "logits/chosen": 1.6526291370391846,
      "logits/rejected": 3.1333537101745605,
      "logps/chosen": -712.4828491210938,
      "logps/rejected": -1150.342529296875,
      "loss": 0.0934,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -4.247818470001221,
      "rewards/margins": 6.466667175292969,
      "rewards/rejected": -10.714487075805664,
      "step": 820
    },
    {
      "epoch": 0.87,
      "grad_norm": 16.905869837081713,
      "learning_rate": 2.567240179368185e-08,
      "logits/chosen": 1.6341140270233154,
      "logits/rejected": 3.0696451663970947,
      "logps/chosen": -696.9953002929688,
      "logps/rejected": -1112.6153564453125,
      "loss": 0.0849,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.9459125995635986,
      "rewards/margins": 6.415045261383057,
      "rewards/rejected": -10.360957145690918,
      "step": 830
    },
    {
      "epoch": 0.88,
      "grad_norm": 25.997036073481123,
      "learning_rate": 2.1787515014630357e-08,
      "logits/chosen": 1.8683927059173584,
      "logits/rejected": 3.2859091758728027,
      "logps/chosen": -663.4827880859375,
      "logps/rejected": -1147.6854248046875,
      "loss": 0.0948,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -3.9030983448028564,
      "rewards/margins": 6.834665775299072,
      "rewards/rejected": -10.737763404846191,
      "step": 840
    },
    {
      "epoch": 0.89,
      "grad_norm": 32.49864502294839,
      "learning_rate": 1.820784220652766e-08,
      "logits/chosen": 1.6341012716293335,
      "logits/rejected": 2.78373384475708,
      "logps/chosen": -746.4293212890625,
      "logps/rejected": -1182.134765625,
      "loss": 0.0973,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -4.279716968536377,
      "rewards/margins": 6.648747444152832,
      "rewards/rejected": -10.928464889526367,
      "step": 850
    },
    {
      "epoch": 0.9,
      "grad_norm": 25.41406034785188,
      "learning_rate": 1.4938170864468636e-08,
      "logits/chosen": 1.3301960229873657,
      "logits/rejected": 2.9591262340545654,
      "logps/chosen": -742.6720581054688,
      "logps/rejected": -1147.730712890625,
      "loss": 0.0923,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -4.631007671356201,
      "rewards/margins": 6.093567848205566,
      "rewards/rejected": -10.724576950073242,
      "step": 860
    },
    {
      "epoch": 0.91,
      "grad_norm": 26.508225180148386,
      "learning_rate": 1.1982873884064465e-08,
      "logits/chosen": 1.877558946609497,
      "logits/rejected": 3.2897047996520996,
      "logps/chosen": -687.8853149414062,
      "logps/rejected": -1140.5772705078125,
      "loss": 0.0903,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -4.041706085205078,
      "rewards/margins": 6.566611289978027,
      "rewards/rejected": -10.608316421508789,
      "step": 870
    },
    {
      "epoch": 0.92,
      "grad_norm": 22.694696529949244,
      "learning_rate": 9.345903713082304e-09,
      "logits/chosen": 1.3334351778030396,
      "logits/rejected": 2.8423912525177,
      "logps/chosen": -728.7959594726562,
      "logps/rejected": -1168.208251953125,
      "loss": 0.0822,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -4.355628490447998,
      "rewards/margins": 6.554152488708496,
      "rewards/rejected": -10.909782409667969,
      "step": 880
    },
    {
      "epoch": 0.93,
      "grad_norm": 22.50666317342384,
      "learning_rate": 7.030787065396865e-09,
      "logits/chosen": 1.6333587169647217,
      "logits/rejected": 3.205723285675049,
      "logps/chosen": -725.650390625,
      "logps/rejected": -1133.517333984375,
      "loss": 0.0988,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -4.209347724914551,
      "rewards/margins": 6.3392205238342285,
      "rewards/rejected": -10.548568725585938,
      "step": 890
    },
    {
      "epoch": 0.94,
      "grad_norm": 26.84118963589271,
      "learning_rate": 5.04062020432286e-09,
      "logits/chosen": 1.553541898727417,
      "logits/rejected": 2.5439181327819824,
      "logps/chosen": -702.75341796875,
      "logps/rejected": -1179.390380859375,
      "loss": 0.0798,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -4.0308685302734375,
      "rewards/margins": 6.955786228179932,
      "rewards/rejected": -10.986656188964844,
      "step": 900
    },
    {
      "epoch": 0.94,
      "eval_logits/chosen": 1.5057443380355835,
      "eval_logits/rejected": 1.5925670862197876,
      "eval_logps/chosen": -732.6488647460938,
      "eval_logps/rejected": -753.6737670898438,
      "eval_loss": 1.4971250295639038,
      "eval_rewards/accuracies": 0.515625,
      "eval_rewards/chosen": -4.510153293609619,
      "eval_rewards/margins": 0.1489681899547577,
      "eval_rewards/rejected": -4.659121513366699,
      "eval_runtime": 98.0937,
      "eval_samples_per_second": 20.389,
      "eval_steps_per_second": 0.326,
      "step": 900
    },
    {
      "epoch": 0.95,
      "grad_norm": 32.5930799681677,
      "learning_rate": 3.3780648016376866e-09,
      "logits/chosen": 1.1988734006881714,
      "logits/rejected": 2.8325839042663574,
      "logps/chosen": -706.9013671875,
      "logps/rejected": -1164.0465087890625,
      "loss": 0.0878,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -4.118927001953125,
      "rewards/margins": 6.752754211425781,
      "rewards/rejected": -10.871681213378906,
      "step": 910
    },
    {
      "epoch": 0.96,
      "grad_norm": 21.71196395147195,
      "learning_rate": 2.0453443778310766e-09,
      "logits/chosen": 1.4445321559906006,
      "logits/rejected": 3.1710243225097656,
      "logps/chosen": -704.4098510742188,
      "logps/rejected": -1139.129638671875,
      "loss": 0.0953,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -4.218876838684082,
      "rewards/margins": 6.4761552810668945,
      "rewards/rejected": -10.69503116607666,
      "step": 920
    },
    {
      "epoch": 0.97,
      "grad_norm": 38.475668210703205,
      "learning_rate": 1.0442413283435758e-09,
      "logits/chosen": 1.560880422592163,
      "logits/rejected": 2.9233765602111816,
      "logps/chosen": -711.7680053710938,
      "logps/rejected": -1192.40234375,
      "loss": 0.0893,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -4.384615421295166,
      "rewards/margins": 6.797455787658691,
      "rewards/rejected": -11.1820707321167,
      "step": 930
    },
    {
      "epoch": 0.98,
      "grad_norm": 18.03527474245485,
      "learning_rate": 3.760945397705828e-10,
      "logits/chosen": 1.3762061595916748,
      "logits/rejected": 2.6221256256103516,
      "logps/chosen": -782.9889526367188,
      "logps/rejected": -1187.587158203125,
      "loss": 0.084,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -4.29364013671875,
      "rewards/margins": 6.716446876525879,
      "rewards/rejected": -11.010087013244629,
      "step": 940
    },
    {
      "epoch": 0.99,
      "grad_norm": 29.436206242522907,
      "learning_rate": 4.17975992204056e-11,
      "logits/chosen": 1.753095030784607,
      "logits/rejected": 2.988640308380127,
      "logps/chosen": -742.6573486328125,
      "logps/rejected": -1160.2423095703125,
      "loss": 0.1017,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -4.522819519042969,
      "rewards/margins": 6.319222927093506,
      "rewards/rejected": -10.842041969299316,
      "step": 950
    },
    {
      "epoch": 1.0,
      "step": 955,
      "total_flos": 0.0,
      "train_loss": 0.04742721384732511,
      "train_runtime": 7350.5302,
      "train_samples_per_second": 16.634,
      "train_steps_per_second": 0.13
    }
  ],
  "logging_steps": 10,
  "max_steps": 955,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}