{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 1751,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "grad_norm": 20.572983553522633,
      "learning_rate": 2.8409090909090907e-09,
      "logits/chosen": -2.7078514099121094,
      "logits/rejected": -2.673313856124878,
      "logps/chosen": -322.9836120605469,
      "logps/rejected": -261.1998291015625,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "grad_norm": 19.20958843122381,
      "learning_rate": 2.8409090909090908e-08,
      "logits/chosen": -2.7531583309173584,
      "logits/rejected": -2.715681314468384,
      "logps/chosen": -219.50958251953125,
      "logps/rejected": -236.27330017089844,
      "loss": 0.6934,
      "rewards/accuracies": 0.4166666567325592,
      "rewards/chosen": -0.0001647783356020227,
      "rewards/margins": 6.51524678687565e-05,
      "rewards/rejected": -0.00022993076709099114,
      "step": 10
    },
    {
      "epoch": 0.01,
      "grad_norm": 18.221209446502318,
      "learning_rate": 5.6818181818181815e-08,
      "logits/chosen": -2.736729145050049,
      "logits/rejected": -2.7084593772888184,
      "logps/chosen": -233.8996124267578,
      "logps/rejected": -230.8451385498047,
      "loss": 0.6931,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": 0.00037134578451514244,
      "rewards/margins": 2.871244305424625e-06,
      "rewards/rejected": 0.000368474458809942,
      "step": 20
    },
    {
      "epoch": 0.02,
      "grad_norm": 19.659186304289324,
      "learning_rate": 8.522727272727271e-08,
      "logits/chosen": -2.718043804168701,
      "logits/rejected": -2.705275535583496,
      "logps/chosen": -265.68658447265625,
      "logps/rejected": -247.42025756835938,
      "loss": 0.6926,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": 0.0039044865407049656,
      "rewards/margins": 0.0005032892222516239,
      "rewards/rejected": 0.003401197027415037,
      "step": 30
    },
    {
      "epoch": 0.02,
      "grad_norm": 21.248085751648183,
      "learning_rate": 1.1363636363636363e-07,
      "logits/chosen": -2.722376585006714,
      "logits/rejected": -2.7113113403320312,
      "logps/chosen": -207.11141967773438,
      "logps/rejected": -212.8821563720703,
      "loss": 0.6916,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": 0.013140101917088032,
      "rewards/margins": 0.0030325299594551325,
      "rewards/rejected": 0.010107570327818394,
      "step": 40
    },
    {
      "epoch": 0.03,
      "grad_norm": 17.062348015914782,
      "learning_rate": 1.4204545454545455e-07,
      "logits/chosen": -2.7086739540100098,
      "logits/rejected": -2.6903626918792725,
      "logps/chosen": -230.5471954345703,
      "logps/rejected": -218.68508911132812,
      "loss": 0.6905,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": 0.030233308672904968,
      "rewards/margins": 0.0065773241221904755,
      "rewards/rejected": 0.023655982688069344,
      "step": 50
    },
    {
      "epoch": 0.03,
      "grad_norm": 16.966597240337634,
      "learning_rate": 1.7045454545454543e-07,
      "logits/chosen": -2.7187132835388184,
      "logits/rejected": -2.707646608352661,
      "logps/chosen": -231.4257354736328,
      "logps/rejected": -222.29745483398438,
      "loss": 0.6856,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.05241071432828903,
      "rewards/margins": 0.018025759607553482,
      "rewards/rejected": 0.03438495472073555,
      "step": 60
    },
    {
      "epoch": 0.04,
      "grad_norm": 19.376648396399435,
      "learning_rate": 1.9886363636363636e-07,
      "logits/chosen": -2.7627768516540527,
      "logits/rejected": -2.699197292327881,
      "logps/chosen": -230.32540893554688,
      "logps/rejected": -220.99795532226562,
      "loss": 0.6812,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.08621755987405777,
      "rewards/margins": 0.024967864155769348,
      "rewards/rejected": 0.06124969199299812,
      "step": 70
    },
    {
      "epoch": 0.05,
      "grad_norm": 16.574962507205655,
      "learning_rate": 2.2727272727272726e-07,
      "logits/chosen": -2.7578330039978027,
      "logits/rejected": -2.758152723312378,
      "logps/chosen": -241.5107879638672,
      "logps/rejected": -234.1169891357422,
      "loss": 0.6775,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.1304091513156891,
      "rewards/margins": 0.03304018825292587,
      "rewards/rejected": 0.09736897051334381,
      "step": 80
    },
    {
      "epoch": 0.05,
      "grad_norm": 17.123245271646947,
      "learning_rate": 2.5568181818181816e-07,
      "logits/chosen": -2.747803211212158,
      "logits/rejected": -2.714414119720459,
      "logps/chosen": -245.60464477539062,
      "logps/rejected": -236.7899932861328,
      "loss": 0.6688,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.12192847579717636,
      "rewards/margins": 0.04756360873579979,
      "rewards/rejected": 0.07436486333608627,
      "step": 90
    },
    {
      "epoch": 0.06,
      "grad_norm": 17.870065861056936,
      "learning_rate": 2.840909090909091e-07,
      "logits/chosen": -2.7195897102355957,
      "logits/rejected": -2.681842803955078,
      "logps/chosen": -207.11083984375,
      "logps/rejected": -211.34866333007812,
      "loss": 0.6628,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.1526772379875183,
      "rewards/margins": 0.07189060747623444,
      "rewards/rejected": 0.08078663051128387,
      "step": 100
    },
    {
      "epoch": 0.06,
      "eval_logits/chosen": -2.7152342796325684,
      "eval_logits/rejected": -2.6720974445343018,
      "eval_logps/chosen": -221.00881958007812,
      "eval_logps/rejected": -221.34713745117188,
      "eval_loss": 0.6611106395721436,
      "eval_rewards/accuracies": 0.6317307949066162,
      "eval_rewards/chosen": 0.13367876410484314,
      "eval_rewards/margins": 0.08476302027702332,
      "eval_rewards/rejected": 0.04891572147607803,
      "eval_runtime": 864.7473,
      "eval_samples_per_second": 14.398,
      "eval_steps_per_second": 0.451,
      "step": 100
    },
    {
      "epoch": 0.06,
      "grad_norm": 17.758173761785073,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": -2.745452642440796,
      "logits/rejected": -2.6345908641815186,
      "logps/chosen": -211.13497924804688,
      "logps/rejected": -245.9041290283203,
      "loss": 0.6526,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": 0.10004226118326187,
      "rewards/margins": 0.09651143848896027,
      "rewards/rejected": 0.0035308226943016052,
      "step": 110
    },
    {
      "epoch": 0.07,
      "grad_norm": 25.3503988826482,
      "learning_rate": 3.4090909090909085e-07,
      "logits/chosen": -2.6189653873443604,
      "logits/rejected": -2.57344388961792,
      "logps/chosen": -228.3636932373047,
      "logps/rejected": -230.4072265625,
      "loss": 0.6476,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.04373621195554733,
      "rewards/margins": 0.13258467614650726,
      "rewards/rejected": -0.176320880651474,
      "step": 120
    },
    {
      "epoch": 0.07,
      "grad_norm": 25.5123273990743,
      "learning_rate": 3.693181818181818e-07,
      "logits/chosen": -2.564441204071045,
      "logits/rejected": -2.547898292541504,
      "logps/chosen": -269.93511962890625,
      "logps/rejected": -285.6426086425781,
      "loss": 0.6484,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.3578580319881439,
      "rewards/margins": 0.16583284735679626,
      "rewards/rejected": -0.5236908793449402,
      "step": 130
    },
    {
      "epoch": 0.08,
      "grad_norm": 25.02924344762446,
      "learning_rate": 3.977272727272727e-07,
      "logits/chosen": -2.5508384704589844,
      "logits/rejected": -2.5335073471069336,
      "logps/chosen": -274.27508544921875,
      "logps/rejected": -272.3787536621094,
      "loss": 0.6395,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.3012983798980713,
      "rewards/margins": 0.09775562584400177,
      "rewards/rejected": -0.3990539610385895,
      "step": 140
    },
    {
      "epoch": 0.09,
      "grad_norm": 23.226996578001714,
      "learning_rate": 4.2613636363636364e-07,
      "logits/chosen": -2.5063862800598145,
      "logits/rejected": -2.4907989501953125,
      "logps/chosen": -242.54183959960938,
      "logps/rejected": -267.40216064453125,
      "loss": 0.6367,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.11261695623397827,
      "rewards/margins": 0.19061629474163055,
      "rewards/rejected": -0.30323326587677,
      "step": 150
    },
    {
      "epoch": 0.09,
      "grad_norm": 22.32459928056343,
      "learning_rate": 4.545454545454545e-07,
      "logits/chosen": -2.538865327835083,
      "logits/rejected": -2.4915454387664795,
      "logps/chosen": -259.973876953125,
      "logps/rejected": -284.99273681640625,
      "loss": 0.6342,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06355403363704681,
      "rewards/margins": 0.3071693778038025,
      "rewards/rejected": -0.3707233965396881,
      "step": 160
    },
    {
      "epoch": 0.1,
      "grad_norm": 28.238806692314206,
      "learning_rate": 4.829545454545455e-07,
      "logits/chosen": -2.5641493797302246,
      "logits/rejected": -2.5063588619232178,
      "logps/chosen": -256.0674133300781,
      "logps/rejected": -281.95965576171875,
      "loss": 0.6207,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.11485638469457626,
      "rewards/margins": 0.2587049603462219,
      "rewards/rejected": -0.3735613226890564,
      "step": 170
    },
    {
      "epoch": 0.1,
      "grad_norm": 34.83431654717648,
      "learning_rate": 4.999920426892062e-07,
      "logits/chosen": -2.6064653396606445,
      "logits/rejected": -2.600895404815674,
      "logps/chosen": -245.4825897216797,
      "logps/rejected": -257.6805725097656,
      "loss": 0.6398,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.1361248791217804,
      "rewards/margins": 0.16745266318321228,
      "rewards/rejected": -0.3035775423049927,
      "step": 180
    },
    {
      "epoch": 0.11,
      "grad_norm": 33.80086885729117,
      "learning_rate": 4.999025287600885e-07,
      "logits/chosen": -2.6627895832061768,
      "logits/rejected": -2.6619114875793457,
      "logps/chosen": -263.55096435546875,
      "logps/rejected": -284.45147705078125,
      "loss": 0.6325,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.09487941861152649,
      "rewards/margins": 0.21393874287605286,
      "rewards/rejected": -0.3088182210922241,
      "step": 190
    },
    {
      "epoch": 0.11,
      "grad_norm": 29.828595993243468,
      "learning_rate": 4.997135899956001e-07,
      "logits/chosen": -2.638547420501709,
      "logits/rejected": -2.5978102684020996,
      "logps/chosen": -245.3714599609375,
      "logps/rejected": -266.0381164550781,
      "loss": 0.6203,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.1466042697429657,
      "rewards/margins": 0.28931504487991333,
      "rewards/rejected": -0.43591928482055664,
      "step": 200
    },
    {
      "epoch": 0.11,
      "eval_logits/chosen": -2.6775360107421875,
      "eval_logits/rejected": -2.621267080307007,
      "eval_logps/chosen": -243.9757843017578,
      "eval_logps/rejected": -266.8084411621094,
      "eval_loss": 0.6120733618736267,
      "eval_rewards/accuracies": 0.660897433757782,
      "eval_rewards/chosen": -0.09599092602729797,
      "eval_rewards/margins": 0.30970630049705505,
      "eval_rewards/rejected": -0.405697226524353,
      "eval_runtime": 841.2052,
      "eval_samples_per_second": 14.801,
      "eval_steps_per_second": 0.464,
      "step": 200
    },
    {
      "epoch": 0.12,
      "grad_norm": 31.207584928179323,
      "learning_rate": 4.994253015658708e-07,
      "logits/chosen": -2.6432251930236816,
      "logits/rejected": -2.6140708923339844,
      "logps/chosen": -232.1490936279297,
      "logps/rejected": -261.6481018066406,
      "loss": 0.6111,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.03973131626844406,
      "rewards/margins": 0.33994999527931213,
      "rewards/rejected": -0.3796812891960144,
      "step": 210
    },
    {
      "epoch": 0.13,
      "grad_norm": 23.510226746553357,
      "learning_rate": 4.990377781677378e-07,
      "logits/chosen": -2.674729108810425,
      "logits/rejected": -2.621265172958374,
      "logps/chosen": -272.51715087890625,
      "logps/rejected": -285.8130187988281,
      "loss": 0.6261,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.12226327508687973,
      "rewards/margins": 0.3179982602596283,
      "rewards/rejected": -0.4402615427970886,
      "step": 220
    },
    {
      "epoch": 0.13,
      "grad_norm": 34.69513987573851,
      "learning_rate": 4.985511739791128e-07,
      "logits/chosen": -2.682669162750244,
      "logits/rejected": -2.6345534324645996,
      "logps/chosen": -243.78695678710938,
      "logps/rejected": -252.6505889892578,
      "loss": 0.5961,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.19093939661979675,
      "rewards/margins": 0.23702657222747803,
      "rewards/rejected": -0.4279659688472748,
      "step": 230
    },
    {
      "epoch": 0.14,
      "grad_norm": 33.90649114668316,
      "learning_rate": 4.979656825976425e-07,
      "logits/chosen": -2.636639356613159,
      "logits/rejected": -2.556272029876709,
      "logps/chosen": -267.24310302734375,
      "logps/rejected": -295.2035217285156,
      "loss": 0.6078,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.3182388246059418,
      "rewards/margins": 0.24964241683483124,
      "rewards/rejected": -0.5678812861442566,
      "step": 240
    },
    {
      "epoch": 0.14,
      "grad_norm": 27.80845453447812,
      "learning_rate": 4.972815369636832e-07,
      "logits/chosen": -2.6218130588531494,
      "logits/rejected": -2.55488920211792,
      "logps/chosen": -299.002197265625,
      "logps/rejected": -309.2835388183594,
      "loss": 0.6176,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.2978210151195526,
      "rewards/margins": 0.2504158914089203,
      "rewards/rejected": -0.5482369661331177,
      "step": 250
    },
    {
      "epoch": 0.15,
      "grad_norm": 28.138070133686533,
      "learning_rate": 4.964990092676262e-07,
      "logits/chosen": -2.6748335361480713,
      "logits/rejected": -2.593806028366089,
      "logps/chosen": -268.3004455566406,
      "logps/rejected": -270.9266662597656,
      "loss": 0.6223,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.1147528737783432,
      "rewards/margins": 0.2601880431175232,
      "rewards/rejected": -0.3749409317970276,
      "step": 260
    },
    {
      "epoch": 0.15,
      "grad_norm": 28.844690039420406,
      "learning_rate": 4.956184108416046e-07,
      "logits/chosen": -2.7262961864471436,
      "logits/rejected": -2.741664409637451,
      "logps/chosen": -238.9697723388672,
      "logps/rejected": -277.444091796875,
      "loss": 0.5909,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.17726224660873413,
      "rewards/margins": 0.36171460151672363,
      "rewards/rejected": -0.538976788520813,
      "step": 270
    },
    {
      "epoch": 0.16,
      "grad_norm": 34.00184014415281,
      "learning_rate": 4.946400920356287e-07,
      "logits/chosen": -2.754479169845581,
      "logits/rejected": -2.707367181777954,
      "logps/chosen": -268.70452880859375,
      "logps/rejected": -305.3086853027344,
      "loss": 0.5949,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.30848780274391174,
      "rewards/margins": 0.48010724782943726,
      "rewards/rejected": -0.7885950803756714,
      "step": 280
    },
    {
      "epoch": 0.17,
      "grad_norm": 24.974211972857105,
      "learning_rate": 4.935644420781978e-07,
      "logits/chosen": -2.6929306983947754,
      "logits/rejected": -2.6737914085388184,
      "logps/chosen": -283.9693603515625,
      "logps/rejected": -305.68646240234375,
      "loss": 0.5971,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.3218172490596771,
      "rewards/margins": 0.37671035528182983,
      "rewards/rejected": -0.6985276341438293,
      "step": 290
    },
    {
      "epoch": 0.17,
      "grad_norm": 22.031451800060307,
      "learning_rate": 4.923918889214436e-07,
      "logits/chosen": -2.8449320793151855,
      "logits/rejected": -2.802635908126831,
      "logps/chosen": -249.634033203125,
      "logps/rejected": -294.66973876953125,
      "loss": 0.6134,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.1441958099603653,
      "rewards/margins": 0.37521737813949585,
      "rewards/rejected": -0.5194131731987,
      "step": 300
    },
    {
      "epoch": 0.17,
      "eval_logits/chosen": -2.8550686836242676,
      "eval_logits/rejected": -2.79876971244812,
      "eval_logps/chosen": -240.60450744628906,
      "eval_logps/rejected": -263.5723571777344,
      "eval_loss": 0.6074427962303162,
      "eval_rewards/accuracies": 0.670192301273346,
      "eval_rewards/chosen": -0.06227818876504898,
      "eval_rewards/margins": 0.31105825304985046,
      "eval_rewards/rejected": -0.37333643436431885,
      "eval_runtime": 861.7371,
      "eval_samples_per_second": 14.449,
      "eval_steps_per_second": 0.453,
      "step": 300
    },
    {
      "epoch": 0.18,
      "grad_norm": 22.548306029706836,
      "learning_rate": 4.91122899070868e-07,
      "logits/chosen": -2.88305401802063,
      "logits/rejected": -2.8267903327941895,
      "logps/chosen": -253.9397735595703,
      "logps/rejected": -268.42498779296875,
      "loss": 0.608,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.02261662669479847,
      "rewards/margins": 0.35017159581184387,
      "rewards/rejected": -0.3727882504463196,
      "step": 310
    },
    {
      "epoch": 0.18,
      "grad_norm": 26.91606740249196,
      "learning_rate": 4.897579773997414e-07,
      "logits/chosen": -2.7912163734436035,
      "logits/rejected": -2.748124599456787,
      "logps/chosen": -260.5610046386719,
      "logps/rejected": -272.39898681640625,
      "loss": 0.6085,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.2463793307542801,
      "rewards/margins": 0.2676694393157959,
      "rewards/rejected": -0.5140487551689148,
      "step": 320
    },
    {
      "epoch": 0.19,
      "grad_norm": 25.26707067068886,
      "learning_rate": 4.882976669482367e-07,
      "logits/chosen": -2.6727445125579834,
      "logits/rejected": -2.5968549251556396,
      "logps/chosen": -253.74581909179688,
      "logps/rejected": -296.7026672363281,
      "loss": 0.6019,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.2068738043308258,
      "rewards/margins": 0.3490857481956482,
      "rewards/rejected": -0.5559597015380859,
      "step": 330
    },
    {
      "epoch": 0.19,
      "grad_norm": 35.48578116044424,
      "learning_rate": 4.867425487073786e-07,
      "logits/chosen": -2.649488925933838,
      "logits/rejected": -2.6318719387054443,
      "logps/chosen": -241.77029418945312,
      "logps/rejected": -246.7797088623047,
      "loss": 0.629,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.01744382455945015,
      "rewards/margins": 0.21225237846374512,
      "rewards/rejected": -0.22969619929790497,
      "step": 340
    },
    {
      "epoch": 0.2,
      "grad_norm": 32.561918770233014,
      "learning_rate": 4.850932413878934e-07,
      "logits/chosen": -2.5814261436462402,
      "logits/rejected": -2.5403079986572266,
      "logps/chosen": -251.0198516845703,
      "logps/rejected": -265.89190673828125,
      "loss": 0.6163,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.004257055930793285,
      "rewards/margins": 0.2787313163280487,
      "rewards/rejected": -0.28298836946487427,
      "step": 350
    },
    {
      "epoch": 0.21,
      "grad_norm": 32.76257794045328,
      "learning_rate": 4.833504011740522e-07,
      "logits/chosen": -2.493020534515381,
      "logits/rejected": -2.4786593914031982,
      "logps/chosen": -239.0045623779297,
      "logps/rejected": -267.716552734375,
      "loss": 0.5962,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.0083407461643219,
      "rewards/margins": 0.33076852560043335,
      "rewards/rejected": -0.33910924196243286,
      "step": 360
    },
    {
      "epoch": 0.21,
      "grad_norm": 25.42313532237448,
      "learning_rate": 4.815147214626056e-07,
      "logits/chosen": -2.403029680252075,
      "logits/rejected": -2.375685930252075,
      "logps/chosen": -286.8995666503906,
      "logps/rejected": -282.9924621582031,
      "loss": 0.5961,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.26247408986091614,
      "rewards/margins": 0.31958019733428955,
      "rewards/rejected": -0.5820542573928833,
      "step": 370
    },
    {
      "epoch": 0.22,
      "grad_norm": 29.80215735650837,
      "learning_rate": 4.795869325869116e-07,
      "logits/chosen": -2.416861057281494,
      "logits/rejected": -2.4126358032226562,
      "logps/chosen": -263.8486022949219,
      "logps/rejected": -291.6919250488281,
      "loss": 0.6312,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.2886821925640106,
      "rewards/margins": 0.29902181029319763,
      "rewards/rejected": -0.5877040028572083,
      "step": 380
    },
    {
      "epoch": 0.22,
      "grad_norm": 22.149160653773386,
      "learning_rate": 4.775678015263708e-07,
      "logits/chosen": -2.5736076831817627,
      "logits/rejected": -2.5460524559020996,
      "logps/chosen": -223.2324981689453,
      "logps/rejected": -245.54092407226562,
      "loss": 0.6081,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.09774526208639145,
      "rewards/margins": 0.38662397861480713,
      "rewards/rejected": -0.2888787090778351,
      "step": 390
    },
    {
      "epoch": 0.23,
      "grad_norm": 28.055639097927603,
      "learning_rate": 4.7545813160127845e-07,
      "logits/chosen": -2.6072590351104736,
      "logits/rejected": -2.505096912384033,
      "logps/chosen": -236.22549438476562,
      "logps/rejected": -265.6544494628906,
      "loss": 0.5967,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.02556512877345085,
      "rewards/margins": 0.45649051666259766,
      "rewards/rejected": -0.4309254288673401,
      "step": 400
    },
    {
      "epoch": 0.23,
      "eval_logits/chosen": -2.519050121307373,
      "eval_logits/rejected": -2.4576427936553955,
      "eval_logps/chosen": -247.5235595703125,
      "eval_logps/rejected": -278.0496826171875,
      "eval_loss": 0.5991776585578918,
      "eval_rewards/accuracies": 0.678205132484436,
      "eval_rewards/chosen": -0.1314685195684433,
      "eval_rewards/margins": 0.38664132356643677,
      "eval_rewards/rejected": -0.5181097984313965,
      "eval_runtime": 861.5357,
      "eval_samples_per_second": 14.452,
      "eval_steps_per_second": 0.453,
      "step": 400
    },
    {
      "epoch": 0.23,
      "grad_norm": 26.14630535783879,
      "learning_rate": 4.732587621532214e-07,
      "logits/chosen": -2.478102922439575,
      "logits/rejected": -2.438175678253174,
      "logps/chosen": -250.90457153320312,
      "logps/rejected": -284.3564453125,
      "loss": 0.5827,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.15595874190330505,
      "rewards/margins": 0.3831656873226166,
      "rewards/rejected": -0.5391243696212769,
      "step": 410
    },
    {
      "epoch": 0.24,
      "grad_norm": 28.031713354054272,
      "learning_rate": 4.709705682111412e-07,
      "logits/chosen": -2.477914810180664,
      "logits/rejected": -2.4634792804718018,
      "logps/chosen": -262.4942321777344,
      "logps/rejected": -290.16009521484375,
      "loss": 0.5869,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.22279219329357147,
      "rewards/margins": 0.4437895715236664,
      "rewards/rejected": -0.6665817499160767,
      "step": 420
    },
    {
      "epoch": 0.25,
      "grad_norm": 25.747403286133764,
      "learning_rate": 4.68594460143201e-07,
      "logits/chosen": -2.5019423961639404,
      "logits/rejected": -2.429969310760498,
      "logps/chosen": -256.59185791015625,
      "logps/rejected": -283.1378173828125,
      "loss": 0.5936,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.14861060678958893,
      "rewards/margins": 0.4206705093383789,
      "rewards/rejected": -0.5692810416221619,
      "step": 430
    },
    {
      "epoch": 0.25,
      "grad_norm": 22.31072442106435,
      "learning_rate": 4.661313832945903e-07,
      "logits/chosen": -2.504993438720703,
      "logits/rejected": -2.467900037765503,
      "logps/chosen": -254.93057250976562,
      "logps/rejected": -312.25250244140625,
      "loss": 0.5925,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.15858420729637146,
      "rewards/margins": 0.5125812292098999,
      "rewards/rejected": -0.671165406703949,
      "step": 440
    },
    {
      "epoch": 0.26,
      "grad_norm": 28.65488774525744,
      "learning_rate": 4.635823176114162e-07,
      "logits/chosen": -2.5073506832122803,
      "logits/rejected": -2.472411870956421,
      "logps/chosen": -284.4822998046875,
      "logps/rejected": -320.66925048828125,
      "loss": 0.5817,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.37685471773147583,
      "rewards/margins": 0.3860743045806885,
      "rewards/rejected": -0.7629289627075195,
      "step": 450
    },
    {
      "epoch": 0.26,
      "grad_norm": 34.815824611343665,
      "learning_rate": 4.6094827725082684e-07,
      "logits/chosen": -2.5682735443115234,
      "logits/rejected": -2.501211166381836,
      "logps/chosen": -265.11126708984375,
      "logps/rejected": -309.05157470703125,
      "loss": 0.5814,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.2719506323337555,
      "rewards/margins": 0.5007379055023193,
      "rewards/rejected": -0.7726885080337524,
      "step": 460
    },
    {
      "epoch": 0.27,
      "grad_norm": 26.238405483406037,
      "learning_rate": 4.582303101775248e-07,
      "logits/chosen": -2.62959623336792,
      "logits/rejected": -2.562079906463623,
      "logps/chosen": -251.88674926757812,
      "logps/rejected": -288.1053771972656,
      "loss": 0.5955,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.16994395852088928,
      "rewards/margins": 0.4775424599647522,
      "rewards/rejected": -0.6474864482879639,
      "step": 470
    },
    {
      "epoch": 0.27,
      "grad_norm": 30.50178144367043,
      "learning_rate": 4.5542949774682956e-07,
      "logits/chosen": -2.568668842315674,
      "logits/rejected": -2.5604805946350098,
      "logps/chosen": -267.90655517578125,
      "logps/rejected": -290.76019287109375,
      "loss": 0.5991,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.22079896926879883,
      "rewards/margins": 0.31227344274520874,
      "rewards/rejected": -0.5330723524093628,
      "step": 480
    },
    {
      "epoch": 0.28,
      "grad_norm": 27.535568193967393,
      "learning_rate": 4.5254695427445517e-07,
      "logits/chosen": -2.637202501296997,
      "logits/rejected": -2.6354382038116455,
      "logps/chosen": -244.74551391601562,
      "logps/rejected": -290.29644775390625,
      "loss": 0.5647,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": 0.011266368441283703,
      "rewards/margins": 0.5976118445396423,
      "rewards/rejected": -0.5863455533981323,
      "step": 490
    },
    {
      "epoch": 0.29,
      "grad_norm": 30.285379350904805,
      "learning_rate": 4.4958382659317536e-07,
      "logits/chosen": -2.740764617919922,
      "logits/rejected": -2.7210445404052734,
      "logps/chosen": -272.13702392578125,
      "logps/rejected": -297.4922790527344,
      "loss": 0.6216,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.258303165435791,
      "rewards/margins": 0.32254815101623535,
      "rewards/rejected": -0.5808513760566711,
      "step": 500
    },
    {
      "epoch": 0.29,
      "eval_logits/chosen": -2.7310774326324463,
      "eval_logits/rejected": -2.687927484512329,
      "eval_logps/chosen": -238.078125,
      "eval_logps/rejected": -267.6940002441406,
      "eval_loss": 0.5941099524497986,
      "eval_rewards/accuracies": 0.6721153855323792,
      "eval_rewards/chosen": -0.037014104425907135,
      "eval_rewards/margins": 0.37753891944885254,
      "eval_rewards/rejected": -0.4145529866218567,
      "eval_runtime": 858.9897,
      "eval_samples_per_second": 14.495,
      "eval_steps_per_second": 0.454,
      "step": 500
    },
    {
      "epoch": 0.29,
      "grad_norm": 23.512447045863215,
      "learning_rate": 4.465412935965506e-07,
      "logits/chosen": -2.7190139293670654,
      "logits/rejected": -2.675178050994873,
      "logps/chosen": -217.1962432861328,
      "logps/rejected": -246.1102294921875,
      "loss": 0.6138,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.14496248960494995,
      "rewards/margins": 0.36382409930229187,
      "rewards/rejected": -0.21886160969734192,
      "step": 510
    },
    {
      "epoch": 0.3,
      "grad_norm": 23.621464405914903,
      "learning_rate": 4.434205657698999e-07,
      "logits/chosen": -2.568403720855713,
      "logits/rejected": -2.551494836807251,
      "logps/chosen": -224.2484588623047,
      "logps/rejected": -278.65936279296875,
      "loss": 0.6125,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.11122129112482071,
      "rewards/margins": 0.36189281940460205,
      "rewards/rejected": -0.47311410307884216,
      "step": 520
    },
    {
      "epoch": 0.3,
      "grad_norm": 23.359399767763318,
      "learning_rate": 4.402228847087046e-07,
      "logits/chosen": -2.5537118911743164,
      "logits/rejected": -2.502847194671631,
      "logps/chosen": -261.56927490234375,
      "logps/rejected": -274.5636291503906,
      "loss": 0.622,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.15581832826137543,
      "rewards/margins": 0.2899227440357208,
      "rewards/rejected": -0.44574108719825745,
      "step": 530
    },
    {
      "epoch": 0.31,
      "grad_norm": 33.81353805062806,
      "learning_rate": 4.36949522624633e-07,
      "logits/chosen": -2.578953742980957,
      "logits/rejected": -2.547420024871826,
      "logps/chosen": -236.4100341796875,
      "logps/rejected": -284.703857421875,
      "loss": 0.5917,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.17350006103515625,
      "rewards/margins": 0.42752647399902344,
      "rewards/rejected": -0.6010265350341797,
      "step": 540
    },
    {
      "epoch": 0.31,
      "grad_norm": 25.836827126785362,
      "learning_rate": 4.33601781839386e-07,
      "logits/chosen": -2.516505718231201,
      "logits/rejected": -2.4714202880859375,
      "logps/chosen": -271.7455139160156,
      "logps/rejected": -320.60003662109375,
      "loss": 0.583,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.1934550553560257,
      "rewards/margins": 0.46987539529800415,
      "rewards/rejected": -0.6633304357528687,
      "step": 550
    },
    {
      "epoch": 0.32,
      "grad_norm": 29.317285148378573,
      "learning_rate": 4.301809942665625e-07,
      "logits/chosen": -2.5027713775634766,
      "logits/rejected": -2.4169793128967285,
      "logps/chosen": -247.04306030273438,
      "logps/rejected": -263.53033447265625,
      "loss": 0.6041,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.13370783627033234,
      "rewards/margins": 0.29323458671569824,
      "rewards/rejected": -0.4269424080848694,
      "step": 560
    },
    {
      "epoch": 0.33,
      "grad_norm": 23.273024353193524,
      "learning_rate": 4.2668852088175145e-07,
      "logits/chosen": -2.429138660430908,
      "logits/rejected": -2.4112179279327393,
      "logps/chosen": -261.13232421875,
      "logps/rejected": -292.0643005371094,
      "loss": 0.613,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.12648805975914001,
      "rewards/margins": 0.3239407241344452,
      "rewards/rejected": -0.4504288136959076,
      "step": 570
    },
    {
      "epoch": 0.33,
      "grad_norm": 27.396331045603723,
      "learning_rate": 4.231257511810618e-07,
      "logits/chosen": -2.55159068107605,
      "logits/rejected": -2.4876770973205566,
      "logps/chosen": -251.07192993164062,
      "logps/rejected": -293.2655334472656,
      "loss": 0.6128,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.010355165228247643,
      "rewards/margins": 0.33735883235931396,
      "rewards/rejected": -0.34771400690078735,
      "step": 580
    },
    {
      "epoch": 0.34,
      "grad_norm": 22.123964853357307,
      "learning_rate": 4.1949410262830523e-07,
      "logits/chosen": -2.5525479316711426,
      "logits/rejected": -2.4439752101898193,
      "logps/chosen": -233.11337280273438,
      "logps/rejected": -262.0377502441406,
      "loss": 0.5785,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.024647405371069908,
      "rewards/margins": 0.3654420077800751,
      "rewards/rejected": -0.3900894522666931,
      "step": 590
    },
    {
      "epoch": 0.34,
      "grad_norm": 29.777775051728057,
      "learning_rate": 4.1579502009105164e-07,
      "logits/chosen": -2.488654613494873,
      "logits/rejected": -2.4201626777648926,
      "logps/chosen": -246.08486938476562,
      "logps/rejected": -273.70709228515625,
      "loss": 0.5919,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.1482144594192505,
      "rewards/margins": 0.40206265449523926,
      "rewards/rejected": -0.5502771139144897,
      "step": 600
    },
    {
      "epoch": 0.34,
      "eval_logits/chosen": -2.47454833984375,
      "eval_logits/rejected": -2.404360294342041,
      "eval_logps/chosen": -249.46990966796875,
      "eval_logps/rejected": -283.90716552734375,
      "eval_loss": 0.5903669595718384,
      "eval_rewards/accuracies": 0.6865384578704834,
      "eval_rewards/chosen": -0.15093201398849487,
      "eval_rewards/margins": 0.42575252056121826,
      "eval_rewards/rejected": -0.5766844749450684,
      "eval_runtime": 858.5855,
      "eval_samples_per_second": 14.502,
      "eval_steps_per_second": 0.454,
      "step": 600
    },
    {
      "epoch": 0.35,
      "grad_norm": 27.691202685227417,
      "learning_rate": 4.120299752657827e-07,
      "logits/chosen": -2.5120370388031006,
      "logits/rejected": -2.4610724449157715,
      "logps/chosen": -261.5104064941406,
      "logps/rejected": -296.66217041015625,
      "loss": 0.592,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.18553677201271057,
      "rewards/margins": 0.3792334496974945,
      "rewards/rejected": -0.5647702217102051,
      "step": 610
    },
    {
      "epoch": 0.35,
      "grad_norm": 24.36661678943314,
      "learning_rate": 4.082004660923702e-07,
      "logits/chosen": -2.4631857872009277,
      "logits/rejected": -2.373511791229248,
      "logps/chosen": -248.58895874023438,
      "logps/rejected": -271.892333984375,
      "loss": 0.5945,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.06783659011125565,
      "rewards/margins": 0.3515480160713196,
      "rewards/rejected": -0.419384628534317,
      "step": 620
    },
    {
      "epoch": 0.36,
      "grad_norm": 23.60175971446507,
      "learning_rate": 4.0430801615811437e-07,
      "logits/chosen": -2.4521324634552,
      "logits/rejected": -2.4140613079071045,
      "logps/chosen": -231.697021484375,
      "logps/rejected": -277.6011962890625,
      "loss": 0.5938,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.045631490647792816,
      "rewards/margins": 0.44818124175071716,
      "rewards/rejected": -0.4938127398490906,
      "step": 630
    },
    {
      "epoch": 0.37,
      "grad_norm": 28.337765984985374,
      "learning_rate": 4.00354174091578e-07,
      "logits/chosen": -2.4540791511535645,
      "logits/rejected": -2.3845746517181396,
      "logps/chosen": -229.52133178710938,
      "logps/rejected": -277.82452392578125,
      "loss": 0.6079,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.0009828682523220778,
      "rewards/margins": 0.471419095993042,
      "rewards/rejected": -0.47240200638771057,
      "step": 640
    },
    {
      "epoch": 0.37,
      "grad_norm": 27.293175260711642,
      "learning_rate": 3.963405129464569e-07,
      "logits/chosen": -2.3416523933410645,
      "logits/rejected": -2.244310140609741,
      "logps/chosen": -261.36871337890625,
      "logps/rejected": -316.395263671875,
      "loss": 0.5519,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.22408190369606018,
      "rewards/margins": 0.5848034024238586,
      "rewards/rejected": -0.8088852167129517,
      "step": 650
    },
    {
      "epoch": 0.38,
      "grad_norm": 27.68918047728001,
      "learning_rate": 3.922686295757347e-07,
      "logits/chosen": -2.336780071258545,
      "logits/rejected": -2.2526936531066895,
      "logps/chosen": -301.7099609375,
      "logps/rejected": -356.14056396484375,
      "loss": 0.5933,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.42652687430381775,
      "rewards/margins": 0.5547691583633423,
      "rewards/rejected": -0.9812959432601929,
      "step": 660
    },
    {
      "epoch": 0.38,
      "grad_norm": 24.193971795815184,
      "learning_rate": 3.881401439963666e-07,
      "logits/chosen": -2.272087574005127,
      "logits/rejected": -2.220580577850342,
      "logps/chosen": -229.0252227783203,
      "logps/rejected": -266.34405517578125,
      "loss": 0.5979,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.23234577476978302,
      "rewards/margins": 0.3420170247554779,
      "rewards/rejected": -0.5743628144264221,
      "step": 670
    },
    {
      "epoch": 0.39,
      "grad_norm": 33.515555503395184,
      "learning_rate": 3.839566987447491e-07,
      "logits/chosen": -2.345500946044922,
      "logits/rejected": -2.2792842388153076,
      "logps/chosen": -268.81317138671875,
      "logps/rejected": -309.93463134765625,
      "loss": 0.6034,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.14895500242710114,
      "rewards/margins": 0.49111080169677734,
      "rewards/rejected": -0.6400657892227173,
      "step": 680
    },
    {
      "epoch": 0.39,
      "grad_norm": 22.584232277933243,
      "learning_rate": 3.7971995822322947e-07,
      "logits/chosen": -2.27201247215271,
      "logits/rejected": -2.226531505584717,
      "logps/chosen": -262.95782470703125,
      "logps/rejected": -290.07696533203125,
      "loss": 0.6172,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.2456875741481781,
      "rewards/margins": 0.29335761070251465,
      "rewards/rejected": -0.5390451550483704,
      "step": 690
    },
    {
      "epoch": 0.4,
      "grad_norm": 28.603695886249437,
      "learning_rate": 3.7543160803791545e-07,
      "logits/chosen": -2.301879644393921,
      "logits/rejected": -2.223996162414551,
      "logps/chosen": -261.08795166015625,
      "logps/rejected": -307.17010498046875,
      "loss": 0.5769,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.22030536830425262,
      "rewards/margins": 0.49444580078125,
      "rewards/rejected": -0.714751124382019,
      "step": 700
    },
    {
      "epoch": 0.4,
      "eval_logits/chosen": -2.2923786640167236,
      "eval_logits/rejected": -2.2189748287200928,
      "eval_logps/chosen": -258.4495849609375,
      "eval_logps/rejected": -292.7128601074219,
      "eval_loss": 0.5901564955711365,
      "eval_rewards/accuracies": 0.6772435903549194,
      "eval_rewards/chosen": -0.24072882533073425,
      "eval_rewards/margins": 0.4240129590034485,
      "eval_rewards/rejected": -0.6647418141365051,
      "eval_runtime": 860.8912,
      "eval_samples_per_second": 14.463,
      "eval_steps_per_second": 0.453,
      "step": 700
    },
    {
      "epoch": 0.41,
      "grad_norm": 26.031424540267988,
      "learning_rate": 3.7109335432805e-07,
      "logits/chosen": -2.2880516052246094,
      "logits/rejected": -2.229980707168579,
      "logps/chosen": -265.29754638671875,
      "logps/rejected": -275.8728942871094,
      "loss": 0.6037,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11812520027160645,
      "rewards/margins": 0.3968770205974579,
      "rewards/rejected": -0.5150021910667419,
      "step": 710
    },
    {
      "epoch": 0.41,
      "grad_norm": 22.719027068474563,
      "learning_rate": 3.667069230872155e-07,
      "logits/chosen": -2.254481554031372,
      "logits/rejected": -2.231606960296631,
      "logps/chosen": -259.39422607421875,
      "logps/rejected": -272.6926574707031,
      "loss": 0.5933,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.046945445239543915,
      "rewards/margins": 0.3208449184894562,
      "rewards/rejected": -0.3677903413772583,
      "step": 720
    },
    {
      "epoch": 0.42,
      "grad_norm": 29.374605264798323,
      "learning_rate": 3.6227405947664003e-07,
      "logits/chosen": -2.2828311920166016,
      "logits/rejected": -2.2138302326202393,
      "logps/chosen": -256.0699768066406,
      "logps/rejected": -295.7370910644531,
      "loss": 0.5858,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.0820874497294426,
      "rewards/margins": 0.5297425985336304,
      "rewards/rejected": -0.6118300557136536,
      "step": 730
    },
    {
      "epoch": 0.42,
      "grad_norm": 24.91144032228272,
      "learning_rate": 3.577965271308771e-07,
      "logits/chosen": -2.2585289478302,
      "logits/rejected": -2.201707124710083,
      "logps/chosen": -266.5726318359375,
      "logps/rejected": -301.1708679199219,
      "loss": 0.5891,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.2869926691055298,
      "rewards/margins": 0.4772910475730896,
      "rewards/rejected": -0.7642836570739746,
      "step": 740
    },
    {
      "epoch": 0.43,
      "grad_norm": 31.59826426014115,
      "learning_rate": 3.5327610745613546e-07,
      "logits/chosen": -2.3542895317077637,
      "logits/rejected": -2.242069721221924,
      "logps/chosen": -250.84194946289062,
      "logps/rejected": -285.4515380859375,
      "loss": 0.5963,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.19332554936408997,
      "rewards/margins": 0.4167536795139313,
      "rewards/rejected": -0.6100791692733765,
      "step": 750
    },
    {
      "epoch": 0.43,
      "grad_norm": 25.89736672152189,
      "learning_rate": 3.487145989215391e-07,
      "logits/chosen": -2.383197069168091,
      "logits/rejected": -2.3221888542175293,
      "logps/chosen": -251.72390747070312,
      "logps/rejected": -282.80804443359375,
      "loss": 0.5814,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.09259854257106781,
      "rewards/margins": 0.45896273851394653,
      "rewards/rejected": -0.5515612363815308,
      "step": 760
    },
    {
      "epoch": 0.44,
      "grad_norm": 24.879610991566807,
      "learning_rate": 3.4411381634359796e-07,
      "logits/chosen": -2.389742612838745,
      "logits/rejected": -2.3619580268859863,
      "logps/chosen": -255.8960418701172,
      "logps/rejected": -290.5157165527344,
      "loss": 0.5646,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.1797618567943573,
      "rewards/margins": 0.44017988443374634,
      "rewards/rejected": -0.6199417114257812,
      "step": 770
    },
    {
      "epoch": 0.45,
      "grad_norm": 26.915784595282116,
      "learning_rate": 3.3947559016417554e-07,
      "logits/chosen": -2.4469265937805176,
      "logits/rejected": -2.4010143280029297,
      "logps/chosen": -261.87322998046875,
      "logps/rejected": -302.0085754394531,
      "loss": 0.6045,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.26690053939819336,
      "rewards/margins": 0.33789390325546265,
      "rewards/rejected": -0.6047943830490112,
      "step": 780
    },
    {
      "epoch": 0.45,
      "grad_norm": 27.76077206811895,
      "learning_rate": 3.3480176572223885e-07,
      "logits/chosen": -2.5314059257507324,
      "logits/rejected": -2.4485652446746826,
      "logps/chosen": -238.18112182617188,
      "logps/rejected": -297.2239074707031,
      "loss": 0.554,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.199807807803154,
      "rewards/margins": 0.6575660705566406,
      "rewards/rejected": -0.857373833656311,
      "step": 790
    },
    {
      "epoch": 0.46,
      "grad_norm": 24.226296856947638,
      "learning_rate": 3.300942025196824e-07,
      "logits/chosen": -2.587006092071533,
      "logits/rejected": -2.541341781616211,
      "logps/chosen": -256.3536376953125,
      "logps/rejected": -278.5965270996094,
      "loss": 0.5725,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.13480545580387115,
      "rewards/margins": 0.4779321551322937,
      "rewards/rejected": -0.6127376556396484,
      "step": 800
    },
    {
      "epoch": 0.46,
      "eval_logits/chosen": -2.573244094848633,
      "eval_logits/rejected": -2.527627468109131,
      "eval_logps/chosen": -238.99398803710938,
      "eval_logps/rejected": -274.5382995605469,
      "eval_loss": 0.5882474780082703,
      "eval_rewards/accuracies": 0.6836538314819336,
      "eval_rewards/chosen": -0.04617273062467575,
      "eval_rewards/margins": 0.43682336807250977,
      "eval_rewards/rejected": -0.4829961061477661,
      "eval_runtime": 862.9985,
      "eval_samples_per_second": 14.428,
      "eval_steps_per_second": 0.452,
      "step": 800
    },
    {
      "epoch": 0.46,
      "grad_norm": 26.141416208141568,
      "learning_rate": 3.253547734815173e-07,
      "logits/chosen": -2.607003688812256,
      "logits/rejected": -2.554546594619751,
      "logps/chosen": -232.52743530273438,
      "logps/rejected": -314.36346435546875,
      "loss": 0.5801,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.0894065648317337,
      "rewards/margins": 0.6495140790939331,
      "rewards/rejected": -0.5601075887680054,
      "step": 810
    },
    {
      "epoch": 0.47,
      "grad_norm": 26.097129519417752,
      "learning_rate": 3.2058536421071914e-07,
      "logits/chosen": -2.5027403831481934,
      "logits/rejected": -2.471726417541504,
      "logps/chosen": -257.47320556640625,
      "logps/rejected": -263.6692810058594,
      "loss": 0.5805,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.03840960934758186,
      "rewards/margins": 0.3363339304924011,
      "rewards/rejected": -0.29792430996894836,
      "step": 820
    },
    {
      "epoch": 0.47,
      "grad_norm": 25.23275414096734,
      "learning_rate": 3.1578787223803294e-07,
      "logits/chosen": -2.426281452178955,
      "logits/rejected": -2.399183750152588,
      "logps/chosen": -272.22869873046875,
      "logps/rejected": -290.59649658203125,
      "loss": 0.6043,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.32681387662887573,
      "rewards/margins": 0.41791266202926636,
      "rewards/rejected": -0.7447265386581421,
      "step": 830
    },
    {
      "epoch": 0.48,
      "grad_norm": 24.11509059960286,
      "learning_rate": 3.109642062670318e-07,
      "logits/chosen": -2.5103323459625244,
      "logits/rejected": -2.4195475578308105,
      "logps/chosen": -244.50863647460938,
      "logps/rejected": -288.40069580078125,
      "loss": 0.5889,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.2207474410533905,
      "rewards/margins": 0.4590122699737549,
      "rewards/rejected": -0.6797597408294678,
      "step": 840
    },
    {
      "epoch": 0.49,
      "grad_norm": 33.37568508664132,
      "learning_rate": 3.0611628541473074e-07,
      "logits/chosen": -2.497424602508545,
      "logits/rejected": -2.472369432449341,
      "logps/chosen": -266.05230712890625,
      "logps/rejected": -307.9107360839844,
      "loss": 0.6051,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.008460281416773796,
      "rewards/margins": 0.4078546166419983,
      "rewards/rejected": -0.39939433336257935,
      "step": 850
    },
    {
      "epoch": 0.49,
      "grad_norm": 24.365317735651118,
      "learning_rate": 3.012460384480576e-07,
      "logits/chosen": -2.4374663829803467,
      "logits/rejected": -2.3599624633789062,
      "logps/chosen": -259.6045837402344,
      "logps/rejected": -302.10345458984375,
      "loss": 0.5678,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.18751467764377594,
      "rewards/margins": 0.49369174242019653,
      "rewards/rejected": -0.6812063455581665,
      "step": 860
    },
    {
      "epoch": 0.5,
      "grad_norm": 22.6057869532,
      "learning_rate": 2.963554030164842e-07,
      "logits/chosen": -2.2620365619659424,
      "logits/rejected": -2.280235767364502,
      "logps/chosen": -270.291748046875,
      "logps/rejected": -298.3429870605469,
      "loss": 0.6028,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.1897084265947342,
      "rewards/margins": 0.3970777988433838,
      "rewards/rejected": -0.5867862105369568,
      "step": 870
    },
    {
      "epoch": 0.5,
      "grad_norm": 26.035338300340943,
      "learning_rate": 2.9144632488112416e-07,
      "logits/chosen": -2.274094343185425,
      "logits/rejected": -2.2430529594421387,
      "logps/chosen": -270.6689758300781,
      "logps/rejected": -314.1228942871094,
      "loss": 0.5872,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.21771720051765442,
      "rewards/margins": 0.5008502006530762,
      "rewards/rejected": -0.7185673713684082,
      "step": 880
    },
    {
      "epoch": 0.51,
      "grad_norm": 22.12211938779161,
      "learning_rate": 2.865207571406029e-07,
      "logits/chosen": -2.3526949882507324,
      "logits/rejected": -2.253375768661499,
      "logps/chosen": -269.4740905761719,
      "logps/rejected": -295.1158752441406,
      "loss": 0.5956,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.25454986095428467,
      "rewards/margins": 0.40211352705955505,
      "rewards/rejected": -0.6566634774208069,
      "step": 890
    },
    {
      "epoch": 0.51,
      "grad_norm": 26.967953686731473,
      "learning_rate": 2.815806594540089e-07,
      "logits/chosen": -2.3986849784851074,
      "logits/rejected": -2.3849453926086426,
      "logps/chosen": -241.43505859375,
      "logps/rejected": -276.5702819824219,
      "loss": 0.5814,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.002503329422324896,
      "rewards/margins": 0.3857470154762268,
      "rewards/rejected": -0.38825035095214844,
      "step": 900
    },
    {
      "epoch": 0.51,
      "eval_logits/chosen": -2.409844160079956,
      "eval_logits/rejected": -2.335495948791504,
      "eval_logps/chosen": -246.1585693359375,
      "eval_logps/rejected": -279.9914245605469,
      "eval_loss": 0.5864266753196716,
      "eval_rewards/accuracies": 0.6810897588729858,
      "eval_rewards/chosen": -0.11781882494688034,
      "eval_rewards/margins": 0.41970840096473694,
      "eval_rewards/rejected": -0.5375272035598755,
      "eval_runtime": 861.4542,
      "eval_samples_per_second": 14.453,
      "eval_steps_per_second": 0.453,
      "step": 900
    },
    {
      "epoch": 0.52,
      "grad_norm": 24.93686478364852,
      "learning_rate": 2.7662799726123454e-07,
      "logits/chosen": -2.364938974380493,
      "logits/rejected": -2.3040971755981445,
      "logps/chosen": -242.66366577148438,
      "logps/rejected": -277.5296936035156,
      "loss": 0.6005,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.25096192955970764,
      "rewards/margins": 0.2948233187198639,
      "rewards/rejected": -0.5457853078842163,
      "step": 910
    },
    {
      "epoch": 0.53,
      "grad_norm": 29.741661581034126,
      "learning_rate": 2.7166474100101674e-07,
      "logits/chosen": -2.362717390060425,
      "logits/rejected": -2.29923677444458,
      "logps/chosen": -260.1564636230469,
      "logps/rejected": -299.52142333984375,
      "loss": 0.5972,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.2629412114620209,
      "rewards/margins": 0.4658204913139343,
      "rewards/rejected": -0.7287617325782776,
      "step": 920
    },
    {
      "epoch": 0.53,
      "grad_norm": 23.70048734007165,
      "learning_rate": 2.6669286532698964e-07,
      "logits/chosen": -2.4182770252227783,
      "logits/rejected": -2.377297878265381,
      "logps/chosen": -272.5760192871094,
      "logps/rejected": -307.6293640136719,
      "loss": 0.5734,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.10112947225570679,
      "rewards/margins": 0.5127297639846802,
      "rewards/rejected": -0.6138591170310974,
      "step": 930
    },
    {
      "epoch": 0.54,
      "grad_norm": 26.612920994033377,
      "learning_rate": 2.6171434832205953e-07,
      "logits/chosen": -2.4122025966644287,
      "logits/rejected": -2.3759803771972656,
      "logps/chosen": -256.7776184082031,
      "logps/rejected": -289.4403381347656,
      "loss": 0.5657,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.20392732322216034,
      "rewards/margins": 0.4127303957939148,
      "rewards/rejected": -0.6166577339172363,
      "step": 940
    },
    {
      "epoch": 0.54,
      "grad_norm": 31.14720620364731,
      "learning_rate": 2.567311707114157e-07,
      "logits/chosen": -2.4095797538757324,
      "logits/rejected": -2.354799747467041,
      "logps/chosen": -229.2027130126953,
      "logps/rejected": -283.2561340332031,
      "loss": 0.604,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.197126567363739,
      "rewards/margins": 0.4597938060760498,
      "rewards/rejected": -0.6569203734397888,
      "step": 950
    },
    {
      "epoch": 0.55,
      "grad_norm": 27.00204010664385,
      "learning_rate": 2.5174531507449037e-07,
      "logits/chosen": -2.4471428394317627,
      "logits/rejected": -2.338510513305664,
      "logps/chosen": -234.0675506591797,
      "logps/rejected": -259.7816467285156,
      "loss": 0.5899,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.190177783370018,
      "rewards/margins": 0.3870656490325928,
      "rewards/rejected": -0.577243447303772,
      "step": 960
    },
    {
      "epoch": 0.55,
      "grad_norm": 19.932243905164242,
      "learning_rate": 2.4675876505618045e-07,
      "logits/chosen": -2.4684808254241943,
      "logits/rejected": -2.4519195556640625,
      "logps/chosen": -262.174560546875,
      "logps/rejected": -282.4520263671875,
      "loss": 0.6019,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.06612774729728699,
      "rewards/margins": 0.45484066009521484,
      "rewards/rejected": -0.5209684371948242,
      "step": 970
    },
    {
      "epoch": 0.56,
      "grad_norm": 26.495466395218312,
      "learning_rate": 2.4177350457764527e-07,
      "logits/chosen": -2.53678822517395,
      "logits/rejected": -2.4713311195373535,
      "logps/chosen": -253.9574432373047,
      "logps/rejected": -298.145263671875,
      "loss": 0.6026,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.07538889348506927,
      "rewards/margins": 0.39861345291137695,
      "rewards/rejected": -0.47400230169296265,
      "step": 980
    },
    {
      "epoch": 0.57,
      "grad_norm": 24.86682957992287,
      "learning_rate": 2.367915170469956e-07,
      "logits/chosen": -2.4735522270202637,
      "logits/rejected": -2.337954044342041,
      "logps/chosen": -256.18963623046875,
      "logps/rejected": -267.07305908203125,
      "loss": 0.5731,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04039744660258293,
      "rewards/margins": 0.35975947976112366,
      "rewards/rejected": -0.4001569151878357,
      "step": 990
    },
    {
      "epoch": 0.57,
      "grad_norm": 32.9954052513008,
      "learning_rate": 2.3181478457018475e-07,
      "logits/chosen": -2.4602394104003906,
      "logits/rejected": -2.372180700302124,
      "logps/chosen": -259.4634704589844,
      "logps/rejected": -298.13836669921875,
      "loss": 0.5514,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.039340369403362274,
      "rewards/margins": 0.5396673083305359,
      "rewards/rejected": -0.5790076851844788,
      "step": 1000
    },
    {
      "epoch": 0.57,
      "eval_logits/chosen": -2.4855430126190186,
      "eval_logits/rejected": -2.411501169204712,
      "eval_logps/chosen": -252.6514892578125,
      "eval_logps/rejected": -291.29022216796875,
      "eval_loss": 0.5839190483093262,
      "eval_rewards/accuracies": 0.6871795058250427,
      "eval_rewards/chosen": -0.18274781107902527,
      "eval_rewards/margins": 0.4677676856517792,
      "eval_rewards/rejected": -0.6505153775215149,
      "eval_runtime": 844.1499,
      "eval_samples_per_second": 14.75,
      "eval_steps_per_second": 0.462,
      "step": 1000
    },
    {
      "epoch": 0.58,
      "grad_norm": 33.832396038055975,
      "learning_rate": 2.2684528716241968e-07,
      "logits/chosen": -2.528446674346924,
      "logits/rejected": -2.4585366249084473,
      "logps/chosen": -278.91461181640625,
      "logps/rejected": -327.67449951171875,
      "loss": 0.5971,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.20916755497455597,
      "rewards/margins": 0.4697844088077545,
      "rewards/rejected": -0.6789519190788269,
      "step": 1010
    },
    {
      "epoch": 0.58,
      "grad_norm": 23.559014104508705,
      "learning_rate": 2.218850019604028e-07,
      "logits/chosen": -2.5937182903289795,
      "logits/rejected": -2.506432056427002,
      "logps/chosen": -244.5588836669922,
      "logps/rejected": -312.0033264160156,
      "loss": 0.5622,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.12260422855615616,
      "rewards/margins": 0.6369132399559021,
      "rewards/rejected": -0.7595174908638,
      "step": 1020
    },
    {
      "epoch": 0.59,
      "grad_norm": 33.04728248607101,
      "learning_rate": 2.1693590243571935e-07,
      "logits/chosen": -2.641472816467285,
      "logits/rejected": -2.5865638256073,
      "logps/chosen": -279.6830139160156,
      "logps/rejected": -306.81243896484375,
      "loss": 0.5842,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.17163579165935516,
      "rewards/margins": 0.36918142437934875,
      "rewards/rejected": -0.5408172607421875,
      "step": 1030
    },
    {
      "epoch": 0.59,
      "grad_norm": 28.35991011420196,
      "learning_rate": 2.119999576096832e-07,
      "logits/chosen": -2.623781681060791,
      "logits/rejected": -2.5274269580841064,
      "logps/chosen": -278.32916259765625,
      "logps/rejected": -306.47198486328125,
      "loss": 0.5772,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.28789210319519043,
      "rewards/margins": 0.4908284544944763,
      "rewards/rejected": -0.7787205576896667,
      "step": 1040
    },
    {
      "epoch": 0.6,
      "grad_norm": 24.13700126033885,
      "learning_rate": 2.0707913126995244e-07,
      "logits/chosen": -2.555406093597412,
      "logits/rejected": -2.5062708854675293,
      "logps/chosen": -302.45416259765625,
      "logps/rejected": -314.2410583496094,
      "loss": 0.579,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.26258963346481323,
      "rewards/margins": 0.5782099962234497,
      "rewards/rejected": -0.8407996296882629,
      "step": 1050
    },
    {
      "epoch": 0.61,
      "grad_norm": 31.0299041852118,
      "learning_rate": 2.021753811892281e-07,
      "logits/chosen": -2.591373920440674,
      "logits/rejected": -2.5653815269470215,
      "logps/chosen": -267.46551513671875,
      "logps/rejected": -289.0950012207031,
      "loss": 0.5749,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.2024902105331421,
      "rewards/margins": 0.5493400692939758,
      "rewards/rejected": -0.7518302202224731,
      "step": 1060
    },
    {
      "epoch": 0.61,
      "grad_norm": 27.46997764843006,
      "learning_rate": 1.9729065834634528e-07,
      "logits/chosen": -2.5627617835998535,
      "logits/rejected": -2.4920713901519775,
      "logps/chosen": -259.22216796875,
      "logps/rejected": -298.4840393066406,
      "loss": 0.5842,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.045971281826496124,
      "rewards/margins": 0.47002631425857544,
      "rewards/rejected": -0.515997588634491,
      "step": 1070
    },
    {
      "epoch": 0.62,
      "grad_norm": 23.969955157596903,
      "learning_rate": 1.924269061500678e-07,
      "logits/chosen": -2.5113027095794678,
      "logits/rejected": -2.500351667404175,
      "logps/chosen": -237.98489379882812,
      "logps/rejected": -280.7731628417969,
      "loss": 0.5878,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.1136646643280983,
      "rewards/margins": 0.36606425046920776,
      "rewards/rejected": -0.4797288477420807,
      "step": 1080
    },
    {
      "epoch": 0.62,
      "grad_norm": 23.516344403891786,
      "learning_rate": 1.8758605966589434e-07,
      "logits/chosen": -2.5660672187805176,
      "logits/rejected": -2.5005881786346436,
      "logps/chosen": -275.2914733886719,
      "logps/rejected": -298.65155029296875,
      "loss": 0.5762,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.09098437428474426,
      "rewards/margins": 0.4184696078300476,
      "rewards/rejected": -0.32748526334762573,
      "step": 1090
    },
    {
      "epoch": 0.63,
      "grad_norm": 29.404877835754245,
      "learning_rate": 1.8277004484618357e-07,
      "logits/chosen": -2.5443501472473145,
      "logits/rejected": -2.4919300079345703,
      "logps/chosen": -245.58584594726562,
      "logps/rejected": -276.38427734375,
      "loss": 0.5946,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.019266966730356216,
      "rewards/margins": 0.3828735649585724,
      "rewards/rejected": -0.4021405279636383,
      "step": 1100
    },
    {
      "epoch": 0.63,
      "eval_logits/chosen": -2.508995532989502,
      "eval_logits/rejected": -2.447495937347412,
      "eval_logps/chosen": -241.06724548339844,
      "eval_logps/rejected": -277.4429626464844,
      "eval_loss": 0.5846326351165771,
      "eval_rewards/accuracies": 0.6846153736114502,
      "eval_rewards/chosen": -0.06690550595521927,
      "eval_rewards/margins": 0.4451369047164917,
      "eval_rewards/rejected": -0.5120424032211304,
      "eval_runtime": 860.4121,
      "eval_samples_per_second": 14.471,
      "eval_steps_per_second": 0.453,
      "step": 1100
    },
    {
      "epoch": 0.63,
      "grad_norm": 28.170030221996313,
      "learning_rate": 1.7798077776390603e-07,
      "logits/chosen": -2.497800827026367,
      "logits/rejected": -2.438910722732544,
      "logps/chosen": -266.95526123046875,
      "logps/rejected": -271.0229187011719,
      "loss": 0.5981,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.20956678688526154,
      "rewards/margins": 0.3521861732006073,
      "rewards/rejected": -0.5617530345916748,
      "step": 1110
    },
    {
      "epoch": 0.64,
      "grad_norm": 25.64342328633492,
      "learning_rate": 1.7322016385032536e-07,
      "logits/chosen": -2.4583446979522705,
      "logits/rejected": -2.400876522064209,
      "logps/chosen": -265.85760498046875,
      "logps/rejected": -305.25830078125,
      "loss": 0.5854,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.011677969247102737,
      "rewards/margins": 0.538432240486145,
      "rewards/rejected": -0.5501102209091187,
      "step": 1120
    },
    {
      "epoch": 0.65,
      "grad_norm": 29.29633909055343,
      "learning_rate": 1.6849009713691454e-07,
      "logits/chosen": -2.458902597427368,
      "logits/rejected": -2.4362378120422363,
      "logps/chosen": -273.21710205078125,
      "logps/rejected": -307.6544494628906,
      "loss": 0.5831,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.21327920258045197,
      "rewards/margins": 0.44302114844322205,
      "rewards/rejected": -0.65630042552948,
      "step": 1130
    },
    {
      "epoch": 0.65,
      "grad_norm": 29.210383152583212,
      "learning_rate": 1.6379245950180666e-07,
      "logits/chosen": -2.4900028705596924,
      "logits/rejected": -2.45780611038208,
      "logps/chosen": -270.41424560546875,
      "logps/rejected": -287.94708251953125,
      "loss": 0.5648,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.17546476423740387,
      "rewards/margins": 0.5030435919761658,
      "rewards/rejected": -0.6785083413124084,
      "step": 1140
    },
    {
      "epoch": 0.66,
      "grad_norm": 25.427016062934616,
      "learning_rate": 1.591291199210813e-07,
      "logits/chosen": -2.519883871078491,
      "logits/rejected": -2.4721555709838867,
      "logps/chosen": -239.39413452148438,
      "logps/rejected": -290.2775573730469,
      "loss": 0.5746,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.21366102993488312,
      "rewards/margins": 0.43268680572509766,
      "rewards/rejected": -0.6463478207588196,
      "step": 1150
    },
    {
      "epoch": 0.66,
      "grad_norm": 27.841026179227654,
      "learning_rate": 1.5450193372518438e-07,
      "logits/chosen": -2.5061633586883545,
      "logits/rejected": -2.446307420730591,
      "logps/chosen": -254.13290405273438,
      "logps/rejected": -282.93585205078125,
      "loss": 0.5938,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.22907939553260803,
      "rewards/margins": 0.40173858404159546,
      "rewards/rejected": -0.6308180093765259,
      "step": 1160
    },
    {
      "epoch": 0.67,
      "grad_norm": 23.462003347624545,
      "learning_rate": 1.4991274186077628e-07,
      "logits/chosen": -2.528425931930542,
      "logits/rejected": -2.4988036155700684,
      "logps/chosen": -256.7923583984375,
      "logps/rejected": -287.1715393066406,
      "loss": 0.5873,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.17279884219169617,
      "rewards/margins": 0.4650405943393707,
      "rewards/rejected": -0.6378394365310669,
      "step": 1170
    },
    {
      "epoch": 0.67,
      "grad_norm": 29.33113372811353,
      "learning_rate": 1.4536337015830323e-07,
      "logits/chosen": -2.567692279815674,
      "logits/rejected": -2.534449815750122,
      "logps/chosen": -267.5567626953125,
      "logps/rejected": -302.30535888671875,
      "loss": 0.5849,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.20980314910411835,
      "rewards/margins": 0.41117554903030396,
      "rewards/rejected": -0.6209786534309387,
      "step": 1180
    },
    {
      "epoch": 0.68,
      "grad_norm": 27.190688664117623,
      "learning_rate": 1.4085562860558253e-07,
      "logits/chosen": -2.571300506591797,
      "logits/rejected": -2.5311543941497803,
      "logps/chosen": -264.1678161621094,
      "logps/rejected": -294.2384338378906,
      "loss": 0.581,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.21912555396556854,
      "rewards/margins": 0.4142730236053467,
      "rewards/rejected": -0.6333985924720764,
      "step": 1190
    },
    {
      "epoch": 0.69,
      "grad_norm": 29.84363481456159,
      "learning_rate": 1.3639131062769075e-07,
      "logits/chosen": -2.495572805404663,
      "logits/rejected": -2.4769346714019775,
      "logps/chosen": -259.7516784667969,
      "logps/rejected": -298.76519775390625,
      "loss": 0.5988,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.23917154967784882,
      "rewards/margins": 0.4562154710292816,
      "rewards/rejected": -0.695387065410614,
      "step": 1200
    },
    {
      "epoch": 0.69,
      "eval_logits/chosen": -2.5292820930480957,
      "eval_logits/rejected": -2.470348596572876,
      "eval_logps/chosen": -261.1407775878906,
      "eval_logps/rejected": -299.3863525390625,
      "eval_loss": 0.5829187631607056,
      "eval_rewards/accuracies": 0.6891025900840759,
      "eval_rewards/chosen": -0.2676407992839813,
      "eval_rewards/margins": 0.4638356566429138,
      "eval_rewards/rejected": -0.7314765453338623,
      "eval_runtime": 847.4482,
      "eval_samples_per_second": 14.692,
      "eval_steps_per_second": 0.46,
      "step": 1200
    },
    {
      "epoch": 0.69,
      "grad_norm": 28.419557011703112,
      "learning_rate": 1.3197219237344155e-07,
      "logits/chosen": -2.5275566577911377,
      "logits/rejected": -2.50520396232605,
      "logps/chosen": -279.53619384765625,
      "logps/rejected": -297.91168212890625,
      "loss": 0.5781,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.25856587290763855,
      "rewards/margins": 0.46807146072387695,
      "rewards/rejected": -0.7266373038291931,
      "step": 1210
    },
    {
      "epoch": 0.7,
      "grad_norm": 23.724168571838458,
      "learning_rate": 1.2760003200873697e-07,
      "logits/chosen": -2.5259270668029785,
      "logits/rejected": -2.463974714279175,
      "logps/chosen": -274.4045715332031,
      "logps/rejected": -303.4898986816406,
      "loss": 0.5929,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.40674060583114624,
      "rewards/margins": 0.42905139923095703,
      "rewards/rejected": -0.8357919454574585,
      "step": 1220
    },
    {
      "epoch": 0.7,
      "grad_norm": 25.666665518773463,
      "learning_rate": 1.2327656901707338e-07,
      "logits/chosen": -2.522529125213623,
      "logits/rejected": -2.4855215549468994,
      "logps/chosen": -291.68292236328125,
      "logps/rejected": -332.4886474609375,
      "loss": 0.5913,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.3141830861568451,
      "rewards/margins": 0.4157239496707916,
      "rewards/rejected": -0.7299070358276367,
      "step": 1230
    },
    {
      "epoch": 0.71,
      "grad_norm": 27.19771337622201,
      "learning_rate": 1.1900352350748024e-07,
      "logits/chosen": -2.5396928787231445,
      "logits/rejected": -2.527820587158203,
      "logps/chosen": -299.08056640625,
      "logps/rejected": -341.8636474609375,
      "loss": 0.5916,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.3272027373313904,
      "rewards/margins": 0.37227770686149597,
      "rewards/rejected": -0.6994804739952087,
      "step": 1240
    },
    {
      "epoch": 0.71,
      "grad_norm": 33.33117238316685,
      "learning_rate": 1.147825955301668e-07,
      "logits/chosen": -2.533803939819336,
      "logits/rejected": -2.4939873218536377,
      "logps/chosen": -256.9107666015625,
      "logps/rejected": -292.6522521972656,
      "loss": 0.581,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.3282284736633301,
      "rewards/margins": 0.4669713079929352,
      "rewards/rejected": -0.7951998114585876,
      "step": 1250
    },
    {
      "epoch": 0.72,
      "grad_norm": 29.14283955300301,
      "learning_rate": 1.1061546440014963e-07,
      "logits/chosen": -2.5241713523864746,
      "logits/rejected": -2.5124003887176514,
      "logps/chosen": -256.1329040527344,
      "logps/rejected": -269.87127685546875,
      "loss": 0.6125,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.24249228835105896,
      "rewards/margins": 0.24842050671577454,
      "rewards/rejected": -0.4909127652645111,
      "step": 1260
    },
    {
      "epoch": 0.73,
      "grad_norm": 25.742222165117763,
      "learning_rate": 1.0650378802913007e-07,
      "logits/chosen": -2.4792118072509766,
      "logits/rejected": -2.428273916244507,
      "logps/chosen": -239.7839813232422,
      "logps/rejected": -273.16070556640625,
      "loss": 0.5855,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.14450299739837646,
      "rewards/margins": 0.43993139266967773,
      "rewards/rejected": -0.5844343900680542,
      "step": 1270
    },
    {
      "epoch": 0.73,
      "grad_norm": 23.149905324082404,
      "learning_rate": 1.0244920226588597e-07,
      "logits/chosen": -2.5420844554901123,
      "logits/rejected": -2.4969589710235596,
      "logps/chosen": -246.9677734375,
      "logps/rejected": -282.594482421875,
      "loss": 0.5884,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.17258282005786896,
      "rewards/margins": 0.34093397855758667,
      "rewards/rejected": -0.5135167837142944,
      "step": 1280
    },
    {
      "epoch": 0.74,
      "grad_norm": 24.301291223143128,
      "learning_rate": 9.845332024544245e-08,
      "logits/chosen": -2.5490660667419434,
      "logits/rejected": -2.5159246921539307,
      "logps/chosen": -253.81149291992188,
      "logps/rejected": -271.77716064453125,
      "loss": 0.5834,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.09992814064025879,
      "rewards/margins": 0.4652518332004547,
      "rewards/rejected": -0.5651799440383911,
      "step": 1290
    },
    {
      "epoch": 0.74,
      "grad_norm": 23.714606581482208,
      "learning_rate": 9.451773174727873e-08,
      "logits/chosen": -2.517934799194336,
      "logits/rejected": -2.467642068862915,
      "logps/chosen": -253.8499298095703,
      "logps/rejected": -283.5979919433594,
      "loss": 0.5725,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.028648119419813156,
      "rewards/margins": 0.5222169756889343,
      "rewards/rejected": -0.5508650541305542,
      "step": 1300
    },
    {
      "epoch": 0.74,
      "eval_logits/chosen": -2.5130560398101807,
      "eval_logits/rejected": -2.4589791297912598,
      "eval_logps/chosen": -245.44595336914062,
      "eval_logps/rejected": -282.7961120605469,
      "eval_loss": 0.580857515335083,
      "eval_rewards/accuracies": 0.6878204941749573,
      "eval_rewards/chosen": -0.11069269478321075,
      "eval_rewards/margins": 0.4548812806606293,
      "eval_rewards/rejected": -0.5655739903450012,
      "eval_runtime": 863.5784,
      "eval_samples_per_second": 14.418,
      "eval_steps_per_second": 0.452,
      "step": 1300
    },
    {
      "epoch": 0.75,
      "grad_norm": 25.51529829677871,
      "learning_rate": 9.064400256282755e-08,
      "logits/chosen": -2.497252941131592,
      "logits/rejected": -2.4359519481658936,
      "logps/chosen": -246.2664794921875,
      "logps/rejected": -289.52105712890625,
      "loss": 0.565,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.18726077675819397,
      "rewards/margins": 0.4956758916378021,
      "rewards/rejected": -0.6829366087913513,
      "step": 1310
    },
    {
      "epoch": 0.75,
      "grad_norm": 28.395653589692625,
      "learning_rate": 8.68336738725177e-08,
      "logits/chosen": -2.5088109970092773,
      "logits/rejected": -2.444235324859619,
      "logps/chosen": -260.43255615234375,
      "logps/rejected": -314.78466796875,
      "loss": 0.585,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.09720905870199203,
      "rewards/margins": 0.5907143950462341,
      "rewards/rejected": -0.6879234910011292,
      "step": 1320
    },
    {
      "epoch": 0.76,
      "grad_norm": 34.946146652842195,
      "learning_rate": 8.308826163260852e-08,
      "logits/chosen": -2.4821484088897705,
      "logits/rejected": -2.437180995941162,
      "logps/chosen": -284.7655029296875,
      "logps/rejected": -339.27410888671875,
      "loss": 0.6013,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.38431376218795776,
      "rewards/margins": 0.5592700242996216,
      "rewards/rejected": -0.9435837864875793,
      "step": 1330
    },
    {
      "epoch": 0.77,
      "grad_norm": 27.1693846661215,
      "learning_rate": 7.940925597206052e-08,
      "logits/chosen": -2.5242748260498047,
      "logits/rejected": -2.423187732696533,
      "logps/chosen": -240.61532592773438,
      "logps/rejected": -303.8311462402344,
      "loss": 0.5867,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.17216572165489197,
      "rewards/margins": 0.4809741973876953,
      "rewards/rejected": -0.6531398892402649,
      "step": 1340
    },
    {
      "epoch": 0.77,
      "grad_norm": 26.85515693070903,
      "learning_rate": 7.579812059968014e-08,
      "logits/chosen": -2.468845844268799,
      "logits/rejected": -2.4025533199310303,
      "logps/chosen": -239.42636108398438,
      "logps/rejected": -292.7440490722656,
      "loss": 0.5718,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.1654459834098816,
      "rewards/margins": 0.45221590995788574,
      "rewards/rejected": -0.6176619529724121,
      "step": 1350
    },
    {
      "epoch": 0.78,
      "grad_norm": 29.04379120857399,
      "learning_rate": 7.225629222177715e-08,
      "logits/chosen": -2.4990878105163574,
      "logits/rejected": -2.483397960662842,
      "logps/chosen": -273.4325256347656,
      "logps/rejected": -326.87835693359375,
      "loss": 0.5592,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.2638954222202301,
      "rewards/margins": 0.4499826431274414,
      "rewards/rejected": -0.7138780355453491,
      "step": 1360
    },
    {
      "epoch": 0.78,
      "grad_norm": 27.003396904472925,
      "learning_rate": 6.878517997056457e-08,
      "logits/chosen": -2.5113110542297363,
      "logits/rejected": -2.4679648876190186,
      "logps/chosen": -231.7012176513672,
      "logps/rejected": -295.1663818359375,
      "loss": 0.5706,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.17551007866859436,
      "rewards/margins": 0.5275388956069946,
      "rewards/rejected": -0.7030488848686218,
      "step": 1370
    },
    {
      "epoch": 0.79,
      "grad_norm": 25.178752513980715,
      "learning_rate": 6.538616484352902e-08,
      "logits/chosen": -2.5119760036468506,
      "logits/rejected": -2.473975658416748,
      "logps/chosen": -261.88665771484375,
      "logps/rejected": -301.8829040527344,
      "loss": 0.5784,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.27380552887916565,
      "rewards/margins": 0.4960559010505676,
      "rewards/rejected": -0.7698613405227661,
      "step": 1380
    },
    {
      "epoch": 0.79,
      "grad_norm": 31.62767752494405,
      "learning_rate": 6.206059915399495e-08,
      "logits/chosen": -2.475964307785034,
      "logits/rejected": -2.415445566177368,
      "logps/chosen": -300.7445373535156,
      "logps/rejected": -290.59259033203125,
      "loss": 0.5803,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.21556460857391357,
      "rewards/margins": 0.4194510579109192,
      "rewards/rejected": -0.6350156664848328,
      "step": 1390
    },
    {
      "epoch": 0.8,
      "grad_norm": 26.11221463277773,
      "learning_rate": 5.88098059931004e-08,
      "logits/chosen": -2.513415575027466,
      "logits/rejected": -2.431800603866577,
      "logps/chosen": -248.53909301757812,
      "logps/rejected": -316.1845703125,
      "loss": 0.5719,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.17952445149421692,
      "rewards/margins": 0.6487424373626709,
      "rewards/rejected": -0.8282669186592102,
      "step": 1400
    },
    {
      "epoch": 0.8,
      "eval_logits/chosen": -2.5095772743225098,
      "eval_logits/rejected": -2.4584505558013916,
      "eval_logps/chosen": -255.48683166503906,
      "eval_logps/rejected": -296.0592346191406,
      "eval_loss": 0.5792786478996277,
      "eval_rewards/accuracies": 0.6894230842590332,
      "eval_rewards/chosen": -0.2111014872789383,
      "eval_rewards/margins": 0.4871039092540741,
      "eval_rewards/rejected": -0.6982054114341736,
      "eval_runtime": 847.3731,
      "eval_samples_per_second": 14.694,
      "eval_steps_per_second": 0.46,
      "step": 1400
    },
    {
      "epoch": 0.81,
      "grad_norm": 31.913715183238843,
      "learning_rate": 5.563507870339962e-08,
      "logits/chosen": -2.4840493202209473,
      "logits/rejected": -2.45074725151062,
      "logps/chosen": -297.8506164550781,
      "logps/rejected": -323.21722412109375,
      "loss": 0.5745,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.21714219450950623,
      "rewards/margins": 0.49252939224243164,
      "rewards/rejected": -0.7096715569496155,
      "step": 1410
    },
    {
      "epoch": 0.81,
      "grad_norm": 35.605251227958966,
      "learning_rate": 5.25376803643007e-08,
      "logits/chosen": -2.5487353801727295,
      "logits/rejected": -2.4819111824035645,
      "logps/chosen": -277.7652282714844,
      "logps/rejected": -307.17413330078125,
      "loss": 0.5827,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.3039516806602478,
      "rewards/margins": 0.4392912983894348,
      "rewards/rejected": -0.7432429194450378,
      "step": 1420
    },
    {
      "epoch": 0.82,
      "grad_norm": 28.285683945551348,
      "learning_rate": 4.9518843289544007e-08,
      "logits/chosen": -2.5319647789001465,
      "logits/rejected": -2.4716343879699707,
      "logps/chosen": -246.0936737060547,
      "logps/rejected": -295.38726806640625,
      "loss": 0.5892,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.1578165590763092,
      "rewards/margins": 0.518903911113739,
      "rewards/rejected": -0.6767204999923706,
      "step": 1430
    },
    {
      "epoch": 0.82,
      "grad_norm": 29.113838409092516,
      "learning_rate": 4.6579768536920695e-08,
      "logits/chosen": -2.5386886596679688,
      "logits/rejected": -2.4646518230438232,
      "logps/chosen": -255.16152954101562,
      "logps/rejected": -311.1474609375,
      "loss": 0.6026,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.15220266580581665,
      "rewards/margins": 0.46218329668045044,
      "rewards/rejected": -0.6143860220909119,
      "step": 1440
    },
    {
      "epoch": 0.83,
      "grad_norm": 33.086643583472295,
      "learning_rate": 4.372162543042623e-08,
      "logits/chosen": -2.506279230117798,
      "logits/rejected": -2.4417097568511963,
      "logps/chosen": -250.19287109375,
      "logps/rejected": -287.5571594238281,
      "loss": 0.5846,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07686599344015121,
      "rewards/margins": 0.5035240650177002,
      "rewards/rejected": -0.5803900957107544,
      "step": 1450
    },
    {
      "epoch": 0.83,
      "grad_norm": 22.051531350612663,
      "learning_rate": 4.094555109503983e-08,
      "logits/chosen": -2.5418009757995605,
      "logits/rejected": -2.4571533203125,
      "logps/chosen": -257.3257141113281,
      "logps/rejected": -294.6969299316406,
      "loss": 0.5669,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.17704418301582336,
      "rewards/margins": 0.46817469596862793,
      "rewards/rejected": -0.6452188491821289,
      "step": 1460
    },
    {
      "epoch": 0.84,
      "grad_norm": 31.588020383365453,
      "learning_rate": 3.825265000431424e-08,
      "logits/chosen": -2.513741970062256,
      "logits/rejected": -2.4840691089630127,
      "logps/chosen": -260.6844177246094,
      "logps/rejected": -297.4983825683594,
      "loss": 0.5591,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.2122991979122162,
      "rewards/margins": 0.5046205520629883,
      "rewards/rejected": -0.7169197797775269,
      "step": 1470
    },
    {
      "epoch": 0.85,
      "grad_norm": 27.48498825785638,
      "learning_rate": 3.5643993540955844e-08,
      "logits/chosen": -2.5381698608398438,
      "logits/rejected": -2.4838833808898926,
      "logps/chosen": -266.23724365234375,
      "logps/rejected": -313.0609130859375,
      "loss": 0.5712,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.22639349102973938,
      "rewards/margins": 0.47134050726890564,
      "rewards/rejected": -0.697733998298645,
      "step": 1480
    },
    {
      "epoch": 0.85,
      "grad_norm": 29.66621598419702,
      "learning_rate": 3.312061957057061e-08,
      "logits/chosen": -2.5433754920959473,
      "logits/rejected": -2.4874513149261475,
      "logps/chosen": -253.4493408203125,
      "logps/rejected": -313.81622314453125,
      "loss": 0.5552,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.29640263319015503,
      "rewards/margins": 0.5009024739265442,
      "rewards/rejected": -0.797305166721344,
      "step": 1490
    },
    {
      "epoch": 0.86,
      "grad_norm": 25.013543267548187,
      "learning_rate": 3.0683532028744184e-08,
      "logits/chosen": -2.5175490379333496,
      "logits/rejected": -2.4559388160705566,
      "logps/chosen": -250.25332641601562,
      "logps/rejected": -292.8772888183594,
      "loss": 0.5702,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.2427365779876709,
      "rewards/margins": 0.4707626700401306,
      "rewards/rejected": -0.7134992480278015,
      "step": 1500
    },
    {
      "epoch": 0.86,
      "eval_logits/chosen": -2.5242793560028076,
      "eval_logits/rejected": -2.4745876789093018,
      "eval_logps/chosen": -261.0100402832031,
      "eval_logps/rejected": -301.7152099609375,
      "eval_loss": 0.5788725018501282,
      "eval_rewards/accuracies": 0.6887820363044739,
      "eval_rewards/chosen": -0.26633328199386597,
      "eval_rewards/margins": 0.4884316027164459,
      "eval_rewards/rejected": -0.754764974117279,
      "eval_runtime": 847.3963,
      "eval_samples_per_second": 14.693,
      "eval_steps_per_second": 0.46,
      "step": 1500
    },
    {
      "epoch": 0.86,
      "grad_norm": 27.931120203439026,
      "learning_rate": 2.8333700521621955e-08,
      "logits/chosen": -2.4960639476776123,
      "logits/rejected": -2.465576648712158,
      "logps/chosen": -262.0352478027344,
      "logps/rejected": -281.86077880859375,
      "loss": 0.5803,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.2684056758880615,
      "rewards/margins": 0.4416535496711731,
      "rewards/rejected": -0.7100592255592346,
      "step": 1510
    },
    {
      "epoch": 0.87,
      "grad_norm": 33.57688090254581,
      "learning_rate": 2.6072059940146772e-08,
      "logits/chosen": -2.545090675354004,
      "logits/rejected": -2.5021095275878906,
      "logps/chosen": -262.41546630859375,
      "logps/rejected": -307.7872009277344,
      "loss": 0.5798,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.2850339114665985,
      "rewards/margins": 0.5246847867965698,
      "rewards/rejected": -0.8097187280654907,
      "step": 1520
    },
    {
      "epoch": 0.87,
      "grad_norm": 25.889341574243268,
      "learning_rate": 2.3899510088108587e-08,
      "logits/chosen": -2.5121829509735107,
      "logits/rejected": -2.485520839691162,
      "logps/chosen": -253.7327117919922,
      "logps/rejected": -294.1796569824219,
      "loss": 0.5486,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.231918066740036,
      "rewards/margins": 0.5059068202972412,
      "rewards/rejected": -0.7378247976303101,
      "step": 1530
    },
    {
      "epoch": 0.88,
      "grad_norm": 25.634449566248733,
      "learning_rate": 2.1816915324153334e-08,
      "logits/chosen": -2.541168212890625,
      "logits/rejected": -2.4927926063537598,
      "logps/chosen": -277.5816650390625,
      "logps/rejected": -327.8016662597656,
      "loss": 0.5588,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.2811830937862396,
      "rewards/margins": 0.6148523092269897,
      "rewards/rejected": -0.896035373210907,
      "step": 1540
    },
    {
      "epoch": 0.89,
      "grad_norm": 24.433006444286615,
      "learning_rate": 1.9825104217894018e-08,
      "logits/chosen": -2.5491321086883545,
      "logits/rejected": -2.494581460952759,
      "logps/chosen": -252.4076690673828,
      "logps/rejected": -305.54437255859375,
      "loss": 0.5959,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.3050937354564667,
      "rewards/margins": 0.45700544118881226,
      "rewards/rejected": -0.7620991468429565,
      "step": 1550
    },
    {
      "epoch": 0.89,
      "grad_norm": 24.678156956964582,
      "learning_rate": 1.7924869220260626e-08,
      "logits/chosen": -2.5328431129455566,
      "logits/rejected": -2.4817864894866943,
      "logps/chosen": -251.4116973876953,
      "logps/rejected": -301.2293395996094,
      "loss": 0.5487,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.2088792771100998,
      "rewards/margins": 0.508228600025177,
      "rewards/rejected": -0.7171077728271484,
      "step": 1560
    },
    {
      "epoch": 0.9,
      "grad_norm": 28.5502202419384,
      "learning_rate": 1.6116966348220046e-08,
      "logits/chosen": -2.5276474952697754,
      "logits/rejected": -2.498145818710327,
      "logps/chosen": -244.6429443359375,
      "logps/rejected": -293.36865234375,
      "loss": 0.56,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.10921525955200195,
      "rewards/margins": 0.6631591320037842,
      "rewards/rejected": -0.7723743319511414,
      "step": 1570
    },
    {
      "epoch": 0.9,
      "grad_norm": 30.92180920593632,
      "learning_rate": 1.4402114883991318e-08,
      "logits/chosen": -2.526020050048828,
      "logits/rejected": -2.4712436199188232,
      "logps/chosen": -261.9678039550781,
      "logps/rejected": -287.0662536621094,
      "loss": 0.5647,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.3635459542274475,
      "rewards/margins": 0.45373767614364624,
      "rewards/rejected": -0.8172836303710938,
      "step": 1580
    },
    {
      "epoch": 0.91,
      "grad_norm": 29.555421546021286,
      "learning_rate": 1.2780997088875866e-08,
      "logits/chosen": -2.5242857933044434,
      "logits/rejected": -2.475109100341797,
      "logps/chosen": -264.02740478515625,
      "logps/rejected": -286.8149108886719,
      "loss": 0.6074,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.1986238658428192,
      "rewards/margins": 0.452684223651886,
      "rewards/rejected": -0.6513080596923828,
      "step": 1590
    },
    {
      "epoch": 0.91,
      "grad_norm": 26.529175018271285,
      "learning_rate": 1.125425793181703e-08,
      "logits/chosen": -2.5457723140716553,
      "logits/rejected": -2.5002970695495605,
      "logps/chosen": -268.09466552734375,
      "logps/rejected": -307.01690673828125,
      "loss": 0.5854,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.31708940863609314,
      "rewards/margins": 0.44073382019996643,
      "rewards/rejected": -0.7578231692314148,
      "step": 1600
    },
    {
      "epoch": 0.91,
      "eval_logits/chosen": -2.552327871322632,
      "eval_logits/rejected": -2.503713846206665,
      "eval_logps/chosen": -257.1977233886719,
      "eval_logps/rejected": -298.1694641113281,
      "eval_loss": 0.5782989859580994,
      "eval_rewards/accuracies": 0.6913461685180664,
      "eval_rewards/chosen": -0.22821034491062164,
      "eval_rewards/margins": 0.49109748005867004,
      "eval_rewards/rejected": -0.7193077802658081,
      "eval_runtime": 860.4396,
      "eval_samples_per_second": 14.471,
      "eval_steps_per_second": 0.453,
      "step": 1600
    },
    {
      "epoch": 0.92,
      "grad_norm": 31.28191437907546,
      "learning_rate": 9.822504832796036e-09,
      "logits/chosen": -2.559476375579834,
      "logits/rejected": -2.5392966270446777,
      "logps/chosen": -271.2260437011719,
      "logps/rejected": -299.50653076171875,
      "loss": 0.5778,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.2566366195678711,
      "rewards/margins": 0.49396759271621704,
      "rewards/rejected": -0.7506042718887329,
      "step": 1610
    },
    {
      "epoch": 0.93,
      "grad_norm": 30.662407347819215,
      "learning_rate": 8.48630742116746e-09,
      "logits/chosen": -2.536745071411133,
      "logits/rejected": -2.4669063091278076,
      "logps/chosen": -273.7347717285156,
      "logps/rejected": -307.0451354980469,
      "loss": 0.6142,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.28270235657691956,
      "rewards/margins": 0.5321863293647766,
      "rewards/rejected": -0.8148886561393738,
      "step": 1620
    },
    {
      "epoch": 0.93,
      "grad_norm": 28.8995046840584,
      "learning_rate": 7.246197309029617e-09,
      "logits/chosen": -2.5540242195129395,
      "logits/rejected": -2.51172137260437,
      "logps/chosen": -288.4748229980469,
      "logps/rejected": -334.9012451171875,
      "loss": 0.5696,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.14674052596092224,
      "rewards/margins": 0.5696360468864441,
      "rewards/rejected": -0.7163766026496887,
      "step": 1630
    },
    {
      "epoch": 0.94,
      "grad_norm": 30.322071826617417,
      "learning_rate": 6.102667879720164e-09,
      "logits/chosen": -2.517651081085205,
      "logits/rejected": -2.470822811126709,
      "logps/chosen": -270.4826354980469,
      "logps/rejected": -301.36944580078125,
      "loss": 0.5924,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.3259834349155426,
      "rewards/margins": 0.4199187755584717,
      "rewards/rejected": -0.7459022402763367,
      "step": 1640
    },
    {
      "epoch": 0.94,
      "grad_norm": 37.10925817248419,
      "learning_rate": 5.056174091521509e-09,
      "logits/chosen": -2.585538387298584,
      "logits/rejected": -2.5188987255096436,
      "logps/chosen": -271.0771179199219,
      "logps/rejected": -313.93499755859375,
      "loss": 0.5735,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.19604423642158508,
      "rewards/margins": 0.5013729333877563,
      "rewards/rejected": -0.697417140007019,
      "step": 1650
    },
    {
      "epoch": 0.95,
      "grad_norm": 28.879184337464647,
      "learning_rate": 4.107132296653548e-09,
      "logits/chosen": -2.545414447784424,
      "logits/rejected": -2.5129075050354004,
      "logps/chosen": -249.78903198242188,
      "logps/rejected": -274.78466796875,
      "loss": 0.5854,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.20237192511558533,
      "rewards/margins": 0.3796917498111725,
      "rewards/rejected": -0.5820637345314026,
      "step": 1660
    },
    {
      "epoch": 0.95,
      "grad_norm": 31.80047095954754,
      "learning_rate": 3.255920075626084e-09,
      "logits/chosen": -2.501981735229492,
      "logits/rejected": -2.5079236030578613,
      "logps/chosen": -243.0762176513672,
      "logps/rejected": -292.73077392578125,
      "loss": 0.5755,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.18036358058452606,
      "rewards/margins": 0.4649125933647156,
      "rewards/rejected": -0.6452761888504028,
      "step": 1670
    },
    {
      "epoch": 0.96,
      "grad_norm": 26.04495569695924,
      "learning_rate": 2.5028760870168253e-09,
      "logits/chosen": -2.5446419715881348,
      "logits/rejected": -2.5122885704040527,
      "logps/chosen": -276.3479919433594,
      "logps/rejected": -304.0161437988281,
      "loss": 0.5854,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.3501695692539215,
      "rewards/margins": 0.2919423580169678,
      "rewards/rejected": -0.6421119570732117,
      "step": 1680
    },
    {
      "epoch": 0.97,
      "grad_norm": 24.450405436519606,
      "learning_rate": 1.8482999327343597e-09,
      "logits/chosen": -2.5676443576812744,
      "logits/rejected": -2.4829487800598145,
      "logps/chosen": -276.8134765625,
      "logps/rejected": -306.0779113769531,
      "loss": 0.6091,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.22208046913146973,
      "rewards/margins": 0.37004339694976807,
      "rewards/rejected": -0.5921238660812378,
      "step": 1690
    },
    {
      "epoch": 0.97,
      "grad_norm": 31.006086476520473,
      "learning_rate": 1.2924520388204462e-09,
      "logits/chosen": -2.5088953971862793,
      "logits/rejected": -2.507538318634033,
      "logps/chosen": -248.55526733398438,
      "logps/rejected": -294.12591552734375,
      "loss": 0.578,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3303738832473755,
      "rewards/margins": 0.3995395600795746,
      "rewards/rejected": -0.7299134135246277,
      "step": 1700
    },
    {
      "epoch": 0.97,
      "eval_logits/chosen": -2.5474650859832764,
      "eval_logits/rejected": -2.4987149238586426,
      "eval_logps/chosen": -255.72315979003906,
      "eval_logps/rejected": -296.4235534667969,
      "eval_loss": 0.5782065987586975,
      "eval_rewards/accuracies": 0.691987156867981,
      "eval_rewards/chosen": -0.21346470713615417,
      "eval_rewards/margins": 0.4883839786052704,
      "eval_rewards/rejected": -0.7018487453460693,
      "eval_runtime": 847.8379,
      "eval_samples_per_second": 14.686,
      "eval_steps_per_second": 0.46,
      "step": 1700
    },
    {
      "epoch": 0.98,
      "grad_norm": 27.94865663504945,
      "learning_rate": 8.355535518381751e-10,
      "logits/chosen": -2.5459036827087402,
      "logits/rejected": -2.523585081100464,
      "logps/chosen": -267.97119140625,
      "logps/rejected": -273.44970703125,
      "loss": 0.5877,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.24003572762012482,
      "rewards/margins": 0.3651558756828308,
      "rewards/rejected": -0.6051916480064392,
      "step": 1710
    },
    {
      "epoch": 0.98,
      "grad_norm": 26.206122309038193,
      "learning_rate": 4.77786250887846e-10,
      "logits/chosen": -2.5695338249206543,
      "logits/rejected": -2.5357468128204346,
      "logps/chosen": -251.841552734375,
      "logps/rejected": -304.9547424316406,
      "loss": 0.5833,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.12936021387577057,
      "rewards/margins": 0.5671738386154175,
      "rewards/rejected": -0.6965340971946716,
      "step": 1720
    },
    {
      "epoch": 0.99,
      "grad_norm": 26.649920950848678,
      "learning_rate": 2.1929247528540418e-10,
      "logits/chosen": -2.554141044616699,
      "logits/rejected": -2.493603467941284,
      "logps/chosen": -265.3050231933594,
      "logps/rejected": -310.0926208496094,
      "loss": 0.5706,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.20094648003578186,
      "rewards/margins": 0.47598400712013245,
      "rewards/rejected": -0.6769304871559143,
      "step": 1730
    },
    {
      "epoch": 0.99,
      "grad_norm": 27.75248252937658,
      "learning_rate": 6.017506793212779e-11,
      "logits/chosen": -2.5380330085754395,
      "logits/rejected": -2.4906625747680664,
      "logps/chosen": -251.3086700439453,
      "logps/rejected": -314.17645263671875,
      "loss": 0.5705,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.11332281678915024,
      "rewards/margins": 0.5656474828720093,
      "rewards/rejected": -0.6789703369140625,
      "step": 1740
    },
    {
      "epoch": 1.0,
      "grad_norm": 31.342302748055804,
      "learning_rate": 4.973343980252398e-13,
      "logits/chosen": -2.543778896331787,
      "logits/rejected": -2.499126434326172,
      "logps/chosen": -258.0113830566406,
      "logps/rejected": -294.2593994140625,
      "loss": 0.5742,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.23090767860412598,
      "rewards/margins": 0.5441216230392456,
      "rewards/rejected": -0.7750292420387268,
      "step": 1750
    },
    {
      "epoch": 1.0,
      "step": 1751,
      "total_flos": 0.0,
      "train_loss": 0.5964619764459262,
      "train_runtime": 39064.845,
      "train_samples_per_second": 2.868,
      "train_steps_per_second": 0.045
    }
  ],
  "logging_steps": 10,
  "max_steps": 1751,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}