diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,2942 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 100,
+  "global_step": 1751,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "grad_norm": 20.572983553522633,
+      "learning_rate": 2.8409090909090907e-09,
+      "logits/chosen": -2.7078514099121094,
+      "logits/rejected": -2.673313856124878,
+      "logps/chosen": -322.9836120605469,
+      "logps/rejected": -261.1998291015625,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 19.20958843122381,
+      "learning_rate": 2.8409090909090908e-08,
+      "logits/chosen": -2.7531583309173584,
+      "logits/rejected": -2.715681314468384,
+      "logps/chosen": -219.50958251953125,
+      "logps/rejected": -236.27330017089844,
+      "loss": 0.6934,
+      "rewards/accuracies": 0.4166666567325592,
+      "rewards/chosen": -0.0001647783356020227,
+      "rewards/margins": 6.51524678687565e-05,
+      "rewards/rejected": -0.00022993076709099114,
+      "step": 10
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 18.221209446502318,
+      "learning_rate": 5.6818181818181815e-08,
+      "logits/chosen": -2.736729145050049,
+      "logits/rejected": -2.7084593772888184,
+      "logps/chosen": -233.8996124267578,
+      "logps/rejected": -230.8451385498047,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": 0.00037134578451514244,
+      "rewards/margins": 2.871244305424625e-06,
+      "rewards/rejected": 0.000368474458809942,
+      "step": 20
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 19.659186304289324,
+      "learning_rate": 8.522727272727271e-08,
+      "logits/chosen": -2.718043804168701,
+      "logits/rejected": -2.705275535583496,
+      "logps/chosen": -265.68658447265625,
+      "logps/rejected": -247.42025756835938,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": 0.0039044865407049656,
+      "rewards/margins": 0.0005032892222516239,
+      "rewards/rejected": 0.003401197027415037,
+      "step": 30
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 21.248085751648183,
+      "learning_rate": 1.1363636363636363e-07,
+      "logits/chosen": -2.722376585006714,
+      "logits/rejected": -2.7113113403320312,
+      "logps/chosen": -207.11141967773438,
+      "logps/rejected": -212.8821563720703,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": 0.013140101917088032,
+      "rewards/margins": 0.0030325299594551325,
+      "rewards/rejected": 0.010107570327818394,
+      "step": 40
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 17.062348015914782,
+      "learning_rate": 1.4204545454545455e-07,
+      "logits/chosen": -2.7086739540100098,
+      "logits/rejected": -2.6903626918792725,
+      "logps/chosen": -230.5471954345703,
+      "logps/rejected": -218.68508911132812,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": 0.030233308672904968,
+      "rewards/margins": 0.0065773241221904755,
+      "rewards/rejected": 0.023655982688069344,
+      "step": 50
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 16.966597240337634,
+      "learning_rate": 1.7045454545454543e-07,
+      "logits/chosen": -2.7187132835388184,
+      "logits/rejected": -2.707646608352661,
+      "logps/chosen": -231.4257354736328,
+      "logps/rejected": -222.29745483398438,
+      "loss": 0.6856,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.05241071432828903,
+      "rewards/margins": 0.018025759607553482,
+      "rewards/rejected": 0.03438495472073555,
+      "step": 60
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 19.376648396399435,
+      "learning_rate": 1.9886363636363636e-07,
+      "logits/chosen": -2.7627768516540527,
+      "logits/rejected": -2.699197292327881,
+      "logps/chosen": -230.32540893554688,
+      "logps/rejected": -220.99795532226562,
+      "loss": 0.6812,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.08621755987405777,
+      "rewards/margins": 0.024967864155769348,
+      "rewards/rejected": 0.06124969199299812,
+      "step": 70
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": 16.574962507205655,
+      "learning_rate": 2.2727272727272726e-07,
+      "logits/chosen": -2.7578330039978027,
+      "logits/rejected": -2.758152723312378,
+      "logps/chosen": -241.5107879638672,
+      "logps/rejected": -234.1169891357422,
+      "loss": 0.6775,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": 0.1304091513156891,
+      "rewards/margins": 0.03304018825292587,
+      "rewards/rejected": 0.09736897051334381,
+      "step": 80
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": 17.123245271646947,
+      "learning_rate": 2.5568181818181816e-07,
+      "logits/chosen": -2.747803211212158,
+      "logits/rejected": -2.714414119720459,
+      "logps/chosen": -245.60464477539062,
+      "logps/rejected": -236.7899932861328,
+      "loss": 0.6688,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.12192847579717636,
+      "rewards/margins": 0.04756360873579979,
+      "rewards/rejected": 0.07436486333608627,
+      "step": 90
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 17.870065861056936,
+      "learning_rate": 2.840909090909091e-07,
+      "logits/chosen": -2.7195897102355957,
+      "logits/rejected": -2.681842803955078,
+      "logps/chosen": -207.11083984375,
+      "logps/rejected": -211.34866333007812,
+      "loss": 0.6628,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.1526772379875183,
+      "rewards/margins": 0.07189060747623444,
+      "rewards/rejected": 0.08078663051128387,
+      "step": 100
+    },
+    {
+      "epoch": 0.06,
+      "eval_logits/chosen": -2.7152342796325684,
+      "eval_logits/rejected": -2.6720974445343018,
+      "eval_logps/chosen": -221.00881958007812,
+      "eval_logps/rejected": -221.34713745117188,
+      "eval_loss": 0.6611106395721436,
+      "eval_rewards/accuracies": 0.6317307949066162,
+      "eval_rewards/chosen": 0.13367876410484314,
+      "eval_rewards/margins": 0.08476302027702332,
+      "eval_rewards/rejected": 0.04891572147607803,
+      "eval_runtime": 864.7473,
+      "eval_samples_per_second": 14.398,
+      "eval_steps_per_second": 0.451,
+      "step": 100
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 17.758173761785073,
+      "learning_rate": 3.1249999999999997e-07,
+      "logits/chosen": -2.745452642440796,
+      "logits/rejected": -2.6345908641815186,
+      "logps/chosen": -211.13497924804688,
+      "logps/rejected": -245.9041290283203,
+      "loss": 0.6526,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": 0.10004226118326187,
+      "rewards/margins": 0.09651143848896027,
+      "rewards/rejected": 0.0035308226943016052,
+      "step": 110
+    },
+    {
+      "epoch": 0.07,
+      "grad_norm": 25.3503988826482,
+      "learning_rate": 3.4090909090909085e-07,
+      "logits/chosen": -2.6189653873443604,
+      "logits/rejected": -2.57344388961792,
+      "logps/chosen": -228.3636932373047,
+      "logps/rejected": -230.4072265625,
+      "loss": 0.6476,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.04373621195554733,
+      "rewards/margins": 0.13258467614650726,
+      "rewards/rejected": -0.176320880651474,
+      "step": 120
+    },
+    {
+      "epoch": 0.07,
+      "grad_norm": 25.5123273990743,
+      "learning_rate": 3.693181818181818e-07,
+      "logits/chosen": -2.564441204071045,
+      "logits/rejected": -2.547898292541504,
+      "logps/chosen": -269.93511962890625,
+      "logps/rejected": -285.6426086425781,
+      "loss": 0.6484,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.3578580319881439,
+      "rewards/margins": 0.16583284735679626,
+      "rewards/rejected": -0.5236908793449402,
+      "step": 130
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 25.02924344762446,
+      "learning_rate": 3.977272727272727e-07,
+      "logits/chosen": -2.5508384704589844,
+      "logits/rejected": -2.5335073471069336,
+      "logps/chosen": -274.27508544921875,
+      "logps/rejected": -272.3787536621094,
+      "loss": 0.6395,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.3012983798980713,
+      "rewards/margins": 0.09775562584400177,
+      "rewards/rejected": -0.3990539610385895,
+      "step": 140
+    },
+    {
+      "epoch": 0.09,
+      "grad_norm": 23.226996578001714,
+      "learning_rate": 4.2613636363636364e-07,
+      "logits/chosen": -2.5063862800598145,
+      "logits/rejected": -2.4907989501953125,
+      "logps/chosen": -242.54183959960938,
+      "logps/rejected": -267.40216064453125,
+      "loss": 0.6367,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.11261695623397827,
+      "rewards/margins": 0.19061629474163055,
+      "rewards/rejected": -0.30323326587677,
+      "step": 150
+    },
+    {
+      "epoch": 0.09,
+      "grad_norm": 22.32459928056343,
+      "learning_rate": 4.545454545454545e-07,
+      "logits/chosen": -2.538865327835083,
+      "logits/rejected": -2.4915454387664795,
+      "logps/chosen": -259.973876953125,
+      "logps/rejected": -284.99273681640625,
+      "loss": 0.6342,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.06355403363704681,
+      "rewards/margins": 0.3071693778038025,
+      "rewards/rejected": -0.3707233965396881,
+      "step": 160
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 28.238806692314206,
+      "learning_rate": 4.829545454545455e-07,
+      "logits/chosen": -2.5641493797302246,
+      "logits/rejected": -2.5063588619232178,
+      "logps/chosen": -256.0674133300781,
+      "logps/rejected": -281.95965576171875,
+      "loss": 0.6207,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.11485638469457626,
+      "rewards/margins": 0.2587049603462219,
+      "rewards/rejected": -0.3735613226890564,
+      "step": 170
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 34.83431654717648,
+      "learning_rate": 4.999920426892062e-07,
+      "logits/chosen": -2.6064653396606445,
+      "logits/rejected": -2.600895404815674,
+      "logps/chosen": -245.4825897216797,
+      "logps/rejected": -257.6805725097656,
+      "loss": 0.6398,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.1361248791217804,
+      "rewards/margins": 0.16745266318321228,
+      "rewards/rejected": -0.3035775423049927,
+      "step": 180
+    },
+    {
+      "epoch": 0.11,
+      "grad_norm": 33.80086885729117,
+      "learning_rate": 4.999025287600885e-07,
+      "logits/chosen": -2.6627895832061768,
+      "logits/rejected": -2.6619114875793457,
+      "logps/chosen": -263.55096435546875,
+      "logps/rejected": -284.45147705078125,
+      "loss": 0.6325,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.09487941861152649,
+      "rewards/margins": 0.21393874287605286,
+      "rewards/rejected": -0.3088182210922241,
+      "step": 190
+    },
+    {
+      "epoch": 0.11,
+      "grad_norm": 29.828595993243468,
+      "learning_rate": 4.997135899956001e-07,
+      "logits/chosen": -2.638547420501709,
+      "logits/rejected": -2.5978102684020996,
+      "logps/chosen": -245.3714599609375,
+      "logps/rejected": -266.0381164550781,
+      "loss": 0.6203,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.1466042697429657,
+      "rewards/margins": 0.28931504487991333,
+      "rewards/rejected": -0.43591928482055664,
+      "step": 200
+    },
+    {
+      "epoch": 0.11,
+      "eval_logits/chosen": -2.6775360107421875,
+      "eval_logits/rejected": -2.621267080307007,
+      "eval_logps/chosen": -243.9757843017578,
+      "eval_logps/rejected": -266.8084411621094,
+      "eval_loss": 0.6120733618736267,
+      "eval_rewards/accuracies": 0.660897433757782,
+      "eval_rewards/chosen": -0.09599092602729797,
+      "eval_rewards/margins": 0.30970630049705505,
+      "eval_rewards/rejected": -0.405697226524353,
+      "eval_runtime": 841.2052,
+      "eval_samples_per_second": 14.801,
+      "eval_steps_per_second": 0.464,
+      "step": 200
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 31.207584928179323,
+      "learning_rate": 4.994253015658708e-07,
+      "logits/chosen": -2.6432251930236816,
+      "logits/rejected": -2.6140708923339844,
+      "logps/chosen": -232.1490936279297,
+      "logps/rejected": -261.6481018066406,
+      "loss": 0.6111,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.03973131626844406,
+      "rewards/margins": 0.33994999527931213,
+      "rewards/rejected": -0.3796812891960144,
+      "step": 210
+    },
+    {
+      "epoch": 0.13,
+      "grad_norm": 23.510226746553357,
+      "learning_rate": 4.990377781677378e-07,
+      "logits/chosen": -2.674729108810425,
+      "logits/rejected": -2.621265172958374,
+      "logps/chosen": -272.51715087890625,
+      "logps/rejected": -285.8130187988281,
+      "loss": 0.6261,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.12226327508687973,
+      "rewards/margins": 0.3179982602596283,
+      "rewards/rejected": -0.4402615427970886,
+      "step": 220
+    },
+    {
+      "epoch": 0.13,
+      "grad_norm": 34.69513987573851,
+      "learning_rate": 4.985511739791128e-07,
+      "logits/chosen": -2.682669162750244,
+      "logits/rejected": -2.6345534324645996,
+      "logps/chosen": -243.78695678710938,
+      "logps/rejected": -252.6505889892578,
+      "loss": 0.5961,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.19093939661979675,
+      "rewards/margins": 0.23702657222747803,
+      "rewards/rejected": -0.4279659688472748,
+      "step": 230
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 33.90649114668316,
+      "learning_rate": 4.979656825976425e-07,
+      "logits/chosen": -2.636639356613159,
+      "logits/rejected": -2.556272029876709,
+      "logps/chosen": -267.24310302734375,
+      "logps/rejected": -295.2035217285156,
+      "loss": 0.6078,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.3182388246059418,
+      "rewards/margins": 0.24964241683483124,
+      "rewards/rejected": -0.5678812861442566,
+      "step": 240
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 27.80845453447812,
+      "learning_rate": 4.972815369636832e-07,
+      "logits/chosen": -2.6218130588531494,
+      "logits/rejected": -2.55488920211792,
+      "logps/chosen": -299.002197265625,
+      "logps/rejected": -309.2835388183594,
+      "loss": 0.6176,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.2978210151195526,
+      "rewards/margins": 0.2504158914089203,
+      "rewards/rejected": -0.5482369661331177,
+      "step": 250
+    },
+    {
+      "epoch": 0.15,
+      "grad_norm": 28.138070133686533,
+      "learning_rate": 4.964990092676262e-07,
+      "logits/chosen": -2.6748335361480713,
+      "logits/rejected": -2.593806028366089,
+      "logps/chosen": -268.3004455566406,
+      "logps/rejected": -270.9266662597656,
+      "loss": 0.6223,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.1147528737783432,
+      "rewards/margins": 0.2601880431175232,
+      "rewards/rejected": -0.3749409317970276,
+      "step": 260
+    },
+    {
+      "epoch": 0.15,
+      "grad_norm": 28.844690039420406,
+      "learning_rate": 4.956184108416046e-07,
+      "logits/chosen": -2.7262961864471436,
+      "logits/rejected": -2.741664409637451,
+      "logps/chosen": -238.9697723388672,
+      "logps/rejected": -277.444091796875,
+      "loss": 0.5909,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.17726224660873413,
+      "rewards/margins": 0.36171460151672363,
+      "rewards/rejected": -0.538976788520813,
+      "step": 270
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 34.00184014415281,
+      "learning_rate": 4.946400920356287e-07,
+      "logits/chosen": -2.754479169845581,
+      "logits/rejected": -2.707367181777954,
+      "logps/chosen": -268.70452880859375,
+      "logps/rejected": -305.3086853027344,
+      "loss": 0.5949,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.30848780274391174,
+      "rewards/margins": 0.48010724782943726,
+      "rewards/rejected": -0.7885950803756714,
+      "step": 280
+    },
+    {
+      "epoch": 0.17,
+      "grad_norm": 24.974211972857105,
+      "learning_rate": 4.935644420781978e-07,
+      "logits/chosen": -2.6929306983947754,
+      "logits/rejected": -2.6737914085388184,
+      "logps/chosen": -283.9693603515625,
+      "logps/rejected": -305.68646240234375,
+      "loss": 0.5971,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.3218172490596771,
+      "rewards/margins": 0.37671035528182983,
+      "rewards/rejected": -0.6985276341438293,
+      "step": 290
+    },
+    {
+      "epoch": 0.17,
+      "grad_norm": 22.031451800060307,
+      "learning_rate": 4.923918889214436e-07,
+      "logits/chosen": -2.8449320793151855,
+      "logits/rejected": -2.802635908126831,
+      "logps/chosen": -249.634033203125,
+      "logps/rejected": -294.66973876953125,
+      "loss": 0.6134,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.1441958099603653,
+      "rewards/margins": 0.37521737813949585,
+      "rewards/rejected": -0.5194131731987,
+      "step": 300
+    },
+    {
+      "epoch": 0.17,
+      "eval_logits/chosen": -2.8550686836242676,
+      "eval_logits/rejected": -2.79876971244812,
+      "eval_logps/chosen": -240.60450744628906,
+      "eval_logps/rejected": -263.5723571777344,
+      "eval_loss": 0.6074427962303162,
+      "eval_rewards/accuracies": 0.670192301273346,
+      "eval_rewards/chosen": -0.06227818876504898,
+      "eval_rewards/margins": 0.31105825304985046,
+      "eval_rewards/rejected": -0.37333643436431885,
+      "eval_runtime": 861.7371,
+      "eval_samples_per_second": 14.449,
+      "eval_steps_per_second": 0.453,
+      "step": 300
+    },
+    {
+      "epoch": 0.18,
+      "grad_norm": 22.548306029706836,
+      "learning_rate": 4.91122899070868e-07,
+      "logits/chosen": -2.88305401802063,
+      "logits/rejected": -2.8267903327941895,
+      "logps/chosen": -253.9397735595703,
+      "logps/rejected": -268.42498779296875,
+      "loss": 0.608,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.02261662669479847,
+      "rewards/margins": 0.35017159581184387,
+      "rewards/rejected": -0.3727882504463196,
+      "step": 310
+    },
+    {
+      "epoch": 0.18,
+      "grad_norm": 26.91606740249196,
+      "learning_rate": 4.897579773997414e-07,
+      "logits/chosen": -2.7912163734436035,
+      "logits/rejected": -2.748124599456787,
+      "logps/chosen": -260.5610046386719,
+      "logps/rejected": -272.39898681640625,
+      "loss": 0.6085,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.2463793307542801,
+      "rewards/margins": 0.2676694393157959,
+      "rewards/rejected": -0.5140487551689148,
+      "step": 320
+    },
+    {
+      "epoch": 0.19,
+      "grad_norm": 25.26707067068886,
+      "learning_rate": 4.882976669482367e-07,
+      "logits/chosen": -2.6727445125579834,
+      "logits/rejected": -2.5968549251556396,
+      "logps/chosen": -253.74581909179688,
+      "logps/rejected": -296.7026672363281,
+      "loss": 0.6019,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.2068738043308258,
+      "rewards/margins": 0.3490857481956482,
+      "rewards/rejected": -0.5559597015380859,
+      "step": 330
+    },
+    {
+      "epoch": 0.19,
+      "grad_norm": 35.48578116044424,
+      "learning_rate": 4.867425487073786e-07,
+      "logits/chosen": -2.649488925933838,
+      "logits/rejected": -2.6318719387054443,
+      "logps/chosen": -241.77029418945312,
+      "logps/rejected": -246.7797088623047,
+      "loss": 0.629,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.01744382455945015,
+      "rewards/margins": 0.21225237846374512,
+      "rewards/rejected": -0.22969619929790497,
+      "step": 340
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 32.561918770233014,
+      "learning_rate": 4.850932413878934e-07,
+      "logits/chosen": -2.5814261436462402,
+      "logits/rejected": -2.5403079986572266,
+      "logps/chosen": -251.0198516845703,
+      "logps/rejected": -265.89190673828125,
+      "loss": 0.6163,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.004257055930793285,
+      "rewards/margins": 0.2787313163280487,
+      "rewards/rejected": -0.28298836946487427,
+      "step": 350
+    },
+    {
+      "epoch": 0.21,
+      "grad_norm": 32.76257794045328,
+      "learning_rate": 4.833504011740522e-07,
+      "logits/chosen": -2.493020534515381,
+      "logits/rejected": -2.4786593914031982,
+      "logps/chosen": -239.0045623779297,
+      "logps/rejected": -267.716552734375,
+      "loss": 0.5962,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.0083407461643219,
+      "rewards/margins": 0.33076852560043335,
+      "rewards/rejected": -0.33910924196243286,
+      "step": 360
+    },
+    {
+      "epoch": 0.21,
+      "grad_norm": 25.42313532237448,
+      "learning_rate": 4.815147214626056e-07,
+      "logits/chosen": -2.403029680252075,
+      "logits/rejected": -2.375685930252075,
+      "logps/chosen": -286.8995666503906,
+      "logps/rejected": -282.9924621582031,
+      "loss": 0.5961,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.26247408986091614,
+      "rewards/margins": 0.31958019733428955,
+      "rewards/rejected": -0.5820542573928833,
+      "step": 370
+    },
+    {
+      "epoch": 0.22,
+      "grad_norm": 29.80215735650837,
+      "learning_rate": 4.795869325869116e-07,
+      "logits/chosen": -2.416861057281494,
+      "logits/rejected": -2.4126358032226562,
+      "logps/chosen": -263.8486022949219,
+      "logps/rejected": -291.6919250488281,
+      "loss": 0.6312,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.2886821925640106,
+      "rewards/margins": 0.29902181029319763,
+      "rewards/rejected": -0.5877040028572083,
+      "step": 380
+    },
+    {
+      "epoch": 0.22,
+      "grad_norm": 22.149160653773386,
+      "learning_rate": 4.775678015263708e-07,
+      "logits/chosen": -2.5736076831817627,
+      "logits/rejected": -2.5460524559020996,
+      "logps/chosen": -223.2324981689453,
+      "logps/rejected": -245.54092407226562,
+      "loss": 0.6081,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.09774526208639145,
+      "rewards/margins": 0.38662397861480713,
+      "rewards/rejected": -0.2888787090778351,
+      "step": 390
+    },
+    {
+      "epoch": 0.23,
+      "grad_norm": 28.055639097927603,
+      "learning_rate": 4.7545813160127845e-07,
+      "logits/chosen": -2.6072590351104736,
+      "logits/rejected": -2.505096912384033,
+      "logps/chosen": -236.22549438476562,
+      "logps/rejected": -265.6544494628906,
+      "loss": 0.5967,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 0.02556512877345085,
+      "rewards/margins": 0.45649051666259766,
+      "rewards/rejected": -0.4309254288673401,
+      "step": 400
+    },
+    {
+      "epoch": 0.23,
+      "eval_logits/chosen": -2.519050121307373,
+      "eval_logits/rejected": -2.4576427936553955,
+      "eval_logps/chosen": -247.5235595703125,
+      "eval_logps/rejected": -278.0496826171875,
+      "eval_loss": 0.5991776585578918,
+      "eval_rewards/accuracies": 0.678205132484436,
+      "eval_rewards/chosen": -0.1314685195684433,
+      "eval_rewards/margins": 0.38664132356643677,
+      "eval_rewards/rejected": -0.5181097984313965,
+      "eval_runtime": 861.5357,
+      "eval_samples_per_second": 14.452,
+      "eval_steps_per_second": 0.453,
+      "step": 400
+    },
+    {
+      "epoch": 0.23,
+      "grad_norm": 26.14630535783879,
+      "learning_rate": 4.732587621532214e-07,
+      "logits/chosen": -2.478102922439575,
+      "logits/rejected": -2.438175678253174,
+      "logps/chosen": -250.90457153320312,
+      "logps/rejected": -284.3564453125,
+      "loss": 0.5827,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.15595874190330505,
+      "rewards/margins": 0.3831656873226166,
+      "rewards/rejected": -0.5391243696212769,
+      "step": 410
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 28.031713354054272,
+      "learning_rate": 4.709705682111412e-07,
+      "logits/chosen": -2.477914810180664,
+      "logits/rejected": -2.4634792804718018,
+      "logps/chosen": -262.4942321777344,
+      "logps/rejected": -290.16009521484375,
+      "loss": 0.5869,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.22279219329357147,
+      "rewards/margins": 0.4437895715236664,
+      "rewards/rejected": -0.6665817499160767,
+      "step": 420
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 25.747403286133764,
+      "learning_rate": 4.68594460143201e-07,
+      "logits/chosen": -2.5019423961639404,
+      "logits/rejected": -2.429969310760498,
+      "logps/chosen": -256.59185791015625,
+      "logps/rejected": -283.1378173828125,
+      "loss": 0.5936,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.14861060678958893,
+      "rewards/margins": 0.4206705093383789,
+      "rewards/rejected": -0.5692810416221619,
+      "step": 430
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 22.31072442106435,
+      "learning_rate": 4.661313832945903e-07,
+      "logits/chosen": -2.504993438720703,
+      "logits/rejected": -2.467900037765503,
+      "logps/chosen": -254.93057250976562,
+      "logps/rejected": -312.25250244140625,
+      "loss": 0.5925,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.15858420729637146,
+      "rewards/margins": 0.5125812292098999,
+      "rewards/rejected": -0.671165406703949,
+      "step": 440
+    },
+    {
+      "epoch": 0.26,
+      "grad_norm": 28.65488774525744,
+      "learning_rate": 4.635823176114162e-07,
+      "logits/chosen": -2.5073506832122803,
+      "logits/rejected": -2.472411870956421,
+      "logps/chosen": -284.4822998046875,
+      "logps/rejected": -320.66925048828125,
+      "loss": 0.5817,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.37685471773147583,
+      "rewards/margins": 0.3860743045806885,
+      "rewards/rejected": -0.7629289627075195,
+      "step": 450
+    },
+    {
+      "epoch": 0.26,
+      "grad_norm": 34.815824611343665,
+      "learning_rate": 4.6094827725082684e-07,
+      "logits/chosen": -2.5682735443115234,
+      "logits/rejected": -2.501211166381836,
+      "logps/chosen": -265.11126708984375,
+      "logps/rejected": -309.05157470703125,
+      "loss": 0.5814,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.2719506323337555,
+      "rewards/margins": 0.5007379055023193,
+      "rewards/rejected": -0.7726885080337524,
+      "step": 460
+    },
+    {
+      "epoch": 0.27,
+      "grad_norm": 26.238405483406037,
+      "learning_rate": 4.582303101775248e-07,
+      "logits/chosen": -2.62959623336792,
+      "logits/rejected": -2.562079906463623,
+      "logps/chosen": -251.88674926757812,
+      "logps/rejected": -288.1053771972656,
+      "loss": 0.5955,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.16994395852088928,
+      "rewards/margins": 0.4775424599647522,
+      "rewards/rejected": -0.6474864482879639,
+      "step": 470
+    },
+    {
+      "epoch": 0.27,
+      "grad_norm": 30.50178144367043,
+      "learning_rate": 4.5542949774682956e-07,
+      "logits/chosen": -2.568668842315674,
+      "logits/rejected": -2.5604805946350098,
+      "logps/chosen": -267.90655517578125,
+      "logps/rejected": -290.76019287109375,
+      "loss": 0.5991,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.22079896926879883,
+      "rewards/margins": 0.31227344274520874,
+      "rewards/rejected": -0.5330723524093628,
+      "step": 480
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 27.535568193967393,
+      "learning_rate": 4.5254695427445517e-07,
+      "logits/chosen": -2.637202501296997,
+      "logits/rejected": -2.6354382038116455,
+      "logps/chosen": -244.74551391601562,
+      "logps/rejected": -290.29644775390625,
+      "loss": 0.5647,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": 0.011266368441283703,
+      "rewards/margins": 0.5976118445396423,
+      "rewards/rejected": -0.5863455533981323,
+      "step": 490
+    },
+    {
+      "epoch": 0.29,
+      "grad_norm": 30.285379350904805,
+      "learning_rate": 4.4958382659317536e-07,
+      "logits/chosen": -2.740764617919922,
+      "logits/rejected": -2.7210445404052734,
+      "logps/chosen": -272.13702392578125,
+      "logps/rejected": -297.4922790527344,
+      "loss": 0.6216,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.258303165435791,
+      "rewards/margins": 0.32254815101623535,
+      "rewards/rejected": -0.5808513760566711,
+      "step": 500
+    },
+    {
+      "epoch": 0.29,
+      "eval_logits/chosen": -2.7310774326324463,
+      "eval_logits/rejected": -2.687927484512329,
+      "eval_logps/chosen": -238.078125,
+      "eval_logps/rejected": -267.6940002441406,
+      "eval_loss": 0.5941099524497986,
+      "eval_rewards/accuracies": 0.6721153855323792,
+      "eval_rewards/chosen": -0.037014104425907135,
+      "eval_rewards/margins": 0.37753891944885254,
+      "eval_rewards/rejected": -0.4145529866218567,
+      "eval_runtime": 858.9897,
+      "eval_samples_per_second": 14.495,
+      "eval_steps_per_second": 0.454,
+      "step": 500
+    },
+    {
+      "epoch": 0.29,
+      "grad_norm": 23.512447045863215,
+      "learning_rate": 4.465412935965506e-07,
+      "logits/chosen": -2.7190139293670654,
+      "logits/rejected": -2.675178050994873,
+      "logps/chosen": -217.1962432861328,
+      "logps/rejected": -246.1102294921875,
+      "loss": 0.6138,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": 0.14496248960494995,
+      "rewards/margins": 0.36382409930229187,
+      "rewards/rejected": -0.21886160969734192,
+      "step": 510
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 23.621464405914903,
+      "learning_rate": 4.434205657698999e-07,
+      "logits/chosen": -2.568403720855713,
+      "logits/rejected": -2.551494836807251,
+      "logps/chosen": -224.2484588623047,
+      "logps/rejected": -278.65936279296875,
+      "loss": 0.6125,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.11122129112482071,
+      "rewards/margins": 0.36189281940460205,
+      "rewards/rejected": -0.47311410307884216,
+      "step": 520
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 23.359399767763318,
+      "learning_rate": 4.402228847087046e-07,
+      "logits/chosen": -2.5537118911743164,
+      "logits/rejected": -2.502847194671631,
+      "logps/chosen": -261.56927490234375,
+      "logps/rejected": -274.5636291503906,
+      "loss": 0.622,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.15581832826137543,
+      "rewards/margins": 0.2899227440357208,
+      "rewards/rejected": -0.44574108719825745,
+      "step": 530
+    },
+    {
+      "epoch": 0.31,
+      "grad_norm": 33.81353805062806,
+      "learning_rate": 4.36949522624633e-07,
+      "logits/chosen": -2.578953742980957,
+      "logits/rejected": -2.547420024871826,
+      "logps/chosen": -236.4100341796875,
+      "logps/rejected": -284.703857421875,
+      "loss": 0.5917,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.17350006103515625,
+      "rewards/margins": 0.42752647399902344,
+      "rewards/rejected": -0.6010265350341797,
+      "step": 540
+    },
+    {
+      "epoch": 0.31,
+      "grad_norm": 25.836827126785362,
+      "learning_rate": 4.33601781839386e-07,
+      "logits/chosen": -2.516505718231201,
+      "logits/rejected": -2.4714202880859375,
+      "logps/chosen": -271.7455139160156,
+      "logps/rejected": -320.60003662109375,
+      "loss": 0.583,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.1934550553560257,
+      "rewards/margins": 0.46987539529800415,
+      "rewards/rejected": -0.6633304357528687,
+      "step": 550
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 29.317285148378573,
+      "learning_rate": 4.301809942665625e-07,
+      "logits/chosen": -2.5027713775634766,
+      "logits/rejected": -2.4169793128967285,
+      "logps/chosen": -247.04306030273438,
+      "logps/rejected": -263.53033447265625,
+      "loss": 0.6041,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.13370783627033234,
+      "rewards/margins": 0.29323458671569824,
+      "rewards/rejected": -0.4269424080848694,
+      "step": 560
+    },
+    {
+      "epoch": 0.33,
+      "grad_norm": 23.273024353193524,
+      "learning_rate": 4.2668852088175145e-07,
+      "logits/chosen": -2.429138660430908,
+      "logits/rejected": -2.4112179279327393,
+      "logps/chosen": -261.13232421875,
+      "logps/rejected": -292.0643005371094,
+      "loss": 0.613,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.12648805975914001,
+      "rewards/margins": 0.3239407241344452,
+      "rewards/rejected": -0.4504288136959076,
+      "step": 570
+    },
+    {
+      "epoch": 0.33,
+      "grad_norm": 27.396331045603723,
+      "learning_rate": 4.231257511810618e-07,
+      "logits/chosen": -2.55159068107605,
+      "logits/rejected": -2.4876770973205566,
+      "logps/chosen": -251.07192993164062,
+      "logps/rejected": -293.2655334472656,
+      "loss": 0.6128,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.010355165228247643,
+      "rewards/margins": 0.33735883235931396,
+      "rewards/rejected": -0.34771400690078735,
+      "step": 580
+    },
+    {
+      "epoch": 0.34,
+      "grad_norm": 22.123964853357307,
+      "learning_rate": 4.1949410262830523e-07,
+      "logits/chosen": -2.5525479316711426,
+      "logits/rejected": -2.4439752101898193,
+      "logps/chosen": -233.11337280273438,
+      "logps/rejected": -262.0377502441406,
+      "loss": 0.5785,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.024647405371069908,
+      "rewards/margins": 0.3654420077800751,
+      "rewards/rejected": -0.3900894522666931,
+      "step": 590
+    },
+    {
+      "epoch": 0.34,
+      "grad_norm": 29.777775051728057,
+      "learning_rate": 4.1579502009105164e-07,
+      "logits/chosen": -2.488654613494873,
+      "logits/rejected": -2.4201626777648926,
+      "logps/chosen": -246.08486938476562,
+      "logps/rejected": -273.70709228515625,
+      "loss": 0.5919,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.1482144594192505,
+      "rewards/margins": 0.40206265449523926,
+      "rewards/rejected": -0.5502771139144897,
+      "step": 600
+    },
+    {
+      "epoch": 0.34,
+      "eval_logits/chosen": -2.47454833984375,
+      "eval_logits/rejected": -2.404360294342041,
+      "eval_logps/chosen": -249.46990966796875,
+      "eval_logps/rejected": -283.90716552734375,
+      "eval_loss": 0.5903669595718384,
+      "eval_rewards/accuracies": 0.6865384578704834,
+      "eval_rewards/chosen": -0.15093201398849487,
+      "eval_rewards/margins": 0.42575252056121826,
+      "eval_rewards/rejected": -0.5766844749450684,
+      "eval_runtime": 858.5855,
+      "eval_samples_per_second": 14.502,
+      "eval_steps_per_second": 0.454,
+      "step": 600
+    },
+    {
+      "epoch": 0.35,
+      "grad_norm": 27.691202685227417,
+      "learning_rate": 4.120299752657827e-07,
+      "logits/chosen": -2.5120370388031006,
+      "logits/rejected": -2.4610724449157715,
+      "logps/chosen": -261.5104064941406,
+      "logps/rejected": -296.66217041015625,
+      "loss": 0.592,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.18553677201271057,
+      "rewards/margins": 0.3792334496974945,
+      "rewards/rejected": -0.5647702217102051,
+      "step": 610
+    },
+    {
+      "epoch": 0.35,
+      "grad_norm": 24.36661678943314,
+      "learning_rate": 4.082004660923702e-07,
+      "logits/chosen": -2.4631857872009277,
+      "logits/rejected": -2.373511791229248,
+      "logps/chosen": -248.58895874023438,
+      "logps/rejected": -271.892333984375,
+      "loss": 0.5945,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.06783659011125565,
+      "rewards/margins": 0.3515480160713196,
+      "rewards/rejected": -0.419384628534317,
+      "step": 620
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 23.60175971446507,
+      "learning_rate": 4.0430801615811437e-07,
+      "logits/chosen": -2.4521324634552,
+      "logits/rejected": -2.4140613079071045,
+      "logps/chosen": -231.697021484375,
+      "logps/rejected": -277.6011962890625,
+      "loss": 0.5938,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.045631490647792816,
+      "rewards/margins": 0.44818124175071716,
+      "rewards/rejected": -0.4938127398490906,
+      "step": 630
+    },
+    {
+      "epoch": 0.37,
+      "grad_norm": 28.337765984985374,
+      "learning_rate": 4.00354174091578e-07,
+      "logits/chosen": -2.4540791511535645,
+      "logits/rejected": -2.3845746517181396,
+      "logps/chosen": -229.52133178710938,
+      "logps/rejected": -277.82452392578125,
+      "loss": 0.6079,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.0009828682523220778,
+      "rewards/margins": 0.471419095993042,
+      "rewards/rejected": -0.47240200638771057,
+      "step": 640
+    },
+    {
+      "epoch": 0.37,
+      "grad_norm": 27.293175260711642,
+      "learning_rate": 3.963405129464569e-07,
+      "logits/chosen": -2.3416523933410645,
+      "logits/rejected": -2.244310140609741,
+      "logps/chosen": -261.36871337890625,
+      "logps/rejected": -316.395263671875,
+      "loss": 0.5519,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.22408190369606018,
+      "rewards/margins": 0.5848034024238586,
+      "rewards/rejected": -0.8088852167129517,
+      "step": 650
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 27.68918047728001,
+      "learning_rate": 3.922686295757347e-07,
+      "logits/chosen": -2.336780071258545,
+      "logits/rejected": -2.2526936531066895,
+      "logps/chosen": -301.7099609375,
+      "logps/rejected": -356.14056396484375,
+      "loss": 0.5933,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.42652687430381775,
+      "rewards/margins": 0.5547691583633423,
+      "rewards/rejected": -0.9812959432601929,
+      "step": 660
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 24.193971795815184,
+      "learning_rate": 3.881401439963666e-07,
+      "logits/chosen": -2.272087574005127,
+      "logits/rejected": -2.220580577850342,
+      "logps/chosen": -229.0252227783203,
+      "logps/rejected": -266.34405517578125,
+      "loss": 0.5979,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.23234577476978302,
+      "rewards/margins": 0.3420170247554779,
+      "rewards/rejected": -0.5743628144264221,
+      "step": 670
+    },
+    {
+      "epoch": 0.39,
+      "grad_norm": 33.515555503395184,
+      "learning_rate": 3.839566987447491e-07,
+      "logits/chosen": -2.345500946044922,
+      "logits/rejected": -2.2792842388153076,
+      "logps/chosen": -268.81317138671875,
+      "logps/rejected": -309.93463134765625,
+      "loss": 0.6034,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.14895500242710114,
+      "rewards/margins": 0.49111080169677734,
+      "rewards/rejected": -0.6400657892227173,
+      "step": 680
+    },
+    {
+      "epoch": 0.39,
+      "grad_norm": 22.584232277933243,
+      "learning_rate": 3.7971995822322947e-07,
+      "logits/chosen": -2.27201247215271,
+      "logits/rejected": -2.226531505584717,
+      "logps/chosen": -262.95782470703125,
+      "logps/rejected": -290.07696533203125,
+      "loss": 0.6172,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.2456875741481781,
+      "rewards/margins": 0.29335761070251465,
+      "rewards/rejected": -0.5390451550483704,
+      "step": 690
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 28.603695886249437,
+      "learning_rate": 3.7543160803791545e-07,
+      "logits/chosen": -2.301879644393921,
+      "logits/rejected": -2.223996162414551,
+      "logps/chosen": -261.08795166015625,
+      "logps/rejected": -307.17010498046875,
+      "loss": 0.5769,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.22030536830425262,
+      "rewards/margins": 0.49444580078125,
+      "rewards/rejected": -0.714751124382019,
+      "step": 700
+    },
+    {
+      "epoch": 0.4,
+      "eval_logits/chosen": -2.2923786640167236,
+      "eval_logits/rejected": -2.2189748287200928,
+      "eval_logps/chosen": -258.4495849609375,
+      "eval_logps/rejected": -292.7128601074219,
+      "eval_loss": 0.5901564955711365,
+      "eval_rewards/accuracies": 0.6772435903549194,
+      "eval_rewards/chosen": -0.24072882533073425,
+      "eval_rewards/margins": 0.4240129590034485,
+      "eval_rewards/rejected": -0.6647418141365051,
+      "eval_runtime": 860.8912,
+      "eval_samples_per_second": 14.463,
+      "eval_steps_per_second": 0.453,
+      "step": 700
+    },
+    {
+      "epoch": 0.41,
+      "grad_norm": 26.031424540267988,
+      "learning_rate": 3.7109335432805e-07,
+      "logits/chosen": -2.2880516052246094,
+      "logits/rejected": -2.229980707168579,
+      "logps/chosen": -265.29754638671875,
+      "logps/rejected": -275.8728942871094,
+      "loss": 0.6037,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.11812520027160645,
+      "rewards/margins": 0.3968770205974579,
+      "rewards/rejected": -0.5150021910667419,
+      "step": 710
+    },
+    {
+      "epoch": 0.41,
+      "grad_norm": 22.719027068474563,
+      "learning_rate": 3.667069230872155e-07,
+      "logits/chosen": -2.254481554031372,
+      "logits/rejected": -2.231606960296631,
+      "logps/chosen": -259.39422607421875,
+      "logps/rejected": -272.6926574707031,
+      "loss": 0.5933,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.046945445239543915,
+      "rewards/margins": 0.3208449184894562,
+      "rewards/rejected": -0.3677903413772583,
+      "step": 720
+    },
+    {
+      "epoch": 0.42,
+      "grad_norm": 29.374605264798323,
+      "learning_rate": 3.6227405947664003e-07,
+      "logits/chosen": -2.2828311920166016,
+      "logits/rejected": -2.2138302326202393,
+      "logps/chosen": -256.0699768066406,
+      "logps/rejected": -295.7370910644531,
+      "loss": 0.5858,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.0820874497294426,
+      "rewards/margins": 0.5297425985336304,
+      "rewards/rejected": -0.6118300557136536,
+      "step": 730
+    },
+    {
+      "epoch": 0.42,
+      "grad_norm": 24.91144032228272,
+      "learning_rate": 3.577965271308771e-07,
+      "logits/chosen": -2.2585289478302,
+      "logits/rejected": -2.201707124710083,
+      "logps/chosen": -266.5726318359375,
+      "logps/rejected": -301.1708679199219,
+      "loss": 0.5891,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.2869926691055298,
+      "rewards/margins": 0.4772910475730896,
+      "rewards/rejected": -0.7642836570739746,
+      "step": 740
+    },
+    {
+      "epoch": 0.43,
+      "grad_norm": 31.59826426014115,
+      "learning_rate": 3.5327610745613546e-07,
+      "logits/chosen": -2.3542895317077637,
+      "logits/rejected": -2.242069721221924,
+      "logps/chosen": -250.84194946289062,
+      "logps/rejected": -285.4515380859375,
+      "loss": 0.5963,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.19332554936408997,
+      "rewards/margins": 0.4167536795139313,
+      "rewards/rejected": -0.6100791692733765,
+      "step": 750
+    },
+    {
+      "epoch": 0.43,
+      "grad_norm": 25.89736672152189,
+      "learning_rate": 3.487145989215391e-07,
+      "logits/chosen": -2.383197069168091,
+      "logits/rejected": -2.3221888542175293,
+      "logps/chosen": -251.72390747070312,
+      "logps/rejected": -282.80804443359375,
+      "loss": 0.5814,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.09259854257106781,
+      "rewards/margins": 0.45896273851394653,
+      "rewards/rejected": -0.5515612363815308,
+      "step": 760
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 24.879610991566807,
+      "learning_rate": 3.4411381634359796e-07,
+      "logits/chosen": -2.389742612838745,
+      "logits/rejected": -2.3619580268859863,
+      "logps/chosen": -255.8960418701172,
+      "logps/rejected": -290.5157165527344,
+      "loss": 0.5646,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.1797618567943573,
+      "rewards/margins": 0.44017988443374634,
+      "rewards/rejected": -0.6199417114257812,
+      "step": 770
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 26.915784595282116,
+      "learning_rate": 3.3947559016417554e-07,
+      "logits/chosen": -2.4469265937805176,
+      "logits/rejected": -2.4010143280029297,
+      "logps/chosen": -261.87322998046875,
+      "logps/rejected": -302.0085754394531,
+      "loss": 0.6045,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.26690053939819336,
+      "rewards/margins": 0.33789390325546265,
+      "rewards/rejected": -0.6047943830490112,
+      "step": 780
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 27.76077206811895,
+      "learning_rate": 3.3480176572223885e-07,
+      "logits/chosen": -2.5314059257507324,
+      "logits/rejected": -2.4485652446746826,
+      "logps/chosen": -238.18112182617188,
+      "logps/rejected": -297.2239074707031,
+      "loss": 0.554,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.199807807803154,
+      "rewards/margins": 0.6575660705566406,
+      "rewards/rejected": -0.857373833656311,
+      "step": 790
+    },
+    {
+      "epoch": 0.46,
+      "grad_norm": 24.226296856947638,
+      "learning_rate": 3.300942025196824e-07,
+      "logits/chosen": -2.587006092071533,
+      "logits/rejected": -2.541341781616211,
+      "logps/chosen": -256.3536376953125,
+      "logps/rejected": -278.5965270996094,
+      "loss": 0.5725,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.13480545580387115,
+      "rewards/margins": 0.4779321551322937,
+      "rewards/rejected": -0.6127376556396484,
+      "step": 800
+    },
+    {
+      "epoch": 0.46,
+      "eval_logits/chosen": -2.573244094848633,
+      "eval_logits/rejected": -2.527627468109131,
+      "eval_logps/chosen": -238.99398803710938,
+      "eval_logps/rejected": -274.5382995605469,
+      "eval_loss": 0.5882474780082703,
+      "eval_rewards/accuracies": 0.6836538314819336,
+      "eval_rewards/chosen": -0.04617273062467575,
+      "eval_rewards/margins": 0.43682336807250977,
+      "eval_rewards/rejected": -0.4829961061477661,
+      "eval_runtime": 862.9985,
+      "eval_samples_per_second": 14.428,
+      "eval_steps_per_second": 0.452,
+      "step": 800
+    },
+    {
+      "epoch": 0.46,
+      "grad_norm": 26.141416208141568,
+      "learning_rate": 3.253547734815173e-07,
+      "logits/chosen": -2.607003688812256,
+      "logits/rejected": -2.554546594619751,
+      "logps/chosen": -232.52743530273438,
+      "logps/rejected": -314.36346435546875,
+      "loss": 0.5801,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.0894065648317337,
+      "rewards/margins": 0.6495140790939331,
+      "rewards/rejected": -0.5601075887680054,
+      "step": 810
+    },
+    {
+      "epoch": 0.47,
+      "grad_norm": 26.097129519417752,
+      "learning_rate": 3.2058536421071914e-07,
+      "logits/chosen": -2.5027403831481934,
+      "logits/rejected": -2.471726417541504,
+      "logps/chosen": -257.47320556640625,
+      "logps/rejected": -263.6692810058594,
+      "loss": 0.5805,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": 0.03840960934758186,
+      "rewards/margins": 0.3363339304924011,
+      "rewards/rejected": -0.29792430996894836,
+      "step": 820
+    },
+    {
+      "epoch": 0.47,
+      "grad_norm": 25.23275414096734,
+      "learning_rate": 3.1578787223803294e-07,
+      "logits/chosen": -2.426281452178955,
+      "logits/rejected": -2.399183750152588,
+      "logps/chosen": -272.22869873046875,
+      "logps/rejected": -290.59649658203125,
+      "loss": 0.6043,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.32681387662887573,
+      "rewards/margins": 0.41791266202926636,
+      "rewards/rejected": -0.7447265386581421,
+      "step": 830
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 24.11509059960286,
+      "learning_rate": 3.109642062670318e-07,
+      "logits/chosen": -2.5103323459625244,
+      "logits/rejected": -2.4195475578308105,
+      "logps/chosen": -244.50863647460938,
+      "logps/rejected": -288.40069580078125,
+      "loss": 0.5889,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.2207474410533905,
+      "rewards/margins": 0.4590122699737549,
+      "rewards/rejected": -0.6797597408294678,
+      "step": 840
+    },
+    {
+      "epoch": 0.49,
+      "grad_norm": 33.37568508664132,
+      "learning_rate": 3.0611628541473074e-07,
+      "logits/chosen": -2.497424602508545,
+      "logits/rejected": -2.472369432449341,
+      "logps/chosen": -266.05230712890625,
+      "logps/rejected": -307.9107360839844,
+      "loss": 0.6051,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": 0.008460281416773796,
+      "rewards/margins": 0.4078546166419983,
+      "rewards/rejected": -0.39939433336257935,
+      "step": 850
+    },
+    {
+      "epoch": 0.49,
+      "grad_norm": 24.365317735651118,
+      "learning_rate": 3.012460384480576e-07,
+      "logits/chosen": -2.4374663829803467,
+      "logits/rejected": -2.3599624633789062,
+      "logps/chosen": -259.6045837402344,
+      "logps/rejected": -302.10345458984375,
+      "loss": 0.5678,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.18751467764377594,
+      "rewards/margins": 0.49369174242019653,
+      "rewards/rejected": -0.6812063455581665,
+      "step": 860
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 22.6057869532,
+      "learning_rate": 2.963554030164842e-07,
+      "logits/chosen": -2.2620365619659424,
+      "logits/rejected": -2.280235767364502,
+      "logps/chosen": -270.291748046875,
+      "logps/rejected": -298.3429870605469,
+      "loss": 0.6028,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.1897084265947342,
+      "rewards/margins": 0.3970777988433838,
+      "rewards/rejected": -0.5867862105369568,
+      "step": 870
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 26.035338300340943,
+      "learning_rate": 2.9144632488112416e-07,
+      "logits/chosen": -2.274094343185425,
+      "logits/rejected": -2.2430529594421387,
+      "logps/chosen": -270.6689758300781,
+      "logps/rejected": -314.1228942871094,
+      "loss": 0.5872,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.21771720051765442,
+      "rewards/margins": 0.5008502006530762,
+      "rewards/rejected": -0.7185673713684082,
+      "step": 880
+    },
+    {
+      "epoch": 0.51,
+      "grad_norm": 22.12211938779161,
+      "learning_rate": 2.865207571406029e-07,
+      "logits/chosen": -2.3526949882507324,
+      "logits/rejected": -2.253375768661499,
+      "logps/chosen": -269.4740905761719,
+      "logps/rejected": -295.1158752441406,
+      "loss": 0.5956,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.25454986095428467,
+      "rewards/margins": 0.40211352705955505,
+      "rewards/rejected": -0.6566634774208069,
+      "step": 890
+    },
+    {
+      "epoch": 0.51,
+      "grad_norm": 26.967953686731473,
+      "learning_rate": 2.815806594540089e-07,
+      "logits/chosen": -2.3986849784851074,
+      "logits/rejected": -2.3849453926086426,
+      "logps/chosen": -241.43505859375,
+      "logps/rejected": -276.5702819824219,
+      "loss": 0.5814,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.002503329422324896,
+      "rewards/margins": 0.3857470154762268,
+      "rewards/rejected": -0.38825035095214844,
+      "step": 900
+    },
+    {
+      "epoch": 0.51,
+      "eval_logits/chosen": -2.409844160079956,
+      "eval_logits/rejected": -2.335495948791504,
+      "eval_logps/chosen": -246.1585693359375,
+      "eval_logps/rejected": -279.9914245605469,
+      "eval_loss": 0.5864266753196716,
+      "eval_rewards/accuracies": 0.6810897588729858,
+      "eval_rewards/chosen": -0.11781882494688034,
+      "eval_rewards/margins": 0.41970840096473694,
+      "eval_rewards/rejected": -0.5375272035598755,
+      "eval_runtime": 861.4542,
+      "eval_samples_per_second": 14.453,
+      "eval_steps_per_second": 0.453,
+      "step": 900
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 24.93686478364852,
+      "learning_rate": 2.7662799726123454e-07,
+      "logits/chosen": -2.364938974380493,
+      "logits/rejected": -2.3040971755981445,
+      "logps/chosen": -242.66366577148438,
+      "logps/rejected": -277.5296936035156,
+      "loss": 0.6005,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.25096192955970764,
+      "rewards/margins": 0.2948233187198639,
+      "rewards/rejected": -0.5457853078842163,
+      "step": 910
+    },
+    {
+      "epoch": 0.53,
+      "grad_norm": 29.741661581034126,
+      "learning_rate": 2.7166474100101674e-07,
+      "logits/chosen": -2.362717390060425,
+      "logits/rejected": -2.29923677444458,
+      "logps/chosen": -260.1564636230469,
+      "logps/rejected": -299.52142333984375,
+      "loss": 0.5972,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.2629412114620209,
+      "rewards/margins": 0.4658204913139343,
+      "rewards/rejected": -0.7287617325782776,
+      "step": 920
+    },
+    {
+      "epoch": 0.53,
+      "grad_norm": 23.70048734007165,
+      "learning_rate": 2.6669286532698964e-07,
+      "logits/chosen": -2.4182770252227783,
+      "logits/rejected": -2.377297878265381,
+      "logps/chosen": -272.5760192871094,
+      "logps/rejected": -307.6293640136719,
+      "loss": 0.5734,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.10112947225570679,
+      "rewards/margins": 0.5127297639846802,
+      "rewards/rejected": -0.6138591170310974,
+      "step": 930
+    },
+    {
+      "epoch": 0.54,
+      "grad_norm": 26.612920994033377,
+      "learning_rate": 2.6171434832205953e-07,
+      "logits/chosen": -2.4122025966644287,
+      "logits/rejected": -2.3759803771972656,
+      "logps/chosen": -256.7776184082031,
+      "logps/rejected": -289.4403381347656,
+      "loss": 0.5657,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.20392732322216034,
+      "rewards/margins": 0.4127303957939148,
+      "rewards/rejected": -0.6166577339172363,
+      "step": 940
+    },
+    {
+      "epoch": 0.54,
+      "grad_norm": 31.14720620364731,
+      "learning_rate": 2.567311707114157e-07,
+      "logits/chosen": -2.4095797538757324,
+      "logits/rejected": -2.354799747467041,
+      "logps/chosen": -229.2027130126953,
+      "logps/rejected": -283.2561340332031,
+      "loss": 0.604,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.197126567363739,
+      "rewards/margins": 0.4597938060760498,
+      "rewards/rejected": -0.6569203734397888,
+      "step": 950
+    },
+    {
+      "epoch": 0.55,
+      "grad_norm": 27.00204010664385,
+      "learning_rate": 2.5174531507449037e-07,
+      "logits/chosen": -2.4471428394317627,
+      "logits/rejected": -2.338510513305664,
+      "logps/chosen": -234.0675506591797,
+      "logps/rejected": -259.7816467285156,
+      "loss": 0.5899,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.190177783370018,
+      "rewards/margins": 0.3870656490325928,
+      "rewards/rejected": -0.577243447303772,
+      "step": 960
+    },
+    {
+      "epoch": 0.55,
+      "grad_norm": 19.932243905164242,
+      "learning_rate": 2.4675876505618045e-07,
+      "logits/chosen": -2.4684808254241943,
+      "logits/rejected": -2.4519195556640625,
+      "logps/chosen": -262.174560546875,
+      "logps/rejected": -282.4520263671875,
+      "loss": 0.6019,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.06612774729728699,
+      "rewards/margins": 0.45484066009521484,
+      "rewards/rejected": -0.5209684371948242,
+      "step": 970
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 26.495466395218312,
+      "learning_rate": 2.4177350457764527e-07,
+      "logits/chosen": -2.53678822517395,
+      "logits/rejected": -2.4713311195373535,
+      "logps/chosen": -253.9574432373047,
+      "logps/rejected": -298.145263671875,
+      "loss": 0.6026,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.07538889348506927,
+      "rewards/margins": 0.39861345291137695,
+      "rewards/rejected": -0.47400230169296265,
+      "step": 980
+    },
+    {
+      "epoch": 0.57,
+      "grad_norm": 24.86682957992287,
+      "learning_rate": 2.367915170469956e-07,
+      "logits/chosen": -2.4735522270202637,
+      "logits/rejected": -2.337954044342041,
+      "logps/chosen": -256.18963623046875,
+      "logps/rejected": -267.07305908203125,
+      "loss": 0.5731,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.04039744660258293,
+      "rewards/margins": 0.35975947976112366,
+      "rewards/rejected": -0.4001569151878357,
+      "step": 990
+    },
+    {
+      "epoch": 0.57,
+      "grad_norm": 32.9954052513008,
+      "learning_rate": 2.3181478457018475e-07,
+      "logits/chosen": -2.4602394104003906,
+      "logits/rejected": -2.372180700302124,
+      "logps/chosen": -259.4634704589844,
+      "logps/rejected": -298.13836669921875,
+      "loss": 0.5514,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.039340369403362274,
+      "rewards/margins": 0.5396673083305359,
+      "rewards/rejected": -0.5790076851844788,
+      "step": 1000
+    },
+    {
+      "epoch": 0.57,
+      "eval_logits/chosen": -2.4855430126190186,
+      "eval_logits/rejected": -2.411501169204712,
+      "eval_logps/chosen": -252.6514892578125,
+      "eval_logps/rejected": -291.29022216796875,
+      "eval_loss": 0.5839190483093262,
+      "eval_rewards/accuracies": 0.6871795058250427,
+      "eval_rewards/chosen": -0.18274781107902527,
+      "eval_rewards/margins": 0.4677676856517792,
+      "eval_rewards/rejected": -0.6505153775215149,
+      "eval_runtime": 844.1499,
+      "eval_samples_per_second": 14.75,
+      "eval_steps_per_second": 0.462,
+      "step": 1000
+    },
+    {
+      "epoch": 0.58,
+      "grad_norm": 33.832396038055975,
+      "learning_rate": 2.2684528716241968e-07,
+      "logits/chosen": -2.528446674346924,
+      "logits/rejected": -2.4585366249084473,
+      "logps/chosen": -278.91461181640625,
+      "logps/rejected": -327.67449951171875,
+      "loss": 0.5971,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.20916755497455597,
+      "rewards/margins": 0.4697844088077545,
+      "rewards/rejected": -0.6789519190788269,
+      "step": 1010
+    },
+    {
+      "epoch": 0.58,
+      "grad_norm": 23.559014104508705,
+      "learning_rate": 2.218850019604028e-07,
+      "logits/chosen": -2.5937182903289795,
+      "logits/rejected": -2.506432056427002,
+      "logps/chosen": -244.5588836669922,
+      "logps/rejected": -312.0033264160156,
+      "loss": 0.5622,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.12260422855615616,
+      "rewards/margins": 0.6369132399559021,
+      "rewards/rejected": -0.7595174908638,
+      "step": 1020
+    },
+    {
+      "epoch": 0.59,
+      "grad_norm": 33.04728248607101,
+      "learning_rate": 2.1693590243571935e-07,
+      "logits/chosen": -2.641472816467285,
+      "logits/rejected": -2.5865638256073,
+      "logps/chosen": -279.6830139160156,
+      "logps/rejected": -306.81243896484375,
+      "loss": 0.5842,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.17163579165935516,
+      "rewards/margins": 0.36918142437934875,
+      "rewards/rejected": -0.5408172607421875,
+      "step": 1030
+    },
+    {
+      "epoch": 0.59,
+      "grad_norm": 28.35991011420196,
+      "learning_rate": 2.119999576096832e-07,
+      "logits/chosen": -2.623781681060791,
+      "logits/rejected": -2.5274269580841064,
+      "logps/chosen": -278.32916259765625,
+      "logps/rejected": -306.47198486328125,
+      "loss": 0.5772,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.28789210319519043,
+      "rewards/margins": 0.4908284544944763,
+      "rewards/rejected": -0.7787205576896667,
+      "step": 1040
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 24.13700126033885,
+      "learning_rate": 2.0707913126995244e-07,
+      "logits/chosen": -2.555406093597412,
+      "logits/rejected": -2.5062708854675293,
+      "logps/chosen": -302.45416259765625,
+      "logps/rejected": -314.2410583496094,
+      "loss": 0.579,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -0.26258963346481323,
+      "rewards/margins": 0.5782099962234497,
+      "rewards/rejected": -0.8407996296882629,
+      "step": 1050
+    },
+    {
+      "epoch": 0.61,
+      "grad_norm": 31.0299041852118,
+      "learning_rate": 2.021753811892281e-07,
+      "logits/chosen": -2.591373920440674,
+      "logits/rejected": -2.5653815269470215,
+      "logps/chosen": -267.46551513671875,
+      "logps/rejected": -289.0950012207031,
+      "loss": 0.5749,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.2024902105331421,
+      "rewards/margins": 0.5493400692939758,
+      "rewards/rejected": -0.7518302202224731,
+      "step": 1060
+    },
+    {
+      "epoch": 0.61,
+      "grad_norm": 27.46997764843006,
+      "learning_rate": 1.9729065834634528e-07,
+      "logits/chosen": -2.5627617835998535,
+      "logits/rejected": -2.4920713901519775,
+      "logps/chosen": -259.22216796875,
+      "logps/rejected": -298.4840393066406,
+      "loss": 0.5842,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.045971281826496124,
+      "rewards/margins": 0.47002631425857544,
+      "rewards/rejected": -0.515997588634491,
+      "step": 1070
+    },
+    {
+      "epoch": 0.62,
+      "grad_norm": 23.969955157596903,
+      "learning_rate": 1.924269061500678e-07,
+      "logits/chosen": -2.5113027095794678,
+      "logits/rejected": -2.500351667404175,
+      "logps/chosen": -237.98489379882812,
+      "logps/rejected": -280.7731628417969,
+      "loss": 0.5878,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.1136646643280983,
+      "rewards/margins": 0.36606425046920776,
+      "rewards/rejected": -0.4797288477420807,
+      "step": 1080
+    },
+    {
+      "epoch": 0.62,
+      "grad_norm": 23.516344403891786,
+      "learning_rate": 1.8758605966589434e-07,
+      "logits/chosen": -2.5660672187805176,
+      "logits/rejected": -2.5005881786346436,
+      "logps/chosen": -275.2914733886719,
+      "logps/rejected": -298.65155029296875,
+      "loss": 0.5762,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.09098437428474426,
+      "rewards/margins": 0.4184696078300476,
+      "rewards/rejected": -0.32748526334762573,
+      "step": 1090
+    },
+    {
+      "epoch": 0.63,
+      "grad_norm": 29.404877835754245,
+      "learning_rate": 1.8277004484618357e-07,
+      "logits/chosen": -2.5443501472473145,
+      "logits/rejected": -2.4919300079345703,
+      "logps/chosen": -245.58584594726562,
+      "logps/rejected": -276.38427734375,
+      "loss": 0.5946,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.019266966730356216,
+      "rewards/margins": 0.3828735649585724,
+      "rewards/rejected": -0.4021405279636383,
+      "step": 1100
+    },
+    {
+      "epoch": 0.63,
+      "eval_logits/chosen": -2.508995532989502,
+      "eval_logits/rejected": -2.447495937347412,
+      "eval_logps/chosen": -241.06724548339844,
+      "eval_logps/rejected": -277.4429626464844,
+      "eval_loss": 0.5846326351165771,
+      "eval_rewards/accuracies": 0.6846153736114502,
+      "eval_rewards/chosen": -0.06690550595521927,
+      "eval_rewards/margins": 0.4451369047164917,
+      "eval_rewards/rejected": -0.5120424032211304,
+      "eval_runtime": 860.4121,
+      "eval_samples_per_second": 14.471,
+      "eval_steps_per_second": 0.453,
+      "step": 1100
+    },
+    {
+      "epoch": 0.63,
+      "grad_norm": 28.170030221996313,
+      "learning_rate": 1.7798077776390603e-07,
+      "logits/chosen": -2.497800827026367,
+      "logits/rejected": -2.438910722732544,
+      "logps/chosen": -266.95526123046875,
+      "logps/rejected": -271.0229187011719,
+      "loss": 0.5981,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.20956678688526154,
+      "rewards/margins": 0.3521861732006073,
+      "rewards/rejected": -0.5617530345916748,
+      "step": 1110
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 25.64342328633492,
+      "learning_rate": 1.7322016385032536e-07,
+      "logits/chosen": -2.4583446979522705,
+      "logits/rejected": -2.400876522064209,
+      "logps/chosen": -265.85760498046875,
+      "logps/rejected": -305.25830078125,
+      "loss": 0.5854,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.011677969247102737,
+      "rewards/margins": 0.538432240486145,
+      "rewards/rejected": -0.5501102209091187,
+      "step": 1120
+    },
+    {
+      "epoch": 0.65,
+      "grad_norm": 29.29633909055343,
+      "learning_rate": 1.6849009713691454e-07,
+      "logits/chosen": -2.458902597427368,
+      "logits/rejected": -2.4362378120422363,
+      "logps/chosen": -273.21710205078125,
+      "logps/rejected": -307.6544494628906,
+      "loss": 0.5831,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.21327920258045197,
+      "rewards/margins": 0.44302114844322205,
+      "rewards/rejected": -0.65630042552948,
+      "step": 1130
+    },
+    {
+      "epoch": 0.65,
+      "grad_norm": 29.210383152583212,
+      "learning_rate": 1.6379245950180666e-07,
+      "logits/chosen": -2.4900028705596924,
+      "logits/rejected": -2.45780611038208,
+      "logps/chosen": -270.41424560546875,
+      "logps/rejected": -287.94708251953125,
+      "loss": 0.5648,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.17546476423740387,
+      "rewards/margins": 0.5030435919761658,
+      "rewards/rejected": -0.6785083413124084,
+      "step": 1140
+    },
+    {
+      "epoch": 0.66,
+      "grad_norm": 25.427016062934616,
+      "learning_rate": 1.591291199210813e-07,
+      "logits/chosen": -2.519883871078491,
+      "logits/rejected": -2.4721555709838867,
+      "logps/chosen": -239.39413452148438,
+      "logps/rejected": -290.2775573730469,
+      "loss": 0.5746,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.21366102993488312,
+      "rewards/margins": 0.43268680572509766,
+      "rewards/rejected": -0.6463478207588196,
+      "step": 1150
+    },
+    {
+      "epoch": 0.66,
+      "grad_norm": 27.841026179227654,
+      "learning_rate": 1.5450193372518438e-07,
+      "logits/chosen": -2.5061633586883545,
+      "logits/rejected": -2.446307420730591,
+      "logps/chosen": -254.13290405273438,
+      "logps/rejected": -282.93585205078125,
+      "loss": 0.5938,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.22907939553260803,
+      "rewards/margins": 0.40173858404159546,
+      "rewards/rejected": -0.6308180093765259,
+      "step": 1160
+    },
+    {
+      "epoch": 0.67,
+      "grad_norm": 23.462003347624545,
+      "learning_rate": 1.4991274186077628e-07,
+      "logits/chosen": -2.528425931930542,
+      "logits/rejected": -2.4988036155700684,
+      "logps/chosen": -256.7923583984375,
+      "logps/rejected": -287.1715393066406,
+      "loss": 0.5873,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.17279884219169617,
+      "rewards/margins": 0.4650405943393707,
+      "rewards/rejected": -0.6378394365310669,
+      "step": 1170
+    },
+    {
+      "epoch": 0.67,
+      "grad_norm": 29.33113372811353,
+      "learning_rate": 1.4536337015830323e-07,
+      "logits/chosen": -2.567692279815674,
+      "logits/rejected": -2.534449815750122,
+      "logps/chosen": -267.5567626953125,
+      "logps/rejected": -302.30535888671875,
+      "loss": 0.5849,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.20980314910411835,
+      "rewards/margins": 0.41117554903030396,
+      "rewards/rejected": -0.6209786534309387,
+      "step": 1180
+    },
+    {
+      "epoch": 0.68,
+      "grad_norm": 27.190688664117623,
+      "learning_rate": 1.4085562860558253e-07,
+      "logits/chosen": -2.571300506591797,
+      "logits/rejected": -2.5311543941497803,
+      "logps/chosen": -264.1678161621094,
+      "logps/rejected": -294.2384338378906,
+      "loss": 0.581,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.21912555396556854,
+      "rewards/margins": 0.4142730236053467,
+      "rewards/rejected": -0.6333985924720764,
+      "step": 1190
+    },
+    {
+      "epoch": 0.69,
+      "grad_norm": 29.84363481456159,
+      "learning_rate": 1.3639131062769075e-07,
+      "logits/chosen": -2.495572805404663,
+      "logits/rejected": -2.4769346714019775,
+      "logps/chosen": -259.7516784667969,
+      "logps/rejected": -298.76519775390625,
+      "loss": 0.5988,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.23917154967784882,
+      "rewards/margins": 0.4562154710292816,
+      "rewards/rejected": -0.695387065410614,
+      "step": 1200
+    },
+    {
+      "epoch": 0.69,
+      "eval_logits/chosen": -2.5292820930480957,
+      "eval_logits/rejected": -2.470348596572876,
+      "eval_logps/chosen": -261.1407775878906,
+      "eval_logps/rejected": -299.3863525390625,
+      "eval_loss": 0.5829187631607056,
+      "eval_rewards/accuracies": 0.6891025900840759,
+      "eval_rewards/chosen": -0.2676407992839813,
+      "eval_rewards/margins": 0.4638356566429138,
+      "eval_rewards/rejected": -0.7314765453338623,
+      "eval_runtime": 847.4482,
+      "eval_samples_per_second": 14.692,
+      "eval_steps_per_second": 0.46,
+      "step": 1200
+    },
+    {
+      "epoch": 0.69,
+      "grad_norm": 28.419557011703112,
+      "learning_rate": 1.3197219237344155e-07,
+      "logits/chosen": -2.5275566577911377,
+      "logits/rejected": -2.50520396232605,
+      "logps/chosen": -279.53619384765625,
+      "logps/rejected": -297.91168212890625,
+      "loss": 0.5781,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.25856587290763855,
+      "rewards/margins": 0.46807146072387695,
+      "rewards/rejected": -0.7266373038291931,
+      "step": 1210
+    },
+    {
+      "epoch": 0.7,
+      "grad_norm": 23.724168571838458,
+      "learning_rate": 1.2760003200873697e-07,
+      "logits/chosen": -2.5259270668029785,
+      "logits/rejected": -2.463974714279175,
+      "logps/chosen": -274.4045715332031,
+      "logps/rejected": -303.4898986816406,
+      "loss": 0.5929,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.40674060583114624,
+      "rewards/margins": 0.42905139923095703,
+      "rewards/rejected": -0.8357919454574585,
+      "step": 1220
+    },
+    {
+      "epoch": 0.7,
+      "grad_norm": 25.666665518773463,
+      "learning_rate": 1.2327656901707338e-07,
+      "logits/chosen": -2.522529125213623,
+      "logits/rejected": -2.4855215549468994,
+      "logps/chosen": -291.68292236328125,
+      "logps/rejected": -332.4886474609375,
+      "loss": 0.5913,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.3141830861568451,
+      "rewards/margins": 0.4157239496707916,
+      "rewards/rejected": -0.7299070358276367,
+      "step": 1230
+    },
+    {
+      "epoch": 0.71,
+      "grad_norm": 27.19771337622201,
+      "learning_rate": 1.1900352350748024e-07,
+      "logits/chosen": -2.5396928787231445,
+      "logits/rejected": -2.527820587158203,
+      "logps/chosen": -299.08056640625,
+      "logps/rejected": -341.8636474609375,
+      "loss": 0.5916,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3272027373313904,
+      "rewards/margins": 0.37227770686149597,
+      "rewards/rejected": -0.6994804739952087,
+      "step": 1240
+    },
+    {
+      "epoch": 0.71,
+      "grad_norm": 33.33117238316685,
+      "learning_rate": 1.147825955301668e-07,
+      "logits/chosen": -2.533803939819336,
+      "logits/rejected": -2.4939873218536377,
+      "logps/chosen": -256.9107666015625,
+      "logps/rejected": -292.6522521972656,
+      "loss": 0.581,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3282284736633301,
+      "rewards/margins": 0.4669713079929352,
+      "rewards/rejected": -0.7951998114585876,
+      "step": 1250
+    },
+    {
+      "epoch": 0.72,
+      "grad_norm": 29.14283955300301,
+      "learning_rate": 1.1061546440014963e-07,
+      "logits/chosen": -2.5241713523864746,
+      "logits/rejected": -2.5124003887176514,
+      "logps/chosen": -256.1329040527344,
+      "logps/rejected": -269.87127685546875,
+      "loss": 0.6125,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.24249228835105896,
+      "rewards/margins": 0.24842050671577454,
+      "rewards/rejected": -0.4909127652645111,
+      "step": 1260
+    },
+    {
+      "epoch": 0.73,
+      "grad_norm": 25.742222165117763,
+      "learning_rate": 1.0650378802913007e-07,
+      "logits/chosen": -2.4792118072509766,
+      "logits/rejected": -2.428273916244507,
+      "logps/chosen": -239.7839813232422,
+      "logps/rejected": -273.16070556640625,
+      "loss": 0.5855,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.14450299739837646,
+      "rewards/margins": 0.43993139266967773,
+      "rewards/rejected": -0.5844343900680542,
+      "step": 1270
+    },
+    {
+      "epoch": 0.73,
+      "grad_norm": 23.149905324082404,
+      "learning_rate": 1.0244920226588597e-07,
+      "logits/chosen": -2.5420844554901123,
+      "logits/rejected": -2.4969589710235596,
+      "logps/chosen": -246.9677734375,
+      "logps/rejected": -282.594482421875,
+      "loss": 0.5884,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.17258282005786896,
+      "rewards/margins": 0.34093397855758667,
+      "rewards/rejected": -0.5135167837142944,
+      "step": 1280
+    },
+    {
+      "epoch": 0.74,
+      "grad_norm": 24.301291223143128,
+      "learning_rate": 9.845332024544245e-08,
+      "logits/chosen": -2.5490660667419434,
+      "logits/rejected": -2.5159246921539307,
+      "logps/chosen": -253.81149291992188,
+      "logps/rejected": -271.77716064453125,
+      "loss": 0.5834,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.09992814064025879,
+      "rewards/margins": 0.4652518332004547,
+      "rewards/rejected": -0.5651799440383911,
+      "step": 1290
+    },
+    {
+      "epoch": 0.74,
+      "grad_norm": 23.714606581482208,
+      "learning_rate": 9.451773174727873e-08,
+      "logits/chosen": -2.517934799194336,
+      "logits/rejected": -2.467642068862915,
+      "logps/chosen": -253.8499298095703,
+      "logps/rejected": -283.5979919433594,
+      "loss": 0.5725,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.028648119419813156,
+      "rewards/margins": 0.5222169756889343,
+      "rewards/rejected": -0.5508650541305542,
+      "step": 1300
+    },
+    {
+      "epoch": 0.74,
+      "eval_logits/chosen": -2.5130560398101807,
+      "eval_logits/rejected": -2.4589791297912598,
+      "eval_logps/chosen": -245.44595336914062,
+      "eval_logps/rejected": -282.7961120605469,
+      "eval_loss": 0.580857515335083,
+      "eval_rewards/accuracies": 0.6878204941749573,
+      "eval_rewards/chosen": -0.11069269478321075,
+      "eval_rewards/margins": 0.4548812806606293,
+      "eval_rewards/rejected": -0.5655739903450012,
+      "eval_runtime": 863.5784,
+      "eval_samples_per_second": 14.418,
+      "eval_steps_per_second": 0.452,
+      "step": 1300
+    },
+    {
+      "epoch": 0.75,
+      "grad_norm": 25.51529829677871,
+      "learning_rate": 9.064400256282755e-08,
+      "logits/chosen": -2.497252941131592,
+      "logits/rejected": -2.4359519481658936,
+      "logps/chosen": -246.2664794921875,
+      "logps/rejected": -289.52105712890625,
+      "loss": 0.565,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.18726077675819397,
+      "rewards/margins": 0.4956758916378021,
+      "rewards/rejected": -0.6829366087913513,
+      "step": 1310
+    },
+    {
+      "epoch": 0.75,
+      "grad_norm": 28.395653589692625,
+      "learning_rate": 8.68336738725177e-08,
+      "logits/chosen": -2.5088109970092773,
+      "logits/rejected": -2.444235324859619,
+      "logps/chosen": -260.43255615234375,
+      "logps/rejected": -314.78466796875,
+      "loss": 0.585,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.09720905870199203,
+      "rewards/margins": 0.5907143950462341,
+      "rewards/rejected": -0.6879234910011292,
+      "step": 1320
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 34.946146652842195,
+      "learning_rate": 8.308826163260852e-08,
+      "logits/chosen": -2.4821484088897705,
+      "logits/rejected": -2.437180995941162,
+      "logps/chosen": -284.7655029296875,
+      "logps/rejected": -339.27410888671875,
+      "loss": 0.6013,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.38431376218795776,
+      "rewards/margins": 0.5592700242996216,
+      "rewards/rejected": -0.9435837864875793,
+      "step": 1330
+    },
+    {
+      "epoch": 0.77,
+      "grad_norm": 27.1693846661215,
+      "learning_rate": 7.940925597206052e-08,
+      "logits/chosen": -2.5242748260498047,
+      "logits/rejected": -2.423187732696533,
+      "logps/chosen": -240.61532592773438,
+      "logps/rejected": -303.8311462402344,
+      "loss": 0.5867,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.17216572165489197,
+      "rewards/margins": 0.4809741973876953,
+      "rewards/rejected": -0.6531398892402649,
+      "step": 1340
+    },
+    {
+      "epoch": 0.77,
+      "grad_norm": 26.85515693070903,
+      "learning_rate": 7.579812059968014e-08,
+      "logits/chosen": -2.468845844268799,
+      "logits/rejected": -2.4025533199310303,
+      "logps/chosen": -239.42636108398438,
+      "logps/rejected": -292.7440490722656,
+      "loss": 0.5718,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.1654459834098816,
+      "rewards/margins": 0.45221590995788574,
+      "rewards/rejected": -0.6176619529724121,
+      "step": 1350
+    },
+    {
+      "epoch": 0.78,
+      "grad_norm": 29.04379120857399,
+      "learning_rate": 7.225629222177715e-08,
+      "logits/chosen": -2.4990878105163574,
+      "logits/rejected": -2.483397960662842,
+      "logps/chosen": -273.4325256347656,
+      "logps/rejected": -326.87835693359375,
+      "loss": 0.5592,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.2638954222202301,
+      "rewards/margins": 0.4499826431274414,
+      "rewards/rejected": -0.7138780355453491,
+      "step": 1360
+    },
+    {
+      "epoch": 0.78,
+      "grad_norm": 27.003396904472925,
+      "learning_rate": 6.878517997056457e-08,
+      "logits/chosen": -2.5113110542297363,
+      "logits/rejected": -2.4679648876190186,
+      "logps/chosen": -231.7012176513672,
+      "logps/rejected": -295.1663818359375,
+      "loss": 0.5706,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.17551007866859436,
+      "rewards/margins": 0.5275388956069946,
+      "rewards/rejected": -0.7030488848686218,
+      "step": 1370
+    },
+    {
+      "epoch": 0.79,
+      "grad_norm": 25.178752513980715,
+      "learning_rate": 6.538616484352902e-08,
+      "logits/chosen": -2.5119760036468506,
+      "logits/rejected": -2.473975658416748,
+      "logps/chosen": -261.88665771484375,
+      "logps/rejected": -301.8829040527344,
+      "loss": 0.5784,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.27380552887916565,
+      "rewards/margins": 0.4960559010505676,
+      "rewards/rejected": -0.7698613405227661,
+      "step": 1380
+    },
+    {
+      "epoch": 0.79,
+      "grad_norm": 31.62767752494405,
+      "learning_rate": 6.206059915399495e-08,
+      "logits/chosen": -2.475964307785034,
+      "logits/rejected": -2.415445566177368,
+      "logps/chosen": -300.7445373535156,
+      "logps/rejected": -290.59259033203125,
+      "loss": 0.5803,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.21556460857391357,
+      "rewards/margins": 0.4194510579109192,
+      "rewards/rejected": -0.6350156664848328,
+      "step": 1390
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 26.11221463277773,
+      "learning_rate": 5.88098059931004e-08,
+      "logits/chosen": -2.513415575027466,
+      "logits/rejected": -2.431800603866577,
+      "logps/chosen": -248.53909301757812,
+      "logps/rejected": -316.1845703125,
+      "loss": 0.5719,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.17952445149421692,
+      "rewards/margins": 0.6487424373626709,
+      "rewards/rejected": -0.8282669186592102,
+      "step": 1400
+    },
+    {
+      "epoch": 0.8,
+      "eval_logits/chosen": -2.5095772743225098,
+      "eval_logits/rejected": -2.4584505558013916,
+      "eval_logps/chosen": -255.48683166503906,
+      "eval_logps/rejected": -296.0592346191406,
+      "eval_loss": 0.5792786478996277,
+      "eval_rewards/accuracies": 0.6894230842590332,
+      "eval_rewards/chosen": -0.2111014872789383,
+      "eval_rewards/margins": 0.4871039092540741,
+      "eval_rewards/rejected": -0.6982054114341736,
+      "eval_runtime": 847.3731,
+      "eval_samples_per_second": 14.694,
+      "eval_steps_per_second": 0.46,
+      "step": 1400
+    },
+    {
+      "epoch": 0.81,
+      "grad_norm": 31.913715183238843,
+      "learning_rate": 5.563507870339962e-08,
+      "logits/chosen": -2.4840493202209473,
+      "logits/rejected": -2.45074725151062,
+      "logps/chosen": -297.8506164550781,
+      "logps/rejected": -323.21722412109375,
+      "loss": 0.5745,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.21714219450950623,
+      "rewards/margins": 0.49252939224243164,
+      "rewards/rejected": -0.7096715569496155,
+      "step": 1410
+    },
+    {
+      "epoch": 0.81,
+      "grad_norm": 35.605251227958966,
+      "learning_rate": 5.25376803643007e-08,
+      "logits/chosen": -2.5487353801727295,
+      "logits/rejected": -2.4819111824035645,
+      "logps/chosen": -277.7652282714844,
+      "logps/rejected": -307.17413330078125,
+      "loss": 0.5827,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3039516806602478,
+      "rewards/margins": 0.4392912983894348,
+      "rewards/rejected": -0.7432429194450378,
+      "step": 1420
+    },
+    {
+      "epoch": 0.82,
+      "grad_norm": 28.285683945551348,
+      "learning_rate": 4.9518843289544007e-08,
+      "logits/chosen": -2.5319647789001465,
+      "logits/rejected": -2.4716343879699707,
+      "logps/chosen": -246.0936737060547,
+      "logps/rejected": -295.38726806640625,
+      "loss": 0.5892,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.1578165590763092,
+      "rewards/margins": 0.518903911113739,
+      "rewards/rejected": -0.6767204999923706,
+      "step": 1430
+    },
+    {
+      "epoch": 0.82,
+      "grad_norm": 29.113838409092516,
+      "learning_rate": 4.6579768536920695e-08,
+      "logits/chosen": -2.5386886596679688,
+      "logits/rejected": -2.4646518230438232,
+      "logps/chosen": -255.16152954101562,
+      "logps/rejected": -311.1474609375,
+      "loss": 0.6026,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.15220266580581665,
+      "rewards/margins": 0.46218329668045044,
+      "rewards/rejected": -0.6143860220909119,
+      "step": 1440
+    },
+    {
+      "epoch": 0.83,
+      "grad_norm": 33.086643583472295,
+      "learning_rate": 4.372162543042623e-08,
+      "logits/chosen": -2.506279230117798,
+      "logits/rejected": -2.4417097568511963,
+      "logps/chosen": -250.19287109375,
+      "logps/rejected": -287.5571594238281,
+      "loss": 0.5846,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.07686599344015121,
+      "rewards/margins": 0.5035240650177002,
+      "rewards/rejected": -0.5803900957107544,
+      "step": 1450
+    },
+    {
+      "epoch": 0.83,
+      "grad_norm": 22.051531350612663,
+      "learning_rate": 4.094555109503983e-08,
+      "logits/chosen": -2.5418009757995605,
+      "logits/rejected": -2.4571533203125,
+      "logps/chosen": -257.3257141113281,
+      "logps/rejected": -294.6969299316406,
+      "loss": 0.5669,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.17704418301582336,
+      "rewards/margins": 0.46817469596862793,
+      "rewards/rejected": -0.6452188491821289,
+      "step": 1460
+    },
+    {
+      "epoch": 0.84,
+      "grad_norm": 31.588020383365453,
+      "learning_rate": 3.825265000431424e-08,
+      "logits/chosen": -2.513741970062256,
+      "logits/rejected": -2.4840691089630127,
+      "logps/chosen": -260.6844177246094,
+      "logps/rejected": -297.4983825683594,
+      "loss": 0.5591,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.2122991979122162,
+      "rewards/margins": 0.5046205520629883,
+      "rewards/rejected": -0.7169197797775269,
+      "step": 1470
+    },
+    {
+      "epoch": 0.85,
+      "grad_norm": 27.48498825785638,
+      "learning_rate": 3.5643993540955844e-08,
+      "logits/chosen": -2.5381698608398438,
+      "logits/rejected": -2.4838833808898926,
+      "logps/chosen": -266.23724365234375,
+      "logps/rejected": -313.0609130859375,
+      "loss": 0.5712,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.22639349102973938,
+      "rewards/margins": 0.47134050726890564,
+      "rewards/rejected": -0.697733998298645,
+      "step": 1480
+    },
+    {
+      "epoch": 0.85,
+      "grad_norm": 29.66621598419702,
+      "learning_rate": 3.312061957057061e-08,
+      "logits/chosen": -2.5433754920959473,
+      "logits/rejected": -2.4874513149261475,
+      "logps/chosen": -253.4493408203125,
+      "logps/rejected": -313.81622314453125,
+      "loss": 0.5552,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.29640263319015503,
+      "rewards/margins": 0.5009024739265442,
+      "rewards/rejected": -0.797305166721344,
+      "step": 1490
+    },
+    {
+      "epoch": 0.86,
+      "grad_norm": 25.013543267548187,
+      "learning_rate": 3.0683532028744184e-08,
+      "logits/chosen": -2.5175490379333496,
+      "logits/rejected": -2.4559388160705566,
+      "logps/chosen": -250.25332641601562,
+      "logps/rejected": -292.8772888183594,
+      "loss": 0.5702,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.2427365779876709,
+      "rewards/margins": 0.4707626700401306,
+      "rewards/rejected": -0.7134992480278015,
+      "step": 1500
+    },
+    {
+      "epoch": 0.86,
+      "eval_logits/chosen": -2.5242793560028076,
+      "eval_logits/rejected": -2.4745876789093018,
+      "eval_logps/chosen": -261.0100402832031,
+      "eval_logps/rejected": -301.7152099609375,
+      "eval_loss": 0.5788725018501282,
+      "eval_rewards/accuracies": 0.6887820363044739,
+      "eval_rewards/chosen": -0.26633328199386597,
+      "eval_rewards/margins": 0.4884316027164459,
+      "eval_rewards/rejected": -0.754764974117279,
+      "eval_runtime": 847.3963,
+      "eval_samples_per_second": 14.693,
+      "eval_steps_per_second": 0.46,
+      "step": 1500
+    },
+    {
+      "epoch": 0.86,
+      "grad_norm": 27.931120203439026,
+      "learning_rate": 2.8333700521621955e-08,
+      "logits/chosen": -2.4960639476776123,
+      "logits/rejected": -2.465576648712158,
+      "logps/chosen": -262.0352478027344,
+      "logps/rejected": -281.86077880859375,
+      "loss": 0.5803,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.2684056758880615,
+      "rewards/margins": 0.4416535496711731,
+      "rewards/rejected": -0.7100592255592346,
+      "step": 1510
+    },
+    {
+      "epoch": 0.87,
+      "grad_norm": 33.57688090254581,
+      "learning_rate": 2.6072059940146772e-08,
+      "logits/chosen": -2.545090675354004,
+      "logits/rejected": -2.5021095275878906,
+      "logps/chosen": -262.41546630859375,
+      "logps/rejected": -307.7872009277344,
+      "loss": 0.5798,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.2850339114665985,
+      "rewards/margins": 0.5246847867965698,
+      "rewards/rejected": -0.8097187280654907,
+      "step": 1520
+    },
+    {
+      "epoch": 0.87,
+      "grad_norm": 25.889341574243268,
+      "learning_rate": 2.3899510088108587e-08,
+      "logits/chosen": -2.5121829509735107,
+      "logits/rejected": -2.485520839691162,
+      "logps/chosen": -253.7327117919922,
+      "logps/rejected": -294.1796569824219,
+      "loss": 0.5486,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.231918066740036,
+      "rewards/margins": 0.5059068202972412,
+      "rewards/rejected": -0.7378247976303101,
+      "step": 1530
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 25.634449566248733,
+      "learning_rate": 2.1816915324153334e-08,
+      "logits/chosen": -2.541168212890625,
+      "logits/rejected": -2.4927926063537598,
+      "logps/chosen": -277.5816650390625,
+      "logps/rejected": -327.8016662597656,
+      "loss": 0.5588,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.2811830937862396,
+      "rewards/margins": 0.6148523092269897,
+      "rewards/rejected": -0.896035373210907,
+      "step": 1540
+    },
+    {
+      "epoch": 0.89,
+      "grad_norm": 24.433006444286615,
+      "learning_rate": 1.9825104217894018e-08,
+      "logits/chosen": -2.5491321086883545,
+      "logits/rejected": -2.494581460952759,
+      "logps/chosen": -252.4076690673828,
+      "logps/rejected": -305.54437255859375,
+      "loss": 0.5959,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3050937354564667,
+      "rewards/margins": 0.45700544118881226,
+      "rewards/rejected": -0.7620991468429565,
+      "step": 1550
+    },
+    {
+      "epoch": 0.89,
+      "grad_norm": 24.678156956964582,
+      "learning_rate": 1.7924869220260626e-08,
+      "logits/chosen": -2.5328431129455566,
+      "logits/rejected": -2.4817864894866943,
+      "logps/chosen": -251.4116973876953,
+      "logps/rejected": -301.2293395996094,
+      "loss": 0.5487,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.2088792771100998,
+      "rewards/margins": 0.508228600025177,
+      "rewards/rejected": -0.7171077728271484,
+      "step": 1560
+    },
+    {
+      "epoch": 0.9,
+      "grad_norm": 28.5502202419384,
+      "learning_rate": 1.6116966348220046e-08,
+      "logits/chosen": -2.5276474952697754,
+      "logits/rejected": -2.498145818710327,
+      "logps/chosen": -244.6429443359375,
+      "logps/rejected": -293.36865234375,
+      "loss": 0.56,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.10921525955200195,
+      "rewards/margins": 0.6631591320037842,
+      "rewards/rejected": -0.7723743319511414,
+      "step": 1570
+    },
+    {
+      "epoch": 0.9,
+      "grad_norm": 30.92180920593632,
+      "learning_rate": 1.4402114883991318e-08,
+      "logits/chosen": -2.526020050048828,
+      "logits/rejected": -2.4712436199188232,
+      "logps/chosen": -261.9678039550781,
+      "logps/rejected": -287.0662536621094,
+      "loss": 0.5647,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3635459542274475,
+      "rewards/margins": 0.45373767614364624,
+      "rewards/rejected": -0.8172836303710938,
+      "step": 1580
+    },
+    {
+      "epoch": 0.91,
+      "grad_norm": 29.555421546021286,
+      "learning_rate": 1.2780997088875866e-08,
+      "logits/chosen": -2.5242857933044434,
+      "logits/rejected": -2.475109100341797,
+      "logps/chosen": -264.02740478515625,
+      "logps/rejected": -286.8149108886719,
+      "loss": 0.6074,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.1986238658428192,
+      "rewards/margins": 0.452684223651886,
+      "rewards/rejected": -0.6513080596923828,
+      "step": 1590
+    },
+    {
+      "epoch": 0.91,
+      "grad_norm": 26.529175018271285,
+      "learning_rate": 1.125425793181703e-08,
+      "logits/chosen": -2.5457723140716553,
+      "logits/rejected": -2.5002970695495605,
+      "logps/chosen": -268.09466552734375,
+      "logps/rejected": -307.01690673828125,
+      "loss": 0.5854,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.31708940863609314,
+      "rewards/margins": 0.44073382019996643,
+      "rewards/rejected": -0.7578231692314148,
+      "step": 1600
+    },
+    {
+      "epoch": 0.91,
+      "eval_logits/chosen": -2.552327871322632,
+      "eval_logits/rejected": -2.503713846206665,
+      "eval_logps/chosen": -257.1977233886719,
+      "eval_logps/rejected": -298.1694641113281,
+      "eval_loss": 0.5782989859580994,
+      "eval_rewards/accuracies": 0.6913461685180664,
+      "eval_rewards/chosen": -0.22821034491062164,
+      "eval_rewards/margins": 0.49109748005867004,
+      "eval_rewards/rejected": -0.7193077802658081,
+      "eval_runtime": 860.4396,
+      "eval_samples_per_second": 14.471,
+      "eval_steps_per_second": 0.453,
+      "step": 1600
+    },
+    {
+      "epoch": 0.92,
+      "grad_norm": 31.28191437907546,
+      "learning_rate": 9.822504832796036e-09,
+      "logits/chosen": -2.559476375579834,
+      "logits/rejected": -2.5392966270446777,
+      "logps/chosen": -271.2260437011719,
+      "logps/rejected": -299.50653076171875,
+      "loss": 0.5778,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.2566366195678711,
+      "rewards/margins": 0.49396759271621704,
+      "rewards/rejected": -0.7506042718887329,
+      "step": 1610
+    },
+    {
+      "epoch": 0.93,
+      "grad_norm": 30.662407347819215,
+      "learning_rate": 8.48630742116746e-09,
+      "logits/chosen": -2.536745071411133,
+      "logits/rejected": -2.4669063091278076,
+      "logps/chosen": -273.7347717285156,
+      "logps/rejected": -307.0451354980469,
+      "loss": 0.6142,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.28270235657691956,
+      "rewards/margins": 0.5321863293647766,
+      "rewards/rejected": -0.8148886561393738,
+      "step": 1620
+    },
+    {
+      "epoch": 0.93,
+      "grad_norm": 28.8995046840584,
+      "learning_rate": 7.246197309029617e-09,
+      "logits/chosen": -2.5540242195129395,
+      "logits/rejected": -2.51172137260437,
+      "logps/chosen": -288.4748229980469,
+      "logps/rejected": -334.9012451171875,
+      "loss": 0.5696,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.14674052596092224,
+      "rewards/margins": 0.5696360468864441,
+      "rewards/rejected": -0.7163766026496887,
+      "step": 1630
+    },
+    {
+      "epoch": 0.94,
+      "grad_norm": 30.322071826617417,
+      "learning_rate": 6.102667879720164e-09,
+      "logits/chosen": -2.517651081085205,
+      "logits/rejected": -2.470822811126709,
+      "logps/chosen": -270.4826354980469,
+      "logps/rejected": -301.36944580078125,
+      "loss": 0.5924,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3259834349155426,
+      "rewards/margins": 0.4199187755584717,
+      "rewards/rejected": -0.7459022402763367,
+      "step": 1640
+    },
+    {
+      "epoch": 0.94,
+      "grad_norm": 37.10925817248419,
+      "learning_rate": 5.056174091521509e-09,
+      "logits/chosen": -2.585538387298584,
+      "logits/rejected": -2.5188987255096436,
+      "logps/chosen": -271.0771179199219,
+      "logps/rejected": -313.93499755859375,
+      "loss": 0.5735,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.19604423642158508,
+      "rewards/margins": 0.5013729333877563,
+      "rewards/rejected": -0.697417140007019,
+      "step": 1650
+    },
+    {
+      "epoch": 0.95,
+      "grad_norm": 28.879184337464647,
+      "learning_rate": 4.107132296653548e-09,
+      "logits/chosen": -2.545414447784424,
+      "logits/rejected": -2.5129075050354004,
+      "logps/chosen": -249.78903198242188,
+      "logps/rejected": -274.78466796875,
+      "loss": 0.5854,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.20237192511558533,
+      "rewards/margins": 0.3796917498111725,
+      "rewards/rejected": -0.5820637345314026,
+      "step": 1660
+    },
+    {
+      "epoch": 0.95,
+      "grad_norm": 31.80047095954754,
+      "learning_rate": 3.255920075626084e-09,
+      "logits/chosen": -2.501981735229492,
+      "logits/rejected": -2.5079236030578613,
+      "logps/chosen": -243.0762176513672,
+      "logps/rejected": -292.73077392578125,
+      "loss": 0.5755,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.18036358058452606,
+      "rewards/margins": 0.4649125933647156,
+      "rewards/rejected": -0.6452761888504028,
+      "step": 1670
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 26.04495569695924,
+      "learning_rate": 2.5028760870168253e-09,
+      "logits/chosen": -2.5446419715881348,
+      "logits/rejected": -2.5122885704040527,
+      "logps/chosen": -276.3479919433594,
+      "logps/rejected": -304.0161437988281,
+      "loss": 0.5854,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.3501695692539215,
+      "rewards/margins": 0.2919423580169678,
+      "rewards/rejected": -0.6421119570732117,
+      "step": 1680
+    },
+    {
+      "epoch": 0.97,
+      "grad_norm": 24.450405436519606,
+      "learning_rate": 1.8482999327343597e-09,
+      "logits/chosen": -2.5676443576812744,
+      "logits/rejected": -2.4829487800598145,
+      "logps/chosen": -276.8134765625,
+      "logps/rejected": -306.0779113769531,
+      "loss": 0.6091,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.22208046913146973,
+      "rewards/margins": 0.37004339694976807,
+      "rewards/rejected": -0.5921238660812378,
+      "step": 1690
+    },
+    {
+      "epoch": 0.97,
+      "grad_norm": 31.006086476520473,
+      "learning_rate": 1.2924520388204462e-09,
+      "logits/chosen": -2.5088953971862793,
+      "logits/rejected": -2.507538318634033,
+      "logps/chosen": -248.55526733398438,
+      "logps/rejected": -294.12591552734375,
+      "loss": 0.578,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3303738832473755,
+      "rewards/margins": 0.3995395600795746,
+      "rewards/rejected": -0.7299134135246277,
+      "step": 1700
+    },
+    {
+      "epoch": 0.97,
+      "eval_logits/chosen": -2.5474650859832764,
+      "eval_logits/rejected": -2.4987149238586426,
+      "eval_logps/chosen": -255.72315979003906,
+      "eval_logps/rejected": -296.4235534667969,
+      "eval_loss": 0.5782065987586975,
+      "eval_rewards/accuracies": 0.691987156867981,
+      "eval_rewards/chosen": -0.21346470713615417,
+      "eval_rewards/margins": 0.4883839786052704,
+      "eval_rewards/rejected": -0.7018487453460693,
+      "eval_runtime": 847.8379,
+      "eval_samples_per_second": 14.686,
+      "eval_steps_per_second": 0.46,
+      "step": 1700
+    },
+    {
+      "epoch": 0.98,
+      "grad_norm": 27.94865663504945,
+      "learning_rate": 8.355535518381751e-10,
+      "logits/chosen": -2.5459036827087402,
+      "logits/rejected": -2.523585081100464,
+      "logps/chosen": -267.97119140625,
+      "logps/rejected": -273.44970703125,
+      "loss": 0.5877,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.24003572762012482,
+      "rewards/margins": 0.3651558756828308,
+      "rewards/rejected": -0.6051916480064392,
+      "step": 1710
+    },
+    {
+      "epoch": 0.98,
+      "grad_norm": 26.206122309038193,
+      "learning_rate": 4.77786250887846e-10,
+      "logits/chosen": -2.5695338249206543,
+      "logits/rejected": -2.5357468128204346,
+      "logps/chosen": -251.841552734375,
+      "logps/rejected": -304.9547424316406,
+      "loss": 0.5833,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.12936021387577057,
+      "rewards/margins": 0.5671738386154175,
+      "rewards/rejected": -0.6965340971946716,
+      "step": 1720
+    },
+    {
+      "epoch": 0.99,
+      "grad_norm": 26.649920950848678,
+      "learning_rate": 2.1929247528540418e-10,
+      "logits/chosen": -2.554141044616699,
+      "logits/rejected": -2.493603467941284,
+      "logps/chosen": -265.3050231933594,
+      "logps/rejected": -310.0926208496094,
+      "loss": 0.5706,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.20094648003578186,
+      "rewards/margins": 0.47598400712013245,
+      "rewards/rejected": -0.6769304871559143,
+      "step": 1730
+    },
+    {
+      "epoch": 0.99,
+      "grad_norm": 27.75248252937658,
+      "learning_rate": 6.017506793212779e-11,
+      "logits/chosen": -2.5380330085754395,
+      "logits/rejected": -2.4906625747680664,
+      "logps/chosen": -251.3086700439453,
+      "logps/rejected": -314.17645263671875,
+      "loss": 0.5705,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.11332281678915024,
+      "rewards/margins": 0.5656474828720093,
+      "rewards/rejected": -0.6789703369140625,
+      "step": 1740
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 31.342302748055804,
+      "learning_rate": 4.973343980252398e-13,
+      "logits/chosen": -2.543778896331787,
+      "logits/rejected": -2.499126434326172,
+      "logps/chosen": -258.0113830566406,
+      "logps/rejected": -294.2593994140625,
+      "loss": 0.5742,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.23090767860412598,
+      "rewards/margins": 0.5441216230392456,
+      "rewards/rejected": -0.7750292420387268,
+      "step": 1750
+    },
+    {
+      "epoch": 1.0,
+      "step": 1751,
+      "total_flos": 0.0,
+      "train_loss": 0.5964619764459262,
+      "train_runtime": 39064.845,
+      "train_samples_per_second": 2.868,
+      "train_steps_per_second": 0.045
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1751,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}