diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,6000 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 100,
+  "global_step": 3821,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.3054830287206268e-08,
+      "logits/chosen": 0.9550814628601074,
+      "logits/rejected": 1.0664727687835693,
+      "logps/chosen": -190.47879028320312,
+      "logps/rejected": -177.6958770751953,
+      "loss": 0.1031,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.3054830287206266e-07,
+      "logits/chosen": 1.0218172073364258,
+      "logits/rejected": 1.074446439743042,
+      "logps/chosen": -277.89642333984375,
+      "logps/rejected": -268.27508544921875,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.3888888955116272,
+      "rewards/chosen": -2.27014561460237e-06,
+      "rewards/margins": -7.147283213271294e-06,
+      "rewards/rejected": 4.877134870184818e-06,
+      "step": 10
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 2.610966057441253e-07,
+      "logits/chosen": 1.0545730590820312,
+      "logits/rejected": 1.0357177257537842,
+      "logps/chosen": -258.04461669921875,
+      "logps/rejected": -219.55276489257812,
+      "loss": 0.0678,
+      "rewards/accuracies": 0.4312500059604645,
+      "rewards/chosen": -6.120393663877621e-05,
+      "rewards/margins": -2.3215863620862365e-05,
+      "rewards/rejected": -3.798806210397743e-05,
+      "step": 20
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 3.9164490861618804e-07,
+      "logits/chosen": 0.9781615138053894,
+      "logits/rejected": 0.995823085308075,
+      "logps/chosen": -234.4961700439453,
+      "logps/rejected": -216.3553466796875,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": -0.00010812957771122456,
+      "rewards/margins": -1.6783669707365334e-05,
+      "rewards/rejected": -9.134591527981684e-05,
+      "step": 30
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.221932114882506e-07,
+      "logits/chosen": 1.059520959854126,
+      "logits/rejected": 1.0608160495758057,
+      "logps/chosen": -269.41412353515625,
+      "logps/rejected": -236.53683471679688,
+      "loss": 0.0647,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -9.875606338027865e-05,
+      "rewards/margins": -5.8930483646690845e-05,
+      "rewards/rejected": -3.982557245763019e-05,
+      "step": 40
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 6.527415143603135e-07,
+      "logits/chosen": 1.012832522392273,
+      "logits/rejected": 1.0503787994384766,
+      "logps/chosen": -245.1386260986328,
+      "logps/rejected": -241.9445037841797,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -5.048250750405714e-05,
+      "rewards/margins": 4.718836862593889e-05,
+      "rewards/rejected": -9.767088340595365e-05,
+      "step": 50
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 7.832898172323761e-07,
+      "logits/chosen": 0.9758403897285461,
+      "logits/rejected": 1.0931203365325928,
+      "logps/chosen": -283.77294921875,
+      "logps/rejected": -234.1714324951172,
+      "loss": 0.051,
+      "rewards/accuracies": 0.4437499940395355,
+      "rewards/chosen": -4.2240175389451906e-05,
+      "rewards/margins": -9.017210686579347e-06,
+      "rewards/rejected": -3.3222971978830174e-05,
+      "step": 60
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.138381201044387e-07,
+      "logits/chosen": 1.006019949913025,
+      "logits/rejected": 1.0821138620376587,
+      "logps/chosen": -272.0429992675781,
+      "logps/rejected": -231.1321258544922,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -4.4153930502943695e-05,
+      "rewards/margins": 5.0114922487409785e-05,
+      "rewards/rejected": -9.426885662833229e-05,
+      "step": 70
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.0443864229765013e-06,
+      "logits/chosen": 1.023194432258606,
+      "logits/rejected": 1.0623011589050293,
+      "logps/chosen": -283.9044494628906,
+      "logps/rejected": -261.71441650390625,
+      "loss": 0.0442,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -4.0706909203436226e-05,
+      "rewards/margins": 1.2994561984669417e-05,
+      "rewards/rejected": -5.370147482608445e-05,
+      "step": 80
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.1749347258485642e-06,
+      "logits/chosen": 1.0426299571990967,
+      "logits/rejected": 1.0920602083206177,
+      "logps/chosen": -278.4473876953125,
+      "logps/rejected": -235.7585906982422,
+      "loss": 0.0597,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -8.936020458349958e-05,
+      "rewards/margins": -5.8977882872568443e-05,
+      "rewards/rejected": -3.038232898688875e-05,
+      "step": 90
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.305483028720627e-06,
+      "logits/chosen": 0.9912114143371582,
+      "logits/rejected": 1.066651463508606,
+      "logps/chosen": -237.31021118164062,
+      "logps/rejected": -218.4906005859375,
+      "loss": 0.066,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -3.355559238116257e-05,
+      "rewards/margins": 9.931150998454541e-06,
+      "rewards/rejected": -4.348673246568069e-05,
+      "step": 100
+    },
+    {
+      "epoch": 0.03,
+      "eval_logits/chosen": 0.9712140560150146,
+      "eval_logits/rejected": 1.0636906623840332,
+      "eval_logps/chosen": -277.57818603515625,
+      "eval_logps/rejected": -243.88119506835938,
+      "eval_loss": 0.05371462181210518,
+      "eval_rewards/accuracies": 0.4724999964237213,
+      "eval_rewards/chosen": -3.141459819744341e-05,
+      "eval_rewards/margins": 3.384269439266063e-05,
+      "eval_rewards/rejected": -6.525729259010404e-05,
+      "eval_runtime": 545.9046,
+      "eval_samples_per_second": 3.664,
+      "eval_steps_per_second": 0.916,
+      "step": 100
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.4360313315926894e-06,
+      "logits/chosen": 0.9956586956977844,
+      "logits/rejected": 1.080687165260315,
+      "logps/chosen": -283.57708740234375,
+      "logps/rejected": -250.15853881835938,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 4.337792779551819e-05,
+      "rewards/margins": 0.00017397073679603636,
+      "rewards/rejected": -0.0001305928162764758,
+      "step": 110
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.5665796344647521e-06,
+      "logits/chosen": 1.02920663356781,
+      "logits/rejected": 1.0740941762924194,
+      "logps/chosen": -227.79696655273438,
+      "logps/rejected": -234.1930694580078,
+      "loss": 0.0856,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 1.176113983092364e-05,
+      "rewards/margins": 0.0001008812541840598,
+      "rewards/rejected": -8.912011981010437e-05,
+      "step": 120
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.6971279373368146e-06,
+      "logits/chosen": 1.047863483428955,
+      "logits/rejected": 1.094162106513977,
+      "logps/chosen": -282.6900634765625,
+      "logps/rejected": -239.26968383789062,
+      "loss": 0.0454,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": 1.2405995221342891e-05,
+      "rewards/margins": 0.00015912330127321184,
+      "rewards/rejected": -0.0001467173278797418,
+      "step": 130
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.8276762402088774e-06,
+      "logits/chosen": 1.0264124870300293,
+      "logits/rejected": 1.0263712406158447,
+      "logps/chosen": -264.00384521484375,
+      "logps/rejected": -237.15341186523438,
+      "loss": 0.0476,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 4.7245732275769114e-05,
+      "rewards/margins": 0.00025105843087658286,
+      "rewards/rejected": -0.00020381269860081375,
+      "step": 140
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.9582245430809403e-06,
+      "logits/chosen": 1.0078333616256714,
+      "logits/rejected": 1.0274016857147217,
+      "logps/chosen": -262.72369384765625,
+      "logps/rejected": -234.98580932617188,
+      "loss": 0.0645,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -9.314047929365188e-06,
+      "rewards/margins": 0.00011289575923001394,
+      "rewards/rejected": -0.00012220982171129435,
+      "step": 150
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.0887728459530026e-06,
+      "logits/chosen": 0.9558731317520142,
+      "logits/rejected": 1.085249423980713,
+      "logps/chosen": -258.3452453613281,
+      "logps/rejected": -240.0926971435547,
+      "loss": 0.0577,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -3.5412434954196215e-05,
+      "rewards/margins": 0.0001463442895328626,
+      "rewards/rejected": -0.00018175672448705882,
+      "step": 160
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.2193211488250653e-06,
+      "logits/chosen": 0.9971574544906616,
+      "logits/rejected": 1.073053002357483,
+      "logps/chosen": -268.4542541503906,
+      "logps/rejected": -218.31979370117188,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": 0.00010815112909767777,
+      "rewards/margins": 0.00034510824480094016,
+      "rewards/rejected": -0.00023695713025517762,
+      "step": 170
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.3498694516971284e-06,
+      "logits/chosen": 0.9860492944717407,
+      "logits/rejected": 1.0339651107788086,
+      "logps/chosen": -272.54998779296875,
+      "logps/rejected": -237.78274536132812,
+      "loss": 0.0649,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.00013371471140999347,
+      "rewards/margins": 0.00023159733973443508,
+      "rewards/rejected": -9.7882621048484e-05,
+      "step": 180
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.4804177545691907e-06,
+      "logits/chosen": 0.9933134913444519,
+      "logits/rejected": 1.0112648010253906,
+      "logps/chosen": -269.50372314453125,
+      "logps/rejected": -235.55654907226562,
+      "loss": 0.0618,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.00012103781045880169,
+      "rewards/margins": 0.0003521198814269155,
+      "rewards/rejected": -0.0002310820564161986,
+      "step": 190
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.610966057441254e-06,
+      "logits/chosen": 1.0235344171524048,
+      "logits/rejected": 1.1339811086654663,
+      "logps/chosen": -278.14471435546875,
+      "logps/rejected": -249.69467163085938,
+      "loss": 0.0611,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": 0.00034809770295396447,
+      "rewards/margins": 0.0004462279030121863,
+      "rewards/rejected": -9.813017823034897e-05,
+      "step": 200
+    },
+    {
+      "epoch": 0.05,
+      "eval_logits/chosen": 0.9716129899024963,
+      "eval_logits/rejected": 1.064271330833435,
+      "eval_logps/chosen": -277.24957275390625,
+      "eval_logps/rejected": -243.99212646484375,
+      "eval_loss": 0.05354813113808632,
+      "eval_rewards/accuracies": 0.578000009059906,
+      "eval_rewards/chosen": 0.00029724082560278475,
+      "eval_rewards/margins": 0.00047346166684292257,
+      "eval_rewards/rejected": -0.00017622079758439213,
+      "eval_runtime": 545.7384,
+      "eval_samples_per_second": 3.665,
+      "eval_steps_per_second": 0.916,
+      "step": 200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.741514360313316e-06,
+      "logits/chosen": 1.0143412351608276,
+      "logits/rejected": 1.0932003259658813,
+      "logps/chosen": -260.88043212890625,
+      "logps/rejected": -233.2900848388672,
+      "loss": 0.0405,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": 0.0001968775613931939,
+      "rewards/margins": 0.00041465656249783933,
+      "rewards/rejected": -0.00021777900110464543,
+      "step": 210
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.872062663185379e-06,
+      "logits/chosen": 1.0440694093704224,
+      "logits/rejected": 1.1149526834487915,
+      "logps/chosen": -277.57476806640625,
+      "logps/rejected": -243.1229705810547,
+      "loss": 0.0458,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.0003836105461232364,
+      "rewards/margins": 0.0005119801498949528,
+      "rewards/rejected": -0.00012836960377171636,
+      "step": 220
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.0026109660574416e-06,
+      "logits/chosen": 1.0338958501815796,
+      "logits/rejected": 1.0419671535491943,
+      "logps/chosen": -268.84100341796875,
+      "logps/rejected": -275.18572998046875,
+      "loss": 0.0418,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.0005500637926161289,
+      "rewards/margins": 0.0006700553931295872,
+      "rewards/rejected": -0.0001199916732730344,
+      "step": 230
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.1331592689295043e-06,
+      "logits/chosen": 1.0325114727020264,
+      "logits/rejected": 1.0510814189910889,
+      "logps/chosen": -271.9697265625,
+      "logps/rejected": -231.6917724609375,
+      "loss": 0.05,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.0008386652916669846,
+      "rewards/margins": 0.001031489809975028,
+      "rewards/rejected": -0.0001928243727888912,
+      "step": 240
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.263707571801567e-06,
+      "logits/chosen": 0.9305399656295776,
+      "logits/rejected": 1.0869312286376953,
+      "logps/chosen": -262.2601013183594,
+      "logps/rejected": -207.22415161132812,
+      "loss": 0.0489,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": 0.0006195057649165392,
+      "rewards/margins": 0.0008672000840306282,
+      "rewards/rejected": -0.000247694319114089,
+      "step": 250
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.3942558746736293e-06,
+      "logits/chosen": 1.0228338241577148,
+      "logits/rejected": 1.0134680271148682,
+      "logps/chosen": -255.75350952148438,
+      "logps/rejected": -249.2091522216797,
+      "loss": 0.0575,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": 0.0006173985311761498,
+      "rewards/margins": 0.0008433389593847096,
+      "rewards/rejected": -0.00022594044276047498,
+      "step": 260
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.524804177545692e-06,
+      "logits/chosen": 0.9264955520629883,
+      "logits/rejected": 1.0731738805770874,
+      "logps/chosen": -250.19686889648438,
+      "logps/rejected": -225.1935577392578,
+      "loss": 0.0616,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.0006687111454084516,
+      "rewards/margins": 0.0010838030138984323,
+      "rewards/rejected": -0.00041509178117848933,
+      "step": 270
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.6553524804177547e-06,
+      "logits/chosen": 0.9631627798080444,
+      "logits/rejected": 1.0036388635635376,
+      "logps/chosen": -262.21258544921875,
+      "logps/rejected": -244.89212036132812,
+      "loss": 0.0704,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.0011519830441102386,
+      "rewards/margins": 0.0016097616171464324,
+      "rewards/rejected": -0.00045777857303619385,
+      "step": 280
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.7859007832898174e-06,
+      "logits/chosen": 0.9706422090530396,
+      "logits/rejected": 1.061474084854126,
+      "logps/chosen": -258.235595703125,
+      "logps/rejected": -228.9088897705078,
+      "loss": 0.0584,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.0013363256584852934,
+      "rewards/margins": 0.0018601767951622605,
+      "rewards/rejected": -0.000523851253092289,
+      "step": 290
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.9164490861618806e-06,
+      "logits/chosen": 0.9408682584762573,
+      "logits/rejected": 1.0623794794082642,
+      "logps/chosen": -256.77679443359375,
+      "logps/rejected": -227.1261749267578,
+      "loss": 0.0609,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0012866052566096187,
+      "rewards/margins": 0.001811084570363164,
+      "rewards/rejected": -0.0005244792555458844,
+      "step": 300
+    },
+    {
+      "epoch": 0.08,
+      "eval_logits/chosen": 0.9689084887504578,
+      "eval_logits/rejected": 1.0635567903518677,
+      "eval_logps/chosen": -276.017822265625,
+      "eval_logps/rejected": -244.33355712890625,
+      "eval_loss": 0.05289166420698166,
+      "eval_rewards/accuracies": 0.6165000200271606,
+      "eval_rewards/chosen": 0.0015290172304958105,
+      "eval_rewards/margins": 0.0020466677378863096,
+      "eval_rewards/rejected": -0.0005176504491828382,
+      "eval_runtime": 545.8525,
+      "eval_samples_per_second": 3.664,
+      "eval_steps_per_second": 0.916,
+      "step": 300
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.046997389033943e-06,
+      "logits/chosen": 0.9319702982902527,
+      "logits/rejected": 1.0929656028747559,
+      "logps/chosen": -255.96304321289062,
+      "logps/rejected": -226.93115234375,
+      "loss": 0.0528,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": 0.0019286921015009284,
+      "rewards/margins": 0.0025504238437861204,
+      "rewards/rejected": -0.0006217318587005138,
+      "step": 310
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.177545691906005e-06,
+      "logits/chosen": 0.9062834978103638,
+      "logits/rejected": 1.0360002517700195,
+      "logps/chosen": -282.16961669921875,
+      "logps/rejected": -256.7910461425781,
+      "loss": 0.0448,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.00156155857257545,
+      "rewards/margins": 0.0018548518419265747,
+      "rewards/rejected": -0.0002932931238319725,
+      "step": 320
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.308093994778068e-06,
+      "logits/chosen": 0.9864987134933472,
+      "logits/rejected": 1.0952273607254028,
+      "logps/chosen": -278.2218017578125,
+      "logps/rejected": -243.8715057373047,
+      "loss": 0.0613,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": 0.0010156099451705813,
+      "rewards/margins": 0.0023700734600424767,
+      "rewards/rejected": -0.0013544636312872171,
+      "step": 330
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.4386422976501306e-06,
+      "logits/chosen": 0.974676251411438,
+      "logits/rejected": 1.0076481103897095,
+      "logps/chosen": -273.563232421875,
+      "logps/rejected": -268.89471435546875,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0010632537305355072,
+      "rewards/margins": 0.0025850296951830387,
+      "rewards/rejected": -0.0015217757318168879,
+      "step": 340
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.569190600522193e-06,
+      "logits/chosen": 0.9837932586669922,
+      "logits/rejected": 0.9761275053024292,
+      "logps/chosen": -283.8204650878906,
+      "logps/rejected": -251.16372680664062,
+      "loss": 0.063,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.0015502728056162596,
+      "rewards/margins": 0.0033880569972097874,
+      "rewards/rejected": -0.001837784075178206,
+      "step": 350
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.699738903394257e-06,
+      "logits/chosen": 0.9331616163253784,
+      "logits/rejected": 1.0257459878921509,
+      "logps/chosen": -301.2323913574219,
+      "logps/rejected": -259.16015625,
+      "loss": 0.0406,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0014106493908911943,
+      "rewards/margins": 0.0038649961352348328,
+      "rewards/rejected": -0.0024543465115129948,
+      "step": 360
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.8302872062663196e-06,
+      "logits/chosen": 0.9704321026802063,
+      "logits/rejected": 0.9583215713500977,
+      "logps/chosen": -304.32049560546875,
+      "logps/rejected": -262.0417175292969,
+      "loss": 0.0503,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.0016505718231201172,
+      "rewards/margins": 0.004703259561210871,
+      "rewards/rejected": -0.0030526877380907536,
+      "step": 370
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.9608355091383814e-06,
+      "logits/chosen": 0.9409393072128296,
+      "logits/rejected": 1.0430846214294434,
+      "logps/chosen": -255.3147735595703,
+      "logps/rejected": -225.9354248046875,
+      "loss": 0.0743,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": 0.0009300144156441092,
+      "rewards/margins": 0.005477910861372948,
+      "rewards/rejected": -0.004547896794974804,
+      "step": 380
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.9999488562447675e-06,
+      "logits/chosen": 0.9433630704879761,
+      "logits/rejected": 0.9765726923942566,
+      "logps/chosen": -298.94866943359375,
+      "logps/rejected": -260.1683044433594,
+      "loss": 0.0437,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.0008828797144815326,
+      "rewards/margins": 0.007025745697319508,
+      "rewards/rejected": -0.006142866797745228,
+      "step": 390
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.999698361256577e-06,
+      "logits/chosen": 0.9384250640869141,
+      "logits/rejected": 0.9429944753646851,
+      "logps/chosen": -281.13812255859375,
+      "logps/rejected": -269.3196716308594,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.0018200930207967758,
+      "rewards/margins": 0.007275038864463568,
+      "rewards/rejected": -0.00909513235092163,
+      "step": 400
+    },
+    {
+      "epoch": 0.1,
+      "eval_logits/chosen": 0.8601001501083374,
+      "eval_logits/rejected": 0.9583178162574768,
+      "eval_logps/chosen": -280.6138000488281,
+      "eval_logps/rejected": -253.28575134277344,
+      "eval_loss": 0.051143791526556015,
+      "eval_rewards/accuracies": 0.6150000095367432,
+      "eval_rewards/chosen": -0.003066999837756157,
+      "eval_rewards/margins": 0.006402799393981695,
+      "eval_rewards/rejected": -0.009469798766076565,
+      "eval_runtime": 545.7662,
+      "eval_samples_per_second": 3.665,
+      "eval_steps_per_second": 0.916,
+      "step": 400
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.999239142174581e-06,
+      "logits/chosen": 0.828892707824707,
+      "logits/rejected": 0.8553358912467957,
+      "logps/chosen": -297.42987060546875,
+      "logps/rejected": -253.168212890625,
+      "loss": 0.046,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.0039054974913597107,
+      "rewards/margins": 0.0067556751891970634,
+      "rewards/rejected": -0.010661173611879349,
+      "step": 410
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.99857123734344e-06,
+      "logits/chosen": 0.8542848825454712,
+      "logits/rejected": 0.9624751806259155,
+      "logps/chosen": -283.91229248046875,
+      "logps/rejected": -255.8434600830078,
+      "loss": 0.0533,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0053799813613295555,
+      "rewards/margins": 0.007480146829038858,
+      "rewards/rejected": -0.012860128656029701,
+      "step": 420
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.997694702533016e-06,
+      "logits/chosen": 0.7429074048995972,
+      "logits/rejected": 0.8771979212760925,
+      "logps/chosen": -273.5076904296875,
+      "logps/rejected": -272.1727600097656,
+      "loss": 0.0487,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.009718675166368484,
+      "rewards/margins": 0.00685915257781744,
+      "rewards/rejected": -0.0165778286755085,
+      "step": 430
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.996609610933713e-06,
+      "logits/chosen": 0.7902615666389465,
+      "logits/rejected": 0.7774487733840942,
+      "logps/chosen": -293.1501770019531,
+      "logps/rejected": -275.8284606933594,
+      "loss": 0.0505,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.011228824965655804,
+      "rewards/margins": 0.0071553452871739864,
+      "rewards/rejected": -0.018384169787168503,
+      "step": 440
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.995316053150366e-06,
+      "logits/chosen": 0.7413063645362854,
+      "logits/rejected": 0.8421871066093445,
+      "logps/chosen": -273.8318176269531,
+      "logps/rejected": -252.2345428466797,
+      "loss": 0.0373,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.012360856868326664,
+      "rewards/margins": 0.0071116602048277855,
+      "rewards/rejected": -0.0194725152105093,
+      "step": 450
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.9938141371946815e-06,
+      "logits/chosen": 0.7610155344009399,
+      "logits/rejected": 0.8774874806404114,
+      "logps/chosen": -263.0393981933594,
+      "logps/rejected": -271.81878662109375,
+      "loss": 0.0661,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.011142844334244728,
+      "rewards/margins": 0.01251134742051363,
+      "rewards/rejected": -0.023654192686080933,
+      "step": 460
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.992103988476206e-06,
+      "logits/chosen": 0.6996180415153503,
+      "logits/rejected": 0.7960006594657898,
+      "logps/chosen": -310.3810729980469,
+      "logps/rejected": -271.3680725097656,
+      "loss": 0.0698,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.015032676048576832,
+      "rewards/margins": 0.010951148346066475,
+      "rewards/rejected": -0.025983821600675583,
+      "step": 470
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.990185749791866e-06,
+      "logits/chosen": 0.632653534412384,
+      "logits/rejected": 0.7328468561172485,
+      "logps/chosen": -257.9376220703125,
+      "logps/rejected": -237.2174835205078,
+      "loss": 0.0476,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.015546147711575031,
+      "rewards/margins": 0.013505731709301472,
+      "rewards/rejected": -0.029051881283521652,
+      "step": 480
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.9880595813140395e-06,
+      "logits/chosen": 0.6077014207839966,
+      "logits/rejected": 0.732668399810791,
+      "logps/chosen": -293.5138244628906,
+      "logps/rejected": -267.81561279296875,
+      "loss": 0.0504,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.018740156665444374,
+      "rewards/margins": 0.01524042896926403,
+      "rewards/rejected": -0.033980585634708405,
+      "step": 490
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.985725660577184e-06,
+      "logits/chosen": 0.6373035311698914,
+      "logits/rejected": 0.6293123960494995,
+      "logps/chosen": -266.0309143066406,
+      "logps/rejected": -270.39239501953125,
+      "loss": 0.0501,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.022544419392943382,
+      "rewards/margins": 0.018048131838440895,
+      "rewards/rejected": -0.04059254750609398,
+      "step": 500
+    },
+    {
+      "epoch": 0.13,
+      "eval_logits/chosen": 0.4969506561756134,
+      "eval_logits/rejected": 0.5769693851470947,
+      "eval_logps/chosen": -306.81011962890625,
+      "eval_logps/rejected": -289.31903076171875,
+      "eval_loss": 0.047541093081235886,
+      "eval_rewards/accuracies": 0.6050000190734863,
+      "eval_rewards/chosen": -0.029263317584991455,
+      "eval_rewards/margins": 0.01623981073498726,
+      "eval_rewards/rejected": -0.04550313204526901,
+      "eval_runtime": 546.0666,
+      "eval_samples_per_second": 3.663,
+      "eval_steps_per_second": 0.916,
+      "step": 500
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.983184182463009e-06,
+      "logits/chosen": 0.4564204216003418,
+      "logits/rejected": 0.5843110680580139,
+      "logps/chosen": -310.9439697265625,
+      "logps/rejected": -260.3971252441406,
+      "loss": 0.0558,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.02757311426103115,
+      "rewards/margins": 0.015168674290180206,
+      "rewards/rejected": -0.042741790413856506,
+      "step": 510
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.980435359184203e-06,
+      "logits/chosen": 0.4646351933479309,
+      "logits/rejected": 0.5338501334190369,
+      "logps/chosen": -321.9226379394531,
+      "logps/rejected": -307.0296630859375,
+      "loss": 0.0387,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.033701926469802856,
+      "rewards/margins": 0.02382163144648075,
+      "rewards/rejected": -0.05752355977892876,
+      "step": 520
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.9774794202667236e-06,
+      "logits/chosen": 0.39868661761283875,
+      "logits/rejected": 0.46957048773765564,
+      "logps/chosen": -301.45733642578125,
+      "logps/rejected": -290.30987548828125,
+      "loss": 0.0405,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.03544437885284424,
+      "rewards/margins": 0.022634658962488174,
+      "rewards/rejected": -0.05807904154062271,
+      "step": 530
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.974316612530615e-06,
+      "logits/chosen": 0.32594749331474304,
+      "logits/rejected": 0.36282044649124146,
+      "logps/chosen": -274.9070739746094,
+      "logps/rejected": -279.6643371582031,
+      "loss": 0.0475,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.03333526849746704,
+      "rewards/margins": 0.027274906635284424,
+      "rewards/rejected": -0.060610175132751465,
+      "step": 540
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.970947200069416e-06,
+      "logits/chosen": 0.3247653543949127,
+      "logits/rejected": 0.3564518392086029,
+      "logps/chosen": -303.27362060546875,
+      "logps/rejected": -293.4110412597656,
+      "loss": 0.0623,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.040520280599594116,
+      "rewards/margins": 0.031138747930526733,
+      "rewards/rejected": -0.07165903598070145,
+      "step": 550
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.967371464228096e-06,
+      "logits/chosen": 0.4165423810482025,
+      "logits/rejected": 0.47896915674209595,
+      "logps/chosen": -324.7799377441406,
+      "logps/rejected": -317.0769958496094,
+      "loss": 0.0499,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.0410742312669754,
+      "rewards/margins": 0.0179686788469553,
+      "rewards/rejected": -0.05904290825128555,
+      "step": 560
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.963589703579569e-06,
+      "logits/chosen": 0.4340105950832367,
+      "logits/rejected": 0.5640918016433716,
+      "logps/chosen": -293.931884765625,
+      "logps/rejected": -274.7142333984375,
+      "loss": 0.0609,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.02599266730248928,
+      "rewards/margins": 0.032392553985118866,
+      "rewards/rejected": -0.0583852156996727,
+      "step": 570
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.9596022338997615e-06,
+      "logits/chosen": 0.43371954560279846,
+      "logits/rejected": 0.4177388548851013,
+      "logps/chosen": -288.1071472167969,
+      "logps/rejected": -274.2785949707031,
+      "loss": 0.0504,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.03160632401704788,
+      "rewards/margins": 0.01767369918525219,
+      "rewards/rejected": -0.04928002506494522,
+      "step": 580
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.955409388141243e-06,
+      "logits/chosen": 0.436180979013443,
+      "logits/rejected": 0.388658344745636,
+      "logps/chosen": -301.9410400390625,
+      "logps/rejected": -284.9775085449219,
+      "loss": 0.0444,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.03411916643381119,
+      "rewards/margins": 0.02359338477253914,
+      "rewards/rejected": -0.05771254748106003,
+      "step": 590
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.951011516405429e-06,
+      "logits/chosen": 0.33265748620033264,
+      "logits/rejected": 0.3309306502342224,
+      "logps/chosen": -277.5527648925781,
+      "logps/rejected": -320.6658935546875,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.040917735546827316,
+      "rewards/margins": 0.02358854189515114,
+      "rewards/rejected": -0.06450627744197845,
+      "step": 600
+    },
+    {
+      "epoch": 0.16,
+      "eval_logits/chosen": 0.27490919828414917,
+      "eval_logits/rejected": 0.3281973898410797,
+      "eval_logps/chosen": -321.478271484375,
+      "eval_logps/rejected": -312.9566345214844,
+      "eval_loss": 0.044850025326013565,
+      "eval_rewards/accuracies": 0.6054999828338623,
+      "eval_rewards/chosen": -0.04393147677183151,
+      "eval_rewards/margins": 0.0252092145383358,
+      "eval_rewards/rejected": -0.06914069503545761,
+      "eval_runtime": 545.9668,
+      "eval_samples_per_second": 3.663,
+      "eval_steps_per_second": 0.916,
+      "step": 600
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.946408985913344e-06,
+      "logits/chosen": 0.3061693608760834,
+      "logits/rejected": 0.3176279067993164,
+      "logps/chosen": -294.21490478515625,
+      "logps/rejected": -283.79315185546875,
+      "loss": 0.0451,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.051018811762332916,
+      "rewards/margins": 0.014826146885752678,
+      "rewards/rejected": -0.06584496051073074,
+      "step": 610
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.941602180974958e-06,
+      "logits/chosen": 0.31741276383399963,
+      "logits/rejected": 0.34609508514404297,
+      "logps/chosen": -321.3012390136719,
+      "logps/rejected": -288.84735107421875,
+      "loss": 0.0494,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.03540085256099701,
+      "rewards/margins": 0.02708813175559044,
+      "rewards/rejected": -0.06248898059129715,
+      "step": 620
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.936591502957101e-06,
+      "logits/chosen": 0.2832370400428772,
+      "logits/rejected": 0.37036675214767456,
+      "logps/chosen": -325.6807556152344,
+      "logps/rejected": -296.6420593261719,
+      "loss": 0.0328,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.02868254855275154,
+      "rewards/margins": 0.019185787066817284,
+      "rewards/rejected": -0.047868333756923676,
+      "step": 630
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.931377370249946e-06,
+      "logits/chosen": 0.36788424849510193,
+      "logits/rejected": 0.4224618077278137,
+      "logps/chosen": -306.83856201171875,
+      "logps/rejected": -280.30963134765625,
+      "loss": 0.034,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.03303724154829979,
+      "rewards/margins": 0.0207848958671093,
+      "rewards/rejected": -0.05382214114069939,
+      "step": 640
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.925960218232073e-06,
+      "logits/chosen": 0.37691283226013184,
+      "logits/rejected": 0.44397860765457153,
+      "logps/chosen": -345.39642333984375,
+      "logps/rejected": -350.0865478515625,
+      "loss": 0.0375,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.04245168715715408,
+      "rewards/margins": 0.032366566359996796,
+      "rewards/rejected": -0.07481826096773148,
+      "step": 650
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.920340499234116e-06,
+      "logits/chosen": 0.2553911507129669,
+      "logits/rejected": 0.3338189721107483,
+      "logps/chosen": -324.5698547363281,
+      "logps/rejected": -267.8377685546875,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.04088371992111206,
+      "rewards/margins": 0.01900356262922287,
+      "rewards/rejected": -0.05988728255033493,
+      "step": 660
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.914518682500995e-06,
+      "logits/chosen": 0.3343955874443054,
+      "logits/rejected": 0.33443522453308105,
+      "logps/chosen": -314.8691711425781,
+      "logps/rejected": -336.9117431640625,
+      "loss": 0.0451,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.03791138157248497,
+      "rewards/margins": 0.030242133885622025,
+      "rewards/rejected": -0.068153515458107,
+      "step": 670
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.9084952541527315e-06,
+      "logits/chosen": 0.3156254291534424,
+      "logits/rejected": 0.32356199622154236,
+      "logps/chosen": -283.2015075683594,
+      "logps/rejected": -296.7607116699219,
+      "loss": 0.0424,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.028258180245757103,
+      "rewards/margins": 0.030845260247588158,
+      "rewards/rejected": -0.05910343676805496,
+      "step": 680
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.902270717143858e-06,
+      "logits/chosen": 0.30661842226982117,
+      "logits/rejected": 0.34706059098243713,
+      "logps/chosen": -316.7258605957031,
+      "logps/rejected": -303.54302978515625,
+      "loss": 0.0403,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.031144261360168457,
+      "rewards/margins": 0.028801072388887405,
+      "rewards/rejected": -0.05994533374905586,
+      "step": 690
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.895845591221427e-06,
+      "logits/chosen": 0.22924542427062988,
+      "logits/rejected": 0.321196049451828,
+      "logps/chosen": -298.3323669433594,
+      "logps/rejected": -305.1526184082031,
+      "loss": 0.0421,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.05126624554395676,
+      "rewards/margins": 0.030290687456727028,
+      "rewards/rejected": -0.08155693113803864,
+      "step": 700
+    },
+    {
+      "epoch": 0.18,
+      "eval_logits/chosen": 0.27075129747390747,
+      "eval_logits/rejected": 0.3239870071411133,
+      "eval_logps/chosen": -327.6275634765625,
+      "eval_logps/rejected": -322.8758544921875,
+      "eval_loss": 0.04373079910874367,
+      "eval_rewards/accuracies": 0.6054999828338623,
+      "eval_rewards/chosen": -0.05008076876401901,
+      "eval_rewards/margins": 0.02897915430366993,
+      "eval_rewards/rejected": -0.07905992865562439,
+      "eval_runtime": 545.8817,
+      "eval_samples_per_second": 3.664,
+      "eval_steps_per_second": 0.916,
+      "step": 700
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.8892204128816e-06,
+      "logits/chosen": 0.32626476883888245,
+      "logits/rejected": 0.3420587182044983,
+      "logps/chosen": -334.05633544921875,
+      "logps/rejected": -374.37664794921875,
+      "loss": 0.0363,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.051197003573179245,
+      "rewards/margins": 0.04748116806149483,
+      "rewards/rejected": -0.09867817163467407,
+      "step": 710
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.882395735324864e-06,
+      "logits/chosen": 0.32684141397476196,
+      "logits/rejected": 0.32078030705451965,
+      "logps/chosen": -310.9956359863281,
+      "logps/rejected": -308.5608825683594,
+      "loss": 0.0397,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.032676856964826584,
+      "rewards/margins": 0.027347933501005173,
+      "rewards/rejected": -0.06002478674054146,
+      "step": 720
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.87537212840983e-06,
+      "logits/chosen": 0.4430566728115082,
+      "logits/rejected": 0.4506394863128662,
+      "logps/chosen": -327.2930908203125,
+      "logps/rejected": -304.3199157714844,
+      "loss": 0.0434,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.027792710810899734,
+      "rewards/margins": 0.02709430828690529,
+      "rewards/rejected": -0.05488701909780502,
+      "step": 730
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.8681501786056545e-06,
+      "logits/chosen": 0.4311310648918152,
+      "logits/rejected": 0.48203498125076294,
+      "logps/chosen": -302.21441650390625,
+      "logps/rejected": -264.67327880859375,
+      "loss": 0.0584,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.032291196286678314,
+      "rewards/margins": 0.017856622114777565,
+      "rewards/rejected": -0.05014781281352043,
+      "step": 740
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.860730488943068e-06,
+      "logits/chosen": 0.471640020608902,
+      "logits/rejected": 0.48256558179855347,
+      "logps/chosen": -294.7103271484375,
+      "logps/rejected": -283.69610595703125,
+      "loss": 0.0461,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.017319316044449806,
+      "rewards/margins": 0.028396103531122208,
+      "rewards/rejected": -0.04571542143821716,
+      "step": 750
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.853113678964022e-06,
+      "logits/chosen": 0.473449170589447,
+      "logits/rejected": 0.47665268182754517,
+      "logps/chosen": -274.5668640136719,
+      "logps/rejected": -286.60028076171875,
+      "loss": 0.0405,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.02769545093178749,
+      "rewards/margins": 0.032251257449388504,
+      "rewards/rejected": -0.059946708381175995,
+      "step": 760
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.845300384669958e-06,
+      "logits/chosen": 0.4043947756290436,
+      "logits/rejected": 0.4248669743537903,
+      "logps/chosen": -296.60614013671875,
+      "logps/rejected": -269.669189453125,
+      "loss": 0.0536,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.03397484868764877,
+      "rewards/margins": 0.018872803077101707,
+      "rewards/rejected": -0.05284765362739563,
+      "step": 770
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.837291258468701e-06,
+      "logits/chosen": 0.3566601574420929,
+      "logits/rejected": 0.36324962973594666,
+      "logps/chosen": -314.69207763671875,
+      "logps/rejected": -311.7981872558594,
+      "loss": 0.0452,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.03935214877128601,
+      "rewards/margins": 0.03358140215277672,
+      "rewards/rejected": -0.07293355464935303,
+      "step": 780
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.829086969119984e-06,
+      "logits/chosen": 0.314022958278656,
+      "logits/rejected": 0.363187700510025,
+      "logps/chosen": -287.0098571777344,
+      "logps/rejected": -303.2685546875,
+      "loss": 0.0313,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.04219662398099899,
+      "rewards/margins": 0.028113823384046555,
+      "rewards/rejected": -0.07031045109033585,
+      "step": 790
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.820688201679605e-06,
+      "logits/chosen": 0.3411046862602234,
+      "logits/rejected": 0.372081458568573,
+      "logps/chosen": -299.74371337890625,
+      "logps/rejected": -296.0436096191406,
+      "loss": 0.0437,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.04190176725387573,
+      "rewards/margins": 0.028635144233703613,
+      "rewards/rejected": -0.07053691148757935,
+      "step": 800
+    },
+    {
+      "epoch": 0.21,
+      "eval_logits/chosen": 0.3235890567302704,
+      "eval_logits/rejected": 0.38052046298980713,
+      "eval_logps/chosen": -324.3805236816406,
+      "eval_logps/rejected": -318.0195617675781,
+      "eval_loss": 0.04284180328249931,
+      "eval_rewards/accuracies": 0.6004999876022339,
+      "eval_rewards/chosen": -0.04683370888233185,
+      "eval_rewards/margins": 0.027369938790798187,
+      "eval_rewards/rejected": -0.07420364022254944,
+      "eval_runtime": 546.0444,
+      "eval_samples_per_second": 3.663,
+      "eval_steps_per_second": 0.916,
+      "step": 800
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.8120956574422315e-06,
+      "logits/chosen": 0.370708167552948,
+      "logits/rejected": 0.355935275554657,
+      "logps/chosen": -328.8559265136719,
+      "logps/rejected": -349.7777099609375,
+      "loss": 0.0506,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.04853945970535278,
+      "rewards/margins": 0.02844676375389099,
+      "rewards/rejected": -0.07698621600866318,
+      "step": 810
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.803310053882831e-06,
+      "logits/chosen": 0.3484066128730774,
+      "logits/rejected": 0.3702424168586731,
+      "logps/chosen": -244.5006866455078,
+      "logps/rejected": -256.20587158203125,
+      "loss": 0.0455,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.03452844172716141,
+      "rewards/margins": 0.02526615560054779,
+      "rewards/rejected": -0.0597945973277092,
+      "step": 820
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.794332124596775e-06,
+      "logits/chosen": 0.35595011711120605,
+      "logits/rejected": 0.41656121611595154,
+      "logps/chosen": -308.20556640625,
+      "logps/rejected": -305.54296875,
+      "loss": 0.0389,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.03468139097094536,
+      "rewards/margins": 0.02495122142136097,
+      "rewards/rejected": -0.05963261052966118,
+      "step": 830
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.785162619238575e-06,
+      "logits/chosen": 0.37355029582977295,
+      "logits/rejected": 0.4001145362854004,
+      "logps/chosen": -295.16259765625,
+      "logps/rejected": -305.8782653808594,
+      "loss": 0.0409,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.03432817757129669,
+      "rewards/margins": 0.03204139322042465,
+      "rewards/rejected": -0.06636957079172134,
+      "step": 840
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.775802303459288e-06,
+      "logits/chosen": 0.3418852388858795,
+      "logits/rejected": 0.4243159890174866,
+      "logps/chosen": -326.20831298828125,
+      "logps/rejected": -322.5636291503906,
+      "loss": 0.044,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.03925400227308273,
+      "rewards/margins": 0.030559295788407326,
+      "rewards/rejected": -0.06981329619884491,
+      "step": 850
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.766251958842589e-06,
+      "logits/chosen": 0.32749515771865845,
+      "logits/rejected": 0.3575645089149475,
+      "logps/chosen": -344.33795166015625,
+      "logps/rejected": -339.56365966796875,
+      "loss": 0.0364,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.043786291033029556,
+      "rewards/margins": 0.03594813868403435,
+      "rewards/rejected": -0.0797344297170639,
+      "step": 860
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.7565123828395066e-06,
+      "logits/chosen": 0.26385778188705444,
+      "logits/rejected": 0.3685280382633209,
+      "logps/chosen": -362.4241027832031,
+      "logps/rejected": -367.7202453613281,
+      "loss": 0.0251,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.04894987866282463,
+      "rewards/margins": 0.03410911560058594,
+      "rewards/rejected": -0.08305899798870087,
+      "step": 870
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.746584388701831e-06,
+      "logits/chosen": 0.3249465525150299,
+      "logits/rejected": 0.40691623091697693,
+      "logps/chosen": -306.39874267578125,
+      "logps/rejected": -317.2301025390625,
+      "loss": 0.0591,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.05021210387349129,
+      "rewards/margins": 0.051097672432661057,
+      "rewards/rejected": -0.10130977630615234,
+      "step": 880
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.736468805414218e-06,
+      "logits/chosen": 0.33585670590400696,
+      "logits/rejected": 0.4310707449913025,
+      "logps/chosen": -337.23138427734375,
+      "logps/rejected": -358.4998474121094,
+      "loss": 0.0419,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.04472584277391434,
+      "rewards/margins": 0.05601406842470169,
+      "rewards/rejected": -0.10073991119861603,
+      "step": 890
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.7261664776249595e-06,
+      "logits/chosen": 0.2620421051979065,
+      "logits/rejected": 0.27202528715133667,
+      "logps/chosen": -325.6366882324219,
+      "logps/rejected": -351.74932861328125,
+      "loss": 0.0387,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.05709671229124069,
+      "rewards/margins": 0.04949341341853142,
+      "rewards/rejected": -0.10659012943506241,
+      "step": 900
+    },
+    {
+      "epoch": 0.24,
+      "eval_logits/chosen": 0.1997441053390503,
+      "eval_logits/rejected": 0.2502720355987549,
+      "eval_logps/chosen": -337.85150146484375,
+      "eval_logps/rejected": -341.3826904296875,
+      "eval_loss": 0.04230610653758049,
+      "eval_rewards/accuracies": 0.6054999828338623,
+      "eval_rewards/chosen": -0.06030464917421341,
+      "eval_rewards/margins": 0.037262119352817535,
+      "eval_rewards/rejected": -0.09756677597761154,
+      "eval_runtime": 545.91,
+      "eval_samples_per_second": 3.664,
+      "eval_steps_per_second": 0.916,
+      "step": 900
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.715678265575463e-06,
+      "logits/chosen": 0.283039927482605,
+      "logits/rejected": 0.32125529646873474,
+      "logps/chosen": -361.42388916015625,
+      "logps/rejected": -368.8393249511719,
+      "loss": 0.0405,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.053942956030368805,
+      "rewards/margins": 0.02801678143441677,
+      "rewards/rejected": -0.08195973932743073,
+      "step": 910
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.705005045028415e-06,
+      "logits/chosen": 0.3477228283882141,
+      "logits/rejected": 0.3734593689441681,
+      "logps/chosen": -368.44854736328125,
+      "logps/rejected": -356.55767822265625,
+      "loss": 0.0334,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.036770500242710114,
+      "rewards/margins": 0.04149743542075157,
+      "rewards/rejected": -0.07826793193817139,
+      "step": 920
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.694147707194659e-06,
+      "logits/chosen": 0.28998684883117676,
+      "logits/rejected": 0.33950483798980713,
+      "logps/chosen": -323.11016845703125,
+      "logps/rejected": -309.97064208984375,
+      "loss": 0.0406,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.03780774027109146,
+      "rewards/margins": 0.03742669150233269,
+      "rewards/rejected": -0.07523442804813385,
+      "step": 930
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.683107158658782e-06,
+      "logits/chosen": 0.22522148489952087,
+      "logits/rejected": 0.222640722990036,
+      "logps/chosen": -298.6249084472656,
+      "logps/rejected": -305.4351806640625,
+      "loss": 0.0441,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.03760068491101265,
+      "rewards/margins": 0.04090439900755882,
+      "rewards/rejected": -0.07850509136915207,
+      "step": 940
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.671884321303407e-06,
+      "logits/chosen": 0.199390709400177,
+      "logits/rejected": 0.19853922724723816,
+      "logps/chosen": -335.2737121582031,
+      "logps/rejected": -359.79571533203125,
+      "loss": 0.0389,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.05040610954165459,
+      "rewards/margins": 0.0430118627846241,
+      "rewards/rejected": -0.09341796487569809,
+      "step": 950
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.660480132232224e-06,
+      "logits/chosen": 0.15992198884487152,
+      "logits/rejected": 0.18765407800674438,
+      "logps/chosen": -325.8768005371094,
+      "logps/rejected": -326.781005859375,
+      "loss": 0.037,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.05003608390688896,
+      "rewards/margins": 0.03325340524315834,
+      "rewards/rejected": -0.0832894891500473,
+      "step": 960
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.6488955436917414e-06,
+      "logits/chosen": 0.21872511506080627,
+      "logits/rejected": 0.26562732458114624,
+      "logps/chosen": -328.88458251953125,
+      "logps/rejected": -279.19476318359375,
+      "loss": 0.0383,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.051375340670347214,
+      "rewards/margins": 0.016935134306550026,
+      "rewards/rejected": -0.06831048429012299,
+      "step": 970
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.6371315229917644e-06,
+      "logits/chosen": 0.22645604610443115,
+      "logits/rejected": 0.25760138034820557,
+      "logps/chosen": -303.7605895996094,
+      "logps/rejected": -314.8833923339844,
+      "loss": 0.0338,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.04115596413612366,
+      "rewards/margins": 0.033490367233753204,
+      "rewards/rejected": -0.07464633136987686,
+      "step": 980
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.625189052424638e-06,
+      "logits/chosen": 0.27312830090522766,
+      "logits/rejected": 0.31605759263038635,
+      "logps/chosen": -278.66796875,
+      "logps/rejected": -270.08209228515625,
+      "loss": 0.0448,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.03390710428357124,
+      "rewards/margins": 0.02822614088654518,
+      "rewards/rejected": -0.062133241444826126,
+      "step": 990
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.613069129183218e-06,
+      "logits/chosen": 0.2591247856616974,
+      "logits/rejected": 0.2805691063404083,
+      "logps/chosen": -306.6300964355469,
+      "logps/rejected": -321.5674743652344,
+      "loss": 0.0469,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.0464119091629982,
+      "rewards/margins": 0.0269215889275074,
+      "rewards/rejected": -0.0733334869146347,
+      "step": 1000
+    },
+    {
+      "epoch": 0.26,
+      "eval_logits/chosen": 0.2683161795139313,
+      "eval_logits/rejected": 0.3303489685058594,
+      "eval_logps/chosen": -319.0327453613281,
+      "eval_logps/rejected": -318.28564453125,
+      "eval_loss": 0.04103272035717964,
+      "eval_rewards/accuracies": 0.6119999885559082,
+      "eval_rewards/chosen": -0.04148593172430992,
+      "eval_rewards/margins": 0.03298382833600044,
+      "eval_rewards/rejected": -0.07446976006031036,
+      "eval_runtime": 545.9037,
+      "eval_samples_per_second": 3.664,
+      "eval_steps_per_second": 0.916,
+      "step": 1000
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.600772765277607e-06,
+      "logits/chosen": 0.27400195598602295,
+      "logits/rejected": 0.33995717763900757,
+      "logps/chosen": -337.406005859375,
+      "logps/rejected": -324.3661193847656,
+      "loss": 0.0439,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.0419384203851223,
+      "rewards/margins": 0.03884187340736389,
+      "rewards/rejected": -0.08078029751777649,
+      "step": 1010
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.588300987450652e-06,
+      "logits/chosen": 0.36324959993362427,
+      "logits/rejected": 0.41200870275497437,
+      "logps/chosen": -314.86676025390625,
+      "logps/rejected": -334.1313781738281,
+      "loss": 0.0367,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.03741372376680374,
+      "rewards/margins": 0.04047057032585144,
+      "rewards/rejected": -0.07788427919149399,
+      "step": 1020
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.5756548370922136e-06,
+      "logits/chosen": 0.3560029864311218,
+      "logits/rejected": 0.38821783661842346,
+      "logps/chosen": -314.72100830078125,
+      "logps/rejected": -309.0229187011719,
+      "loss": 0.0305,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.03693241626024246,
+      "rewards/margins": 0.024656260386109352,
+      "rewards/rejected": -0.061588674783706665,
+      "step": 1030
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.562835370152206e-06,
+      "logits/chosen": 0.29770249128341675,
+      "logits/rejected": 0.38089117407798767,
+      "logps/chosen": -284.8513488769531,
+      "logps/rejected": -290.6185607910156,
+      "loss": 0.04,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.03536974638700485,
+      "rewards/margins": 0.023334329947829247,
+      "rewards/rejected": -0.05870407819747925,
+      "step": 1040
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.54984365705243e-06,
+      "logits/chosen": 0.33493131399154663,
+      "logits/rejected": 0.3345734477043152,
+      "logps/chosen": -285.49566650390625,
+      "logps/rejected": -286.21490478515625,
+      "loss": 0.0456,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.03364426642656326,
+      "rewards/margins": 0.028464287519454956,
+      "rewards/rejected": -0.06210855394601822,
+      "step": 1050
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.536680782597191e-06,
+      "logits/chosen": 0.3045693039894104,
+      "logits/rejected": 0.38494449853897095,
+      "logps/chosen": -327.88916015625,
+      "logps/rejected": -326.697509765625,
+      "loss": 0.0371,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.03162507340312004,
+      "rewards/margins": 0.03284720331430435,
+      "rewards/rejected": -0.0644722729921341,
+      "step": 1060
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.523347845882718e-06,
+      "logits/chosen": 0.341009259223938,
+      "logits/rejected": 0.3752421736717224,
+      "logps/chosen": -293.13201904296875,
+      "logps/rejected": -272.94171142578125,
+      "loss": 0.0427,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.03778553009033203,
+      "rewards/margins": 0.03575527295470238,
+      "rewards/rejected": -0.07354080677032471,
+      "step": 1070
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.50984596020539e-06,
+      "logits/chosen": 0.3082283139228821,
+      "logits/rejected": 0.3483355939388275,
+      "logps/chosen": -335.3013916015625,
+      "logps/rejected": -332.9034118652344,
+      "loss": 0.0401,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.037971124053001404,
+      "rewards/margins": 0.03771374002099037,
+      "rewards/rejected": -0.07568486034870148,
+      "step": 1080
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.4961762529687745e-06,
+      "logits/chosen": 0.26932334899902344,
+      "logits/rejected": 0.33773109316825867,
+      "logps/chosen": -356.42828369140625,
+      "logps/rejected": -315.24591064453125,
+      "loss": 0.0408,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.055332690477371216,
+      "rewards/margins": 0.034721001982688904,
+      "rewards/rejected": -0.09005369246006012,
+      "step": 1090
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.482339865589492e-06,
+      "logits/chosen": 0.23695580661296844,
+      "logits/rejected": 0.34014320373535156,
+      "logps/chosen": -309.662109375,
+      "logps/rejected": -320.1878662109375,
+      "loss": 0.0405,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.04775983840227127,
+      "rewards/margins": 0.047388456761837006,
+      "rewards/rejected": -0.09514829516410828,
+      "step": 1100
+    },
+    {
+      "epoch": 0.29,
+      "eval_logits/chosen": 0.30217111110687256,
+      "eval_logits/rejected": 0.3568549156188965,
+      "eval_logps/chosen": -337.92388916015625,
+      "eval_logps/rejected": -339.1554870605469,
+      "eval_loss": 0.041309088468551636,
+      "eval_rewards/accuracies": 0.6065000295639038,
+      "eval_rewards/chosen": -0.060377102345228195,
+      "eval_rewards/margins": 0.034962501376867294,
+      "eval_rewards/rejected": -0.09533959627151489,
+      "eval_runtime": 545.944,
+      "eval_samples_per_second": 3.663,
+      "eval_steps_per_second": 0.916,
+      "step": 1100
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.468337953401909e-06,
+      "logits/chosen": 0.35202285647392273,
+      "logits/rejected": 0.37908273935317993,
+      "logps/chosen": -350.8187255859375,
+      "logps/rejected": -320.697265625,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.05524873733520508,
+      "rewards/margins": 0.02713766135275364,
+      "rewards/rejected": -0.08238638937473297,
+      "step": 1110
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.45417168556166e-06,
+      "logits/chosen": 0.3267337679862976,
+      "logits/rejected": 0.3767111003398895,
+      "logps/chosen": -292.25323486328125,
+      "logps/rejected": -324.53277587890625,
+      "loss": 0.037,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.03478919714689255,
+      "rewards/margins": 0.03577711060643196,
+      "rewards/rejected": -0.07056631147861481,
+      "step": 1120
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.439842244948036e-06,
+      "logits/chosen": 0.29826876521110535,
+      "logits/rejected": 0.3435824513435364,
+      "logps/chosen": -301.586669921875,
+      "logps/rejected": -312.9870300292969,
+      "loss": 0.044,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.04382818937301636,
+      "rewards/margins": 0.032506901770830154,
+      "rewards/rejected": -0.07633509486913681,
+      "step": 1130
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.425350828065204e-06,
+      "logits/chosen": 0.2747456431388855,
+      "logits/rejected": 0.2777434289455414,
+      "logps/chosen": -330.4239196777344,
+      "logps/rejected": -348.69091796875,
+      "loss": 0.037,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.05098523572087288,
+      "rewards/margins": 0.03946097940206528,
+      "rewards/rejected": -0.09044621884822845,
+      "step": 1140
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.410698644942303e-06,
+      "logits/chosen": 0.2730598449707031,
+      "logits/rejected": 0.2936861217021942,
+      "logps/chosen": -327.4627380371094,
+      "logps/rejected": -367.5867004394531,
+      "loss": 0.0358,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.0492265447974205,
+      "rewards/margins": 0.03990650177001953,
+      "rewards/rejected": -0.08913305401802063,
+      "step": 1150
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.395886919032406e-06,
+      "logits/chosen": 0.2447071075439453,
+      "logits/rejected": 0.2843990921974182,
+      "logps/chosen": -317.3401184082031,
+      "logps/rejected": -307.0522155761719,
+      "loss": 0.0389,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.04608890414237976,
+      "rewards/margins": 0.036723993718624115,
+      "rewards/rejected": -0.08281290531158447,
+      "step": 1160
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.380916887110366e-06,
+      "logits/chosen": 0.1792442500591278,
+      "logits/rejected": 0.24730241298675537,
+      "logps/chosen": -292.51007080078125,
+      "logps/rejected": -297.1490783691406,
+      "loss": 0.0374,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.044197600334882736,
+      "rewards/margins": 0.0385957695543766,
+      "rewards/rejected": -0.08279336988925934,
+      "step": 1170
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.365789799169539e-06,
+      "logits/chosen": 0.1932184249162674,
+      "logits/rejected": 0.16758938133716583,
+      "logps/chosen": -361.5670471191406,
+      "logps/rejected": -353.1756591796875,
+      "loss": 0.0382,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.053414952009916306,
+      "rewards/margins": 0.04763371869921684,
+      "rewards/rejected": -0.10104867070913315,
+      "step": 1180
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.350506918317416e-06,
+      "logits/chosen": 0.15172933042049408,
+      "logits/rejected": 0.11963067203760147,
+      "logps/chosen": -350.88055419921875,
+      "logps/rejected": -350.66571044921875,
+      "loss": 0.0321,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.04808942228555679,
+      "rewards/margins": 0.034762658178806305,
+      "rewards/rejected": -0.0828520804643631,
+      "step": 1190
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.335069520670149e-06,
+      "logits/chosen": 0.1585710644721985,
+      "logits/rejected": 0.21485576033592224,
+      "logps/chosen": -327.654541015625,
+      "logps/rejected": -312.57318115234375,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.05130019783973694,
+      "rewards/margins": 0.05567903444170952,
+      "rewards/rejected": -0.10697922855615616,
+      "step": 1200
+    },
+    {
+      "epoch": 0.31,
+      "eval_logits/chosen": 0.12611083686351776,
+      "eval_logits/rejected": 0.1741568148136139,
+      "eval_logps/chosen": -339.1230773925781,
+      "eval_logps/rejected": -347.98687744140625,
+      "eval_loss": 0.041399095207452774,
+      "eval_rewards/accuracies": 0.6150000095367432,
+      "eval_rewards/chosen": -0.06157625466585159,
+      "eval_rewards/margins": 0.04259469732642174,
+      "eval_rewards/rejected": -0.10417096316814423,
+      "eval_runtime": 545.8171,
+      "eval_samples_per_second": 3.664,
+      "eval_steps_per_second": 0.916,
+      "step": 1200
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.319478895246e-06,
+      "logits/chosen": 0.15003447234630585,
+      "logits/rejected": 0.10985253751277924,
+      "logps/chosen": -326.82501220703125,
+      "logps/rejected": -325.7549743652344,
+      "loss": 0.0418,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.0526912696659565,
+      "rewards/margins": 0.04034797102212906,
+      "rewards/rejected": -0.09303922951221466,
+      "step": 1210
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.303736343857704e-06,
+      "logits/chosen": 0.2608622610569,
+      "logits/rejected": 0.2155006378889084,
+      "logps/chosen": -319.96917724609375,
+      "logps/rejected": -343.73944091796875,
+      "loss": 0.049,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.05587207153439522,
+      "rewards/margins": 0.04675058647990227,
+      "rewards/rejected": -0.10262266546487808,
+      "step": 1220
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.287843181003772e-06,
+      "logits/chosen": 0.2440902292728424,
+      "logits/rejected": 0.29910004138946533,
+      "logps/chosen": -323.3177185058594,
+      "logps/rejected": -327.98834228515625,
+      "loss": 0.0404,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.04564564675092697,
+      "rewards/margins": 0.05332164093852043,
+      "rewards/rejected": -0.0989672914147377,
+      "step": 1230
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.27180073375873e-06,
+      "logits/chosen": 0.2225826233625412,
+      "logits/rejected": 0.22676649689674377,
+      "logps/chosen": -330.525634765625,
+      "logps/rejected": -348.46893310546875,
+      "loss": 0.0389,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.05276740342378616,
+      "rewards/margins": 0.05104779079556465,
+      "rewards/rejected": -0.10381519794464111,
+      "step": 1240
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.255610341662304e-06,
+      "logits/chosen": 0.23166510462760925,
+      "logits/rejected": 0.2744370996952057,
+      "logps/chosen": -350.92608642578125,
+      "logps/rejected": -350.80963134765625,
+      "loss": 0.0415,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.046459149569272995,
+      "rewards/margins": 0.04404251649975777,
+      "rewards/rejected": -0.09050166606903076,
+      "step": 1250
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.2392733566075764e-06,
+      "logits/chosen": 0.2495725452899933,
+      "logits/rejected": 0.2777150273323059,
+      "logps/chosen": -291.45001220703125,
+      "logps/rejected": -309.00372314453125,
+      "loss": 0.0433,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.05061071366071701,
+      "rewards/margins": 0.04214351624250412,
+      "rewards/rejected": -0.09275422990322113,
+      "step": 1260
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.2227911427280975e-06,
+      "logits/chosen": 0.28404611349105835,
+      "logits/rejected": 0.3251902461051941,
+      "logps/chosen": -323.915283203125,
+      "logps/rejected": -335.4839172363281,
+      "loss": 0.0421,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.04012744873762131,
+      "rewards/margins": 0.03166784718632698,
+      "rewards/rejected": -0.07179529964923859,
+      "step": 1270
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.206165076283983e-06,
+      "logits/chosen": 0.2753671407699585,
+      "logits/rejected": 0.3080431818962097,
+      "logps/chosen": -267.154052734375,
+      "logps/rejected": -272.03875732421875,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.030480539426207542,
+      "rewards/margins": 0.01944340392947197,
+      "rewards/rejected": -0.04992394149303436,
+      "step": 1280
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.189396545546995e-06,
+      "logits/chosen": 0.33887535333633423,
+      "logits/rejected": 0.4030800461769104,
+      "logps/chosen": -315.8049011230469,
+      "logps/rejected": -323.9139709472656,
+      "loss": 0.0382,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.027971679344773293,
+      "rewards/margins": 0.020772229880094528,
+      "rewards/rejected": -0.04874390736222267,
+      "step": 1290
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.172486950684627e-06,
+      "logits/chosen": 0.32417652010917664,
+      "logits/rejected": 0.34953024983406067,
+      "logps/chosen": -311.4471740722656,
+      "logps/rejected": -318.8915710449219,
+      "loss": 0.0421,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.03494606912136078,
+      "rewards/margins": 0.029453804716467857,
+      "rewards/rejected": -0.06439987570047379,
+      "step": 1300
+    },
+    {
+      "epoch": 0.34,
+      "eval_logits/chosen": 0.26883384585380554,
+      "eval_logits/rejected": 0.32785192131996155,
+      "eval_logps/chosen": -313.69818115234375,
+      "eval_logps/rejected": -311.56353759765625,
+      "eval_loss": 0.04006488621234894,
+      "eval_rewards/accuracies": 0.6240000128746033,
+      "eval_rewards/chosen": -0.03615137189626694,
+      "eval_rewards/margins": 0.03159624710679054,
+      "eval_rewards/rejected": -0.06774762272834778,
+      "eval_runtime": 546.0119,
+      "eval_samples_per_second": 3.663,
+      "eval_steps_per_second": 0.916,
+      "step": 1300
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.155437703643182e-06,
+      "logits/chosen": 0.22474968433380127,
+      "logits/rejected": 0.23807482421398163,
+      "logps/chosen": -317.81878662109375,
+      "logps/rejected": -319.33856201171875,
+      "loss": 0.0352,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.03495235741138458,
+      "rewards/margins": 0.041259001940488815,
+      "rewards/rejected": -0.0762113705277443,
+      "step": 1310
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.138250228029882e-06,
+      "logits/chosen": 0.2451280653476715,
+      "logits/rejected": 0.3115343451499939,
+      "logps/chosen": -271.7696533203125,
+      "logps/rejected": -288.1288146972656,
+      "loss": 0.0487,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.03968612104654312,
+      "rewards/margins": 0.040365613996982574,
+      "rewards/rejected": -0.0800517350435257,
+      "step": 1320
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.120925958993994e-06,
+      "logits/chosen": 0.2183457911014557,
+      "logits/rejected": 0.2512846291065216,
+      "logps/chosen": -341.5397033691406,
+      "logps/rejected": -319.7643127441406,
+      "loss": 0.0417,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.04834596440196037,
+      "rewards/margins": 0.029247086495161057,
+      "rewards/rejected": -0.07759305834770203,
+      "step": 1330
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.103466343106999e-06,
+      "logits/chosen": 0.19329051673412323,
+      "logits/rejected": 0.1946055293083191,
+      "logps/chosen": -321.7462463378906,
+      "logps/rejected": -319.4334716796875,
+      "loss": 0.0397,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.049243781715631485,
+      "rewards/margins": 0.03379587084054947,
+      "rewards/rejected": -0.08303964883089066,
+      "step": 1340
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.085872838241797e-06,
+      "logits/chosen": 0.20951516926288605,
+      "logits/rejected": 0.21771371364593506,
+      "logps/chosen": -342.02996826171875,
+      "logps/rejected": -379.24810791015625,
+      "loss": 0.0364,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.05687708407640457,
+      "rewards/margins": 0.04443209990859032,
+      "rewards/rejected": -0.10130918025970459,
+      "step": 1350
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.06814691345098e-06,
+      "logits/chosen": 0.18391954898834229,
+      "logits/rejected": 0.2398659735918045,
+      "logps/chosen": -372.8047790527344,
+      "logps/rejected": -364.9682312011719,
+      "loss": 0.0422,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.07316961139440536,
+      "rewards/margins": 0.04260207712650299,
+      "rewards/rejected": -0.11577168852090836,
+      "step": 1360
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.050290048844171e-06,
+      "logits/chosen": 0.20069186389446259,
+      "logits/rejected": 0.21868674457073212,
+      "logps/chosen": -323.4859313964844,
+      "logps/rejected": -319.8496398925781,
+      "loss": 0.0458,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.06887100636959076,
+      "rewards/margins": 0.03058711811900139,
+      "rewards/rejected": -0.09945812076330185,
+      "step": 1370
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.032303735464422e-06,
+      "logits/chosen": 0.1730354130268097,
+      "logits/rejected": 0.22356243431568146,
+      "logps/chosen": -353.87158203125,
+      "logps/rejected": -354.8973083496094,
+      "loss": 0.0466,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.0647386908531189,
+      "rewards/margins": 0.03853858262300491,
+      "rewards/rejected": -0.1032772809267044,
+      "step": 1380
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.014189475163727e-06,
+      "logits/chosen": 0.17886124551296234,
+      "logits/rejected": 0.23550216853618622,
+      "logps/chosen": -327.8953857421875,
+      "logps/rejected": -328.9718322753906,
+      "loss": 0.0493,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.06163940578699112,
+      "rewards/margins": 0.03214733302593231,
+      "rewards/rejected": -0.09378673136234283,
+      "step": 1390
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.995948780477605e-06,
+      "logits/chosen": 0.2545778751373291,
+      "logits/rejected": 0.266729474067688,
+      "logps/chosen": -346.52520751953125,
+      "logps/rejected": -335.43988037109375,
+      "loss": 0.0454,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.06019485741853714,
+      "rewards/margins": 0.033053819090127945,
+      "rewards/rejected": -0.09324866533279419,
+      "step": 1400
+    },
+    {
+      "epoch": 0.37,
+      "eval_logits/chosen": 0.20339642465114594,
+      "eval_logits/rejected": 0.25649207830429077,
+      "eval_logps/chosen": -344.02374267578125,
+      "eval_logps/rejected": -346.2301940917969,
+      "eval_loss": 0.04007468745112419,
+      "eval_rewards/accuracies": 0.6129999756813049,
+      "eval_rewards/chosen": -0.0664769634604454,
+      "eval_rewards/margins": 0.035937339067459106,
+      "eval_rewards/rejected": -0.1024143174290657,
+      "eval_runtime": 545.7831,
+      "eval_samples_per_second": 3.664,
+      "eval_steps_per_second": 0.916,
+      "step": 1400
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.977583174498816e-06,
+      "logits/chosen": 0.20291456580162048,
+      "logits/rejected": 0.2241814136505127,
+      "logps/chosen": -337.1974182128906,
+      "logps/rejected": -333.82037353515625,
+      "loss": 0.0396,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.06416570395231247,
+      "rewards/margins": 0.03229355812072754,
+      "rewards/rejected": -0.0964592695236206,
+      "step": 1410
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.959094190750172e-06,
+      "logits/chosen": 0.1814291924238205,
+      "logits/rejected": 0.23034696280956268,
+      "logps/chosen": -369.82977294921875,
+      "logps/rejected": -365.26007080078125,
+      "loss": 0.0455,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.08425476402044296,
+      "rewards/margins": 0.0281287282705307,
+      "rewards/rejected": -0.11238348484039307,
+      "step": 1420
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.9404833730564975e-06,
+      "logits/chosen": 0.22389426827430725,
+      "logits/rejected": 0.2450193464756012,
+      "logps/chosen": -339.3416442871094,
+      "logps/rejected": -371.50006103515625,
+      "loss": 0.0488,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.09302830696105957,
+      "rewards/margins": 0.036248233169317245,
+      "rewards/rejected": -0.12927654385566711,
+      "step": 1430
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.921752275415712e-06,
+      "logits/chosen": 0.1862931251525879,
+      "logits/rejected": 0.21857285499572754,
+      "logps/chosen": -327.964599609375,
+      "logps/rejected": -333.35113525390625,
+      "loss": 0.0417,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.0893184170126915,
+      "rewards/margins": 0.037862036377191544,
+      "rewards/rejected": -0.12718045711517334,
+      "step": 1440
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.902902461869079e-06,
+      "logits/chosen": 0.18089166283607483,
+      "logits/rejected": 0.2563425898551941,
+      "logps/chosen": -347.0341491699219,
+      "logps/rejected": -358.7850341796875,
+      "loss": 0.043,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.07840432226657867,
+      "rewards/margins": 0.044789306819438934,
+      "rewards/rejected": -0.1231936365365982,
+      "step": 1450
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.883935506370605e-06,
+      "logits/chosen": 0.16234511137008667,
+      "logits/rejected": 0.2073783427476883,
+      "logps/chosen": -349.12457275390625,
+      "logps/rejected": -346.66839599609375,
+      "loss": 0.0333,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.07234518229961395,
+      "rewards/margins": 0.040105998516082764,
+      "rewards/rejected": -0.11245118081569672,
+      "step": 1460
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.864852992655617e-06,
+      "logits/chosen": 0.1625242978334427,
+      "logits/rejected": 0.22156274318695068,
+      "logps/chosen": -327.12774658203125,
+      "logps/rejected": -355.7247619628906,
+      "loss": 0.0401,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.06119702011346817,
+      "rewards/margins": 0.04420281946659088,
+      "rewards/rejected": -0.10539983212947845,
+      "step": 1470
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.845656514108516e-06,
+      "logits/chosen": 0.17640124261379242,
+      "logits/rejected": 0.21967816352844238,
+      "logps/chosen": -361.7919006347656,
+      "logps/rejected": -350.9843444824219,
+      "loss": 0.0372,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.0684257298707962,
+      "rewards/margins": 0.04318644851446152,
+      "rewards/rejected": -0.11161217838525772,
+      "step": 1480
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.826347673629738e-06,
+      "logits/chosen": 0.22848467528820038,
+      "logits/rejected": 0.22444598376750946,
+      "logps/chosen": -310.12060546875,
+      "logps/rejected": -348.86627197265625,
+      "loss": 0.0343,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.06757032871246338,
+      "rewards/margins": 0.06018294021487236,
+      "rewards/rejected": -0.12775327265262604,
+      "step": 1490
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.8069280835019062e-06,
+      "logits/chosen": 0.19491688907146454,
+      "logits/rejected": 0.23463113605976105,
+      "logps/chosen": -393.9888916015625,
+      "logps/rejected": -358.51007080078125,
+      "loss": 0.03,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.07187078893184662,
+      "rewards/margins": 0.03775522857904434,
+      "rewards/rejected": -0.10962603241205215,
+      "step": 1500
+    },
+    {
+      "epoch": 0.39,
+      "eval_logits/chosen": 0.19577667117118835,
+      "eval_logits/rejected": 0.2511560022830963,
+      "eval_logps/chosen": -358.4021301269531,
+      "eval_logps/rejected": -367.09576416015625,
+      "eval_loss": 0.03940973058342934,
+      "eval_rewards/accuracies": 0.6184999942779541,
+      "eval_rewards/chosen": -0.08085529506206512,
+      "eval_rewards/margins": 0.04242454469203949,
+      "eval_rewards/rejected": -0.12327982485294342,
+      "eval_runtime": 546.0316,
+      "eval_samples_per_second": 3.663,
+      "eval_steps_per_second": 0.916,
+      "step": 1500
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7873993652552077e-06,
+      "logits/chosen": 0.20397154986858368,
+      "logits/rejected": 0.27242645621299744,
+      "logps/chosen": -337.03643798828125,
+      "logps/rejected": -333.9397888183594,
+      "loss": 0.0351,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.0792369395494461,
+      "rewards/margins": 0.02091343142092228,
+      "rewards/rejected": -0.10015036165714264,
+      "step": 1510
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7677631495319953e-06,
+      "logits/chosen": 0.23687753081321716,
+      "logits/rejected": 0.2800650894641876,
+      "logps/chosen": -324.31964111328125,
+      "logps/rejected": -304.57672119140625,
+      "loss": 0.0358,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.06605817377567291,
+      "rewards/margins": 0.029645001515746117,
+      "rewards/rejected": -0.09570317715406418,
+      "step": 1520
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.748021075950633e-06,
+      "logits/chosen": 0.3244825005531311,
+      "logits/rejected": 0.300965815782547,
+      "logps/chosen": -384.5220947265625,
+      "logps/rejected": -393.98504638671875,
+      "loss": 0.0395,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.0824432298541069,
+      "rewards/margins": 0.04170336574316025,
+      "rewards/rejected": -0.12414660304784775,
+      "step": 1530
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7281747929685824e-06,
+      "logits/chosen": 0.26002877950668335,
+      "logits/rejected": 0.34025731682777405,
+      "logps/chosen": -364.6267395019531,
+      "logps/rejected": -355.21112060546875,
+      "loss": 0.0402,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.0765707939863205,
+      "rewards/margins": 0.03082425333559513,
+      "rewards/rejected": -0.10739505290985107,
+      "step": 1540
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.7082259577447604e-06,
+      "logits/chosen": 0.2326231449842453,
+      "logits/rejected": 0.2870689928531647,
+      "logps/chosen": -408.60430908203125,
+      "logps/rejected": -377.21893310546875,
+      "loss": 0.0399,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.08911335468292236,
+      "rewards/margins": 0.042952969670295715,
+      "rewards/rejected": -0.13206632435321808,
+      "step": 1550
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6881762360011688e-06,
+      "logits/chosen": 0.29370513558387756,
+      "logits/rejected": 0.3216298818588257,
+      "logps/chosen": -370.08233642578125,
+      "logps/rejected": -355.87908935546875,
+      "loss": 0.0489,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.08413463085889816,
+      "rewards/margins": 0.033007871359586716,
+      "rewards/rejected": -0.11714249849319458,
+      "step": 1560
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.668027301883802e-06,
+      "logits/chosen": 0.2686474621295929,
+      "logits/rejected": 0.30647069215774536,
+      "logps/chosen": -333.7930603027344,
+      "logps/rejected": -323.7529296875,
+      "loss": 0.0389,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.07236778736114502,
+      "rewards/margins": 0.03711923211812973,
+      "rewards/rejected": -0.10948701202869415,
+      "step": 1570
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.64778083782286e-06,
+      "logits/chosen": 0.34584125876426697,
+      "logits/rejected": 0.38025742769241333,
+      "logps/chosen": -322.25927734375,
+      "logps/rejected": -361.61773681640625,
+      "loss": 0.0382,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.05563216656446457,
+      "rewards/margins": 0.036859843879938126,
+      "rewards/rejected": -0.0924920067191124,
+      "step": 1580
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.627438534392268e-06,
+      "logits/chosen": 0.3150128126144409,
+      "logits/rejected": 0.3379836976528168,
+      "logps/chosen": -309.7145690917969,
+      "logps/rejected": -283.8314208984375,
+      "loss": 0.0412,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.04880645126104355,
+      "rewards/margins": 0.028770333155989647,
+      "rewards/rejected": -0.07757677882909775,
+      "step": 1590
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.607002090168506e-06,
+      "logits/chosen": 0.3030504584312439,
+      "logits/rejected": 0.3338678181171417,
+      "logps/chosen": -311.2512512207031,
+      "logps/rejected": -315.3307189941406,
+      "loss": 0.0455,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.053125374019145966,
+      "rewards/margins": 0.03133867681026459,
+      "rewards/rejected": -0.08446405827999115,
+      "step": 1600
+    },
+    {
+      "epoch": 0.42,
+      "eval_logits/chosen": 0.2801807224750519,
+      "eval_logits/rejected": 0.3432122468948364,
+      "eval_logps/chosen": -330.363037109375,
+      "eval_logps/rejected": -330.2539367675781,
+      "eval_loss": 0.03898797556757927,
+      "eval_rewards/accuracies": 0.621999979019165,
+      "eval_rewards/chosen": -0.05281621217727661,
+      "eval_rewards/margins": 0.033621810376644135,
+      "eval_rewards/rejected": -0.08643801510334015,
+      "eval_runtime": 545.9003,
+      "eval_samples_per_second": 3.664,
+      "eval_steps_per_second": 0.916,
+      "step": 1600
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.586473211588787e-06,
+      "logits/chosen": 0.31370675563812256,
+      "logits/rejected": 0.38415977358818054,
+      "logps/chosen": -341.82183837890625,
+      "logps/rejected": -319.1004943847656,
+      "loss": 0.0458,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.04285822808742523,
+      "rewards/margins": 0.04011955484747887,
+      "rewards/rejected": -0.082977794110775,
+      "step": 1610
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.5658536128085623e-06,
+      "logits/chosen": 0.22774699330329895,
+      "logits/rejected": 0.26617830991744995,
+      "logps/chosen": -300.5567626953125,
+      "logps/rejected": -328.35601806640625,
+      "loss": 0.0346,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.05109331011772156,
+      "rewards/margins": 0.03925568237900734,
+      "rewards/rejected": -0.0903489887714386,
+      "step": 1620
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.545145015558399e-06,
+      "logits/chosen": 0.25213536620140076,
+      "logits/rejected": 0.2595962584018707,
+      "logps/chosen": -358.02789306640625,
+      "logps/rejected": -374.4208984375,
+      "loss": 0.0498,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.059232644736766815,
+      "rewards/margins": 0.03194702789187431,
+      "rewards/rejected": -0.09117966890335083,
+      "step": 1630
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.5243491490002056e-06,
+      "logits/chosen": 0.2583410143852234,
+      "logits/rejected": 0.2700033187866211,
+      "logps/chosen": -324.75421142578125,
+      "logps/rejected": -319.680419921875,
+      "loss": 0.0457,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.05090164393186569,
+      "rewards/margins": 0.03180782124400139,
+      "rewards/rejected": -0.08270946890115738,
+      "step": 1640
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.503467749582857e-06,
+      "logits/chosen": 0.25508540868759155,
+      "logits/rejected": 0.2542564272880554,
+      "logps/chosen": -287.2757873535156,
+      "logps/rejected": -271.09893798828125,
+      "loss": 0.0408,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.03935082629323006,
+      "rewards/margins": 0.03368367999792099,
+      "rewards/rejected": -0.07303450256586075,
+      "step": 1650
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.4825025608971947e-06,
+      "logits/chosen": 0.21389874815940857,
+      "logits/rejected": 0.2913588881492615,
+      "logps/chosen": -360.9371032714844,
+      "logps/rejected": -329.3777160644531,
+      "loss": 0.0384,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.046096112579107285,
+      "rewards/margins": 0.03987763077020645,
+      "rewards/rejected": -0.08597374707460403,
+      "step": 1660
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4614553335304407e-06,
+      "logits/chosen": 0.24760949611663818,
+      "logits/rejected": 0.3152398467063904,
+      "logps/chosen": -325.326171875,
+      "logps/rejected": -320.1291198730469,
+      "loss": 0.0449,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.049295853823423386,
+      "rewards/margins": 0.0397740975022316,
+      "rewards/rejected": -0.08906995505094528,
+      "step": 1670
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4403278249200222e-06,
+      "logits/chosen": 0.17812004685401917,
+      "logits/rejected": 0.2227652370929718,
+      "logps/chosen": -304.6307373046875,
+      "logps/rejected": -312.78497314453125,
+      "loss": 0.0459,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.04751410335302353,
+      "rewards/margins": 0.030996937304735184,
+      "rewards/rejected": -0.07851104438304901,
+      "step": 1680
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4191217992068293e-06,
+      "logits/chosen": 0.19028082489967346,
+      "logits/rejected": 0.17822106182575226,
+      "logps/chosen": -314.05670166015625,
+      "logps/rejected": -340.91180419921875,
+      "loss": 0.0361,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.05841837078332901,
+      "rewards/margins": 0.03385898843407631,
+      "rewards/rejected": -0.09227735549211502,
+      "step": 1690
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.3978390270879056e-06,
+      "logits/chosen": 0.15301261842250824,
+      "logits/rejected": 0.20801348984241486,
+      "logps/chosen": -346.69677734375,
+      "logps/rejected": -337.8739929199219,
+      "loss": 0.0444,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.05207078531384468,
+      "rewards/margins": 0.039416152983903885,
+      "rewards/rejected": -0.09148693829774857,
+      "step": 1700
+    },
+    {
+      "epoch": 0.44,
+      "eval_logits/chosen": 0.14330101013183594,
+      "eval_logits/rejected": 0.19559501111507416,
+      "eval_logps/chosen": -335.1628723144531,
+      "eval_logps/rejected": -339.50146484375,
+      "eval_loss": 0.0383184589445591,
+      "eval_rewards/accuracies": 0.6215000152587891,
+      "eval_rewards/chosen": -0.05761606991291046,
+      "eval_rewards/margins": 0.038069479167461395,
+      "eval_rewards/rejected": -0.09568555653095245,
+      "eval_runtime": 545.758,
+      "eval_samples_per_second": 3.665,
+      "eval_steps_per_second": 0.916,
+      "step": 1700
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3764812856685995e-06,
+      "logits/chosen": 0.15780293941497803,
+      "logits/rejected": 0.1472981721162796,
+      "logps/chosen": -341.84844970703125,
+      "logps/rejected": -353.3861999511719,
+      "loss": 0.0305,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.05185488611459732,
+      "rewards/margins": 0.04620691388845444,
+      "rewards/rejected": -0.09806180745363235,
+      "step": 1710
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3550503583141726e-06,
+      "logits/chosen": 0.1516754925251007,
+      "logits/rejected": 0.1589139699935913,
+      "logps/chosen": -391.5256652832031,
+      "logps/rejected": -419.0575256347656,
+      "loss": 0.0306,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.06893479079008102,
+      "rewards/margins": 0.0528082549571991,
+      "rewards/rejected": -0.12174304574728012,
+      "step": 1720
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3335480345008907e-06,
+      "logits/chosen": 0.11530669778585434,
+      "logits/rejected": 0.14827430248260498,
+      "logps/chosen": -330.6746520996094,
+      "logps/rejected": -335.7961120605469,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.068514384329319,
+      "rewards/margins": 0.04822620749473572,
+      "rewards/rejected": -0.11674059927463531,
+      "step": 1730
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.3119761096666055e-06,
+      "logits/chosen": 0.16575901210308075,
+      "logits/rejected": 0.14071877300739288,
+      "logps/chosen": -388.40667724609375,
+      "logps/rejected": -365.96282958984375,
+      "loss": 0.04,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.07761996239423752,
+      "rewards/margins": 0.034099433571100235,
+      "rewards/rejected": -0.11171939224004745,
+      "step": 1740
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.290336385060832e-06,
+      "logits/chosen": 0.1507655680179596,
+      "logits/rejected": 0.12852510809898376,
+      "logps/chosen": -352.5316467285156,
+      "logps/rejected": -374.3671875,
+      "loss": 0.0387,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.06853355467319489,
+      "rewards/margins": 0.04978446662425995,
+      "rewards/rejected": -0.11831802129745483,
+      "step": 1750
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.268630667594348e-06,
+      "logits/chosen": 0.0908082127571106,
+      "logits/rejected": 0.12417636066675186,
+      "logps/chosen": -356.6025695800781,
+      "logps/rejected": -366.72003173828125,
+      "loss": 0.0347,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.0775667354464531,
+      "rewards/margins": 0.03338538855314255,
+      "rewards/rejected": -0.11095212399959564,
+      "step": 1760
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.2468607696883147e-06,
+      "logits/chosen": 0.11028660833835602,
+      "logits/rejected": 0.12428678572177887,
+      "logps/chosen": -378.4911804199219,
+      "logps/rejected": -379.71209716796875,
+      "loss": 0.041,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.0814424604177475,
+      "rewards/margins": 0.0518694743514061,
+      "rewards/rejected": -0.1333119422197342,
+      "step": 1770
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.225028509122944e-06,
+      "logits/chosen": 0.13141155242919922,
+      "logits/rejected": 0.1951020061969757,
+      "logps/chosen": -364.9999694824219,
+      "logps/rejected": -351.82818603515625,
+      "loss": 0.0232,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.0604858472943306,
+      "rewards/margins": 0.04045441001653671,
+      "rewards/rejected": -0.1009402647614479,
+      "step": 1780
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.2031357088857083e-06,
+      "logits/chosen": 0.12399832904338837,
+      "logits/rejected": 0.11015460640192032,
+      "logps/chosen": -341.9513854980469,
+      "logps/rejected": -364.69537353515625,
+      "loss": 0.0387,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.07097164541482925,
+      "rewards/margins": 0.04248364642262459,
+      "rewards/rejected": -0.11345528066158295,
+      "step": 1790
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.181184197019127e-06,
+      "logits/chosen": 0.09701532870531082,
+      "logits/rejected": 0.0713806003332138,
+      "logps/chosen": -356.0148010253906,
+      "logps/rejected": -344.892822265625,
+      "loss": 0.0411,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.07698091119527817,
+      "rewards/margins": 0.04195588082075119,
+      "rewards/rejected": -0.11893679946660995,
+      "step": 1800
+    },
+    {
+      "epoch": 0.47,
+      "eval_logits/chosen": 0.07212649285793304,
+      "eval_logits/rejected": 0.11432238668203354,
+      "eval_logps/chosen": -363.9650573730469,
+      "eval_logps/rejected": -373.5191345214844,
+      "eval_loss": 0.039134953171014786,
+      "eval_rewards/accuracies": 0.6165000200271606,
+      "eval_rewards/chosen": -0.0864182636141777,
+      "eval_rewards/margins": 0.04328494146466255,
+      "eval_rewards/rejected": -0.12970322370529175,
+      "eval_runtime": 545.9891,
+      "eval_samples_per_second": 3.663,
+      "eval_steps_per_second": 0.916,
+      "step": 1800
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.159175806468126e-06,
+      "logits/chosen": 0.05868060141801834,
+      "logits/rejected": 0.0898103266954422,
+      "logps/chosen": -365.3113708496094,
+      "logps/rejected": -401.7536315917969,
+      "loss": 0.0277,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.08397440612316132,
+      "rewards/margins": 0.049311526119709015,
+      "rewards/rejected": -0.13328592479228973,
+      "step": 1810
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.1371123749269804e-06,
+      "logits/chosen": 0.09874452650547028,
+      "logits/rejected": 0.13370689749717712,
+      "logps/chosen": -374.6447448730469,
+      "logps/rejected": -385.79779052734375,
+      "loss": 0.0399,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.09536582231521606,
+      "rewards/margins": 0.04811391979455948,
+      "rewards/rejected": -0.14347973465919495,
+      "step": 1820
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.114995744685877e-06,
+      "logits/chosen": 0.08537691086530685,
+      "logits/rejected": 0.04779334366321564,
+      "logps/chosen": -337.15020751953125,
+      "logps/rejected": -377.43157958984375,
+      "loss": 0.0305,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.08057156950235367,
+      "rewards/margins": 0.06588000804185867,
+      "rewards/rejected": -0.14645157754421234,
+      "step": 1830
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.0928277624770743e-06,
+      "logits/chosen": 0.10169104486703873,
+      "logits/rejected": 0.11909240484237671,
+      "logps/chosen": -354.15704345703125,
+      "logps/rejected": -354.2380065917969,
+      "loss": 0.037,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.06953345239162445,
+      "rewards/margins": 0.061396948993206024,
+      "rewards/rejected": -0.13093039393424988,
+      "step": 1840
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.070610279320708e-06,
+      "logits/chosen": 0.09656129032373428,
+      "logits/rejected": 0.16233056783676147,
+      "logps/chosen": -371.34930419921875,
+      "logps/rejected": -339.03887939453125,
+      "loss": 0.0367,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.08772575855255127,
+      "rewards/margins": 0.026259135454893112,
+      "rewards/rejected": -0.11398488283157349,
+      "step": 1850
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.0483451503702264e-06,
+      "logits/chosen": 0.14258061349391937,
+      "logits/rejected": 0.16879236698150635,
+      "logps/chosen": -358.58465576171875,
+      "logps/rejected": -365.378662109375,
+      "loss": 0.0459,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.07283969968557358,
+      "rewards/margins": 0.03544919937849045,
+      "rewards/rejected": -0.10828889906406403,
+      "step": 1860
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.0260342347574916e-06,
+      "logits/chosen": 0.2246214896440506,
+      "logits/rejected": 0.25609272718429565,
+      "logps/chosen": -367.5299377441406,
+      "logps/rejected": -355.5799255371094,
+      "loss": 0.037,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.0721147209405899,
+      "rewards/margins": 0.04538789391517639,
+      "rewards/rejected": -0.1175026074051857,
+      "step": 1870
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.0036793954375358e-06,
+      "logits/chosen": 0.1619095355272293,
+      "logits/rejected": 0.19684650003910065,
+      "logps/chosen": -342.7215881347656,
+      "logps/rejected": -342.2190856933594,
+      "loss": 0.0396,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.06451873481273651,
+      "rewards/margins": 0.04374837130308151,
+      "rewards/rejected": -0.10826709121465683,
+      "step": 1880
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 2.981282499033009e-06,
+      "logits/chosen": 0.15326252579689026,
+      "logits/rejected": 0.17403288185596466,
+      "logps/chosen": -360.48565673828125,
+      "logps/rejected": -348.0400695800781,
+      "loss": 0.0382,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.0688900500535965,
+      "rewards/margins": 0.042695384472608566,
+      "rewards/rejected": -0.11158543825149536,
+      "step": 1890
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.9588454156783163e-06,
+      "logits/chosen": 0.2058980017900467,
+      "logits/rejected": 0.1619640439748764,
+      "logps/chosen": -347.1320495605469,
+      "logps/rejected": -354.5460205078125,
+      "loss": 0.0486,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.07155324518680573,
+      "rewards/margins": 0.03914617374539375,
+      "rewards/rejected": -0.11069941520690918,
+      "step": 1900
+    },
+    {
+      "epoch": 0.5,
+      "eval_logits/chosen": 0.12981759011745453,
+      "eval_logits/rejected": 0.1764288991689682,
+      "eval_logps/chosen": -356.7109375,
+      "eval_logps/rejected": -364.1853332519531,
+      "eval_loss": 0.038182251155376434,
+      "eval_rewards/accuracies": 0.6259999871253967,
+      "eval_rewards/chosen": -0.07916412502527237,
+      "eval_rewards/margins": 0.041205309331417084,
+      "eval_rewards/rejected": -0.12036944925785065,
+      "eval_runtime": 545.957,
+      "eval_samples_per_second": 3.663,
+      "eval_steps_per_second": 0.916,
+      "step": 1900
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.9363700188634597e-06,
+      "logits/chosen": 0.14707890152931213,
+      "logits/rejected": 0.17461742460727692,
+      "logps/chosen": -343.1737976074219,
+      "logps/rejected": -341.931640625,
+      "loss": 0.0318,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.07447867095470428,
+      "rewards/margins": 0.0252214428037405,
+      "rewards/rejected": -0.09970011562108994,
+      "step": 1910
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.9138581852776053e-06,
+      "logits/chosen": 0.14907950162887573,
+      "logits/rejected": 0.19653204083442688,
+      "logps/chosen": -341.96063232421875,
+      "logps/rejected": -369.2284240722656,
+      "loss": 0.0411,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.06778468191623688,
+      "rewards/margins": 0.048592694103717804,
+      "rewards/rejected": -0.11637736856937408,
+      "step": 1920
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8913117946523805e-06,
+      "logits/chosen": 0.18731799721717834,
+      "logits/rejected": 0.16077713668346405,
+      "logps/chosen": -330.68634033203125,
+      "logps/rejected": -323.8900146484375,
+      "loss": 0.0404,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.07359401881694794,
+      "rewards/margins": 0.038732368499040604,
+      "rewards/rejected": -0.11232636868953705,
+      "step": 1930
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8687327296049126e-06,
+      "logits/chosen": 0.12258721888065338,
+      "logits/rejected": 0.16166368126869202,
+      "logps/chosen": -354.1372375488281,
+      "logps/rejected": -357.1199951171875,
+      "loss": 0.0376,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.0759049579501152,
+      "rewards/margins": 0.02912052348256111,
+      "rewards/rejected": -0.10502549260854721,
+      "step": 1940
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8461228754806376e-06,
+      "logits/chosen": 0.11027747392654419,
+      "logits/rejected": 0.14814499020576477,
+      "logps/chosen": -322.1830139160156,
+      "logps/rejected": -353.8628234863281,
+      "loss": 0.0321,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.07510451227426529,
+      "rewards/margins": 0.04564494267106056,
+      "rewards/rejected": -0.12074945122003555,
+      "step": 1950
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.823484120195865e-06,
+      "logits/chosen": 0.12542524933815002,
+      "logits/rejected": 0.09315163642168045,
+      "logps/chosen": -271.7514343261719,
+      "logps/rejected": -326.87554931640625,
+      "loss": 0.0352,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.0656217485666275,
+      "rewards/margins": 0.046793024986982346,
+      "rewards/rejected": -0.11241476237773895,
+      "step": 1960
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.8008183540801486e-06,
+      "logits/chosen": 0.05915481597185135,
+      "logits/rejected": 0.11433479934930801,
+      "logps/chosen": -358.5248718261719,
+      "logps/rejected": -371.9332580566406,
+      "loss": 0.0302,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.07355078309774399,
+      "rewards/margins": 0.0377325564622879,
+      "rewards/rejected": -0.1112833246588707,
+      "step": 1970
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7781274697184353e-06,
+      "logits/chosen": 0.10677427053451538,
+      "logits/rejected": 0.12291960418224335,
+      "logps/chosen": -348.57696533203125,
+      "logps/rejected": -365.7822570800781,
+      "loss": 0.0441,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.0773545503616333,
+      "rewards/margins": 0.042930666357278824,
+      "rewards/rejected": -0.12028521299362183,
+      "step": 1980
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7554133617930397e-06,
+      "logits/chosen": 0.1324932873249054,
+      "logits/rejected": 0.1200353354215622,
+      "logps/chosen": -341.12213134765625,
+      "logps/rejected": -359.984130859375,
+      "loss": 0.035,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.07184281200170517,
+      "rewards/margins": 0.02733670175075531,
+      "rewards/rejected": -0.09917950630187988,
+      "step": 1990
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7326779269254363e-06,
+      "logits/chosen": 0.05920320749282837,
+      "logits/rejected": 0.10000525414943695,
+      "logps/chosen": -315.31793212890625,
+      "logps/rejected": -307.84100341796875,
+      "loss": 0.0378,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.06324402987957001,
+      "rewards/margins": 0.02069897949695587,
+      "rewards/rejected": -0.08394300192594528,
+      "step": 2000
+    },
+    {
+      "epoch": 0.52,
+      "eval_logits/chosen": 0.08077358454465866,
+      "eval_logits/rejected": 0.12937459349632263,
+      "eval_logps/chosen": -341.7246398925781,
+      "eval_logps/rejected": -345.1358642578125,
+      "eval_loss": 0.03781852871179581,
+      "eval_rewards/accuracies": 0.6290000081062317,
+      "eval_rewards/chosen": -0.06417782604694366,
+      "eval_rewards/margins": 0.03714217245578766,
+      "eval_rewards/rejected": -0.10131999850273132,
+      "eval_runtime": 545.7568,
+      "eval_samples_per_second": 3.665,
+      "eval_steps_per_second": 0.916,
+      "step": 2000
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.7099230635178954e-06,
+      "logits/chosen": 0.0865093320608139,
+      "logits/rejected": 0.12565746903419495,
+      "logps/chosen": -306.2552490234375,
+      "logps/rejected": -312.70635986328125,
+      "loss": 0.0489,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.05742295831441879,
+      "rewards/margins": 0.05094178766012192,
+      "rewards/rejected": -0.10836474597454071,
+      "step": 2010
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6871506715949608e-06,
+      "logits/chosen": 0.12608636915683746,
+      "logits/rejected": 0.12410484254360199,
+      "logps/chosen": -348.0589294433594,
+      "logps/rejected": -374.9178771972656,
+      "loss": 0.047,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.07317279279232025,
+      "rewards/margins": 0.04851686209440231,
+      "rewards/rejected": -0.12168965488672256,
+      "step": 2020
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6643626526448063e-06,
+      "logits/chosen": 0.06003843992948532,
+      "logits/rejected": 0.09717948734760284,
+      "logps/chosen": -323.10137939453125,
+      "logps/rejected": -370.0508728027344,
+      "loss": 0.0403,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.08030740171670914,
+      "rewards/margins": 0.04712501913309097,
+      "rewards/rejected": -0.1274324357509613,
+      "step": 2030
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6415609094604562e-06,
+      "logits/chosen": 0.047612786293029785,
+      "logits/rejected": 0.11089674383401871,
+      "logps/chosen": -375.27386474609375,
+      "logps/rejected": -376.94903564453125,
+      "loss": 0.0448,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.07468323409557343,
+      "rewards/margins": 0.05277440696954727,
+      "rewards/rejected": -0.1274576485157013,
+      "step": 2040
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.618747345980904e-06,
+      "logits/chosen": 0.05303157493472099,
+      "logits/rejected": 0.08489685505628586,
+      "logps/chosen": -348.9032287597656,
+      "logps/rejected": -345.81121826171875,
+      "loss": 0.0346,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.061851583421230316,
+      "rewards/margins": 0.04280518367886543,
+      "rewards/rejected": -0.10465677082538605,
+      "step": 2050
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.595923867132136e-06,
+      "logits/chosen": 0.11801224946975708,
+      "logits/rejected": 0.12615171074867249,
+      "logps/chosen": -280.94854736328125,
+      "logps/rejected": -321.2654113769531,
+      "loss": 0.0369,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.055603694170713425,
+      "rewards/margins": 0.04293707758188248,
+      "rewards/rejected": -0.0985407829284668,
+      "step": 2060
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5730923786680672e-06,
+      "logits/chosen": 0.0512966588139534,
+      "logits/rejected": 0.08032461255788803,
+      "logps/chosen": -346.1708679199219,
+      "logps/rejected": -328.2955017089844,
+      "loss": 0.0388,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.07426029443740845,
+      "rewards/margins": 0.03256069868803024,
+      "rewards/rejected": -0.1068209856748581,
+      "step": 2070
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5502547870114137e-06,
+      "logits/chosen": 0.05467385798692703,
+      "logits/rejected": 0.07844047248363495,
+      "logps/chosen": -293.09942626953125,
+      "logps/rejected": -343.6536560058594,
+      "loss": 0.0288,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.06378957629203796,
+      "rewards/margins": 0.03455866128206253,
+      "rewards/rejected": -0.0983482375741005,
+      "step": 2080
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.527412999094507e-06,
+      "logits/chosen": 0.05205492302775383,
+      "logits/rejected": 0.06727848201990128,
+      "logps/chosen": -351.3774108886719,
+      "logps/rejected": -359.9560852050781,
+      "loss": 0.036,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.07127957046031952,
+      "rewards/margins": 0.041381143033504486,
+      "rewards/rejected": -0.1126607209444046,
+      "step": 2090
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.504568922200064e-06,
+      "logits/chosen": 0.006595483515411615,
+      "logits/rejected": 0.03564555197954178,
+      "logps/chosen": -337.95941162109375,
+      "logps/rejected": -358.2017822265625,
+      "loss": 0.0316,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.07606897503137589,
+      "rewards/margins": 0.0420483723282814,
+      "rewards/rejected": -0.11811734735965729,
+      "step": 2100
+    },
+    {
+      "epoch": 0.55,
+      "eval_logits/chosen": 0.024460360407829285,
+      "eval_logits/rejected": 0.06865272670984268,
+      "eval_logps/chosen": -354.5952453613281,
+      "eval_logps/rejected": -362.2671203613281,
+      "eval_loss": 0.03751024603843689,
+      "eval_rewards/accuracies": 0.6274999976158142,
+      "eval_rewards/chosen": -0.07704848051071167,
+      "eval_rewards/margins": 0.04140273109078407,
+      "eval_rewards/rejected": -0.11845120787620544,
+      "eval_runtime": 545.8246,
+      "eval_samples_per_second": 3.664,
+      "eval_steps_per_second": 0.916,
+      "step": 2100
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.4817244638019333e-06,
+      "logits/chosen": 0.04349938780069351,
+      "logits/rejected": 0.08455810695886612,
+      "logps/chosen": -329.2096252441406,
+      "logps/rejected": -368.24212646484375,
+      "loss": 0.04,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.07342585176229477,
+      "rewards/margins": 0.05258835107088089,
+      "rewards/rejected": -0.12601420283317566,
+      "step": 2110
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.4588815314058155e-06,
+      "logits/chosen": 0.023920325562357903,
+      "logits/rejected": 0.03323778882622719,
+      "logps/chosen": -387.12799072265625,
+      "logps/rejected": -396.6259765625,
+      "loss": 0.0382,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.08190621435642242,
+      "rewards/margins": 0.0316036082804203,
+      "rewards/rejected": -0.11350981891155243,
+      "step": 2120
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.4360420323899922e-06,
+      "logits/chosen": -0.0191864762455225,
+      "logits/rejected": -0.00726687628775835,
+      "logps/chosen": -381.15777587890625,
+      "logps/rejected": -369.16033935546875,
+      "loss": 0.0442,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.08500616997480392,
+      "rewards/margins": 0.04921247437596321,
+      "rewards/rejected": -0.13421864807605743,
+      "step": 2130
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.4132078738460585e-06,
+      "logits/chosen": 0.03495310992002487,
+      "logits/rejected": 0.010554613545536995,
+      "logps/chosen": -356.111328125,
+      "logps/rejected": -376.19293212890625,
+      "loss": 0.0349,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.08702333271503448,
+      "rewards/margins": 0.03759434074163437,
+      "rewards/rejected": -0.12461767345666885,
+      "step": 2140
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.3903809624196826e-06,
+      "logits/chosen": 0.025050556287169456,
+      "logits/rejected": 0.0566801019012928,
+      "logps/chosen": -344.64984130859375,
+      "logps/rejected": -355.14349365234375,
+      "loss": 0.037,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.08903811872005463,
+      "rewards/margins": 0.04520445317029953,
+      "rewards/rejected": -0.13424256443977356,
+      "step": 2150
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.3675632041513978e-06,
+      "logits/chosen": 0.056731123477220535,
+      "logits/rejected": 0.07199566066265106,
+      "logps/chosen": -321.55328369140625,
+      "logps/rejected": -328.07220458984375,
+      "loss": 0.0356,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.08337533473968506,
+      "rewards/margins": 0.027652790769934654,
+      "rewards/rejected": -0.11102811992168427,
+      "step": 2160
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.3447565043174533e-06,
+      "logits/chosen": 0.011426372453570366,
+      "logits/rejected": 0.0008334618178196251,
+      "logps/chosen": -341.9546813964844,
+      "logps/rejected": -349.308349609375,
+      "loss": 0.0465,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.07242012023925781,
+      "rewards/margins": 0.03607422113418579,
+      "rewards/rejected": -0.1084943413734436,
+      "step": 2170
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.321962767270724e-06,
+      "logits/chosen": 0.05366501957178116,
+      "logits/rejected": 0.06784092634916306,
+      "logps/chosen": -325.31787109375,
+      "logps/rejected": -364.01702880859375,
+      "loss": 0.0416,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.0713353306055069,
+      "rewards/margins": 0.04891545698046684,
+      "rewards/rejected": -0.12025078386068344,
+      "step": 2180
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.299183896281692e-06,
+      "logits/chosen": -0.010047108866274357,
+      "logits/rejected": 0.05582508444786072,
+      "logps/chosen": -351.2960205078125,
+      "logps/rejected": -352.16510009765625,
+      "loss": 0.0408,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.08284951746463776,
+      "rewards/margins": 0.048341304063797,
+      "rewards/rejected": -0.13119080662727356,
+      "step": 2190
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2764217933795297e-06,
+      "logits/chosen": 0.009800228290259838,
+      "logits/rejected": -0.0444265678524971,
+      "logps/chosen": -372.9019775390625,
+      "logps/rejected": -373.99871826171875,
+      "loss": 0.0375,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.08768996596336365,
+      "rewards/margins": 0.03615245968103409,
+      "rewards/rejected": -0.12384243309497833,
+      "step": 2200
+    },
+    {
+      "epoch": 0.58,
+      "eval_logits/chosen": 0.0007440316840074956,
+      "eval_logits/rejected": 0.0390869677066803,
+      "eval_logps/chosen": -360.0626220703125,
+      "eval_logps/rejected": -368.81878662109375,
+      "eval_loss": 0.037584759294986725,
+      "eval_rewards/accuracies": 0.628000020980835,
+      "eval_rewards/chosen": -0.08251580595970154,
+      "eval_rewards/margins": 0.04248707741498947,
+      "eval_rewards/rejected": -0.1250028908252716,
+      "eval_runtime": 545.8698,
+      "eval_samples_per_second": 3.664,
+      "eval_steps_per_second": 0.916,
+      "step": 2200
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2536783591932786e-06,
+      "logits/chosen": 0.09655176103115082,
+      "logits/rejected": 0.03686312586069107,
+      "logps/chosen": -347.11883544921875,
+      "logps/rejected": -363.9665222167969,
+      "loss": 0.0331,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.07669743150472641,
+      "rewards/margins": 0.04332875832915306,
+      "rewards/rejected": -0.12002619355916977,
+      "step": 2210
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.230955492793149e-06,
+      "logits/chosen": 0.04199524596333504,
+      "logits/rejected": 0.07617248594760895,
+      "logps/chosen": -354.28240966796875,
+      "logps/rejected": -348.9584045410156,
+      "loss": 0.0447,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.08262089639902115,
+      "rewards/margins": 0.03869957849383354,
+      "rewards/rejected": -0.12132047116756439,
+      "step": 2220
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.208255091531947e-06,
+      "logits/chosen": 0.03687300533056259,
+      "logits/rejected": 0.02527170442044735,
+      "logps/chosen": -343.4227294921875,
+      "logps/rejected": -334.44677734375,
+      "loss": 0.0458,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.07905085384845734,
+      "rewards/margins": 0.04343415051698685,
+      "rewards/rejected": -0.12248501926660538,
+      "step": 2230
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1855790508866435e-06,
+      "logits/chosen": 0.010173076763749123,
+      "logits/rejected": 0.02444292977452278,
+      "logps/chosen": -366.0366516113281,
+      "logps/rejected": -384.393310546875,
+      "loss": 0.0212,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.07625563442707062,
+      "rewards/margins": 0.04022028297185898,
+      "rewards/rejected": -0.1164759173989296,
+      "step": 2240
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.162929264300107e-06,
+      "logits/chosen": 0.03199372440576553,
+      "logits/rejected": 0.06810581684112549,
+      "logps/chosen": -329.0974426269531,
+      "logps/rejected": -330.72076416015625,
+      "loss": 0.052,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.07751419395208359,
+      "rewards/margins": 0.03638191148638725,
+      "rewards/rejected": -0.11389610916376114,
+      "step": 2250
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1403076230230006e-06,
+      "logits/chosen": 0.05615055561065674,
+      "logits/rejected": 0.0629628524184227,
+      "logps/chosen": -341.9981689453125,
+      "logps/rejected": -308.8276672363281,
+      "loss": 0.0427,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.07570649683475494,
+      "rewards/margins": 0.03302991762757301,
+      "rewards/rejected": -0.10873641073703766,
+      "step": 2260
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.11771601595586e-06,
+      "logits/chosen": 0.05373881012201309,
+      "logits/rejected": 0.06396917998790741,
+      "logps/chosen": -320.20635986328125,
+      "logps/rejected": -357.5127868652344,
+      "loss": 0.0291,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.0627308115363121,
+      "rewards/margins": 0.040217723697423935,
+      "rewards/rejected": -0.10294853150844574,
+      "step": 2270
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0951563294913737e-06,
+      "logits/chosen": -0.031706538051366806,
+      "logits/rejected": 0.01782875694334507,
+      "logps/chosen": -345.283203125,
+      "logps/rejected": -327.1484375,
+      "loss": 0.0326,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.0846625417470932,
+      "rewards/margins": 0.028733674436807632,
+      "rewards/rejected": -0.11339621245861053,
+      "step": 2280
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0726304473568693e-06,
+      "logits/chosen": 0.03354809433221817,
+      "logits/rejected": 0.04455564543604851,
+      "logps/chosen": -325.090576171875,
+      "logps/rejected": -382.6881408691406,
+      "loss": 0.0352,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.07758642733097076,
+      "rewards/margins": 0.04382765293121338,
+      "rewards/rejected": -0.12141408026218414,
+      "step": 2290
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.050140250457023e-06,
+      "logits/chosen": 0.07273958623409271,
+      "logits/rejected": 0.04130532592535019,
+      "logps/chosen": -315.1903381347656,
+      "logps/rejected": -341.3672790527344,
+      "loss": 0.0344,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.06705816835165024,
+      "rewards/margins": 0.041755735874176025,
+      "rewards/rejected": -0.10881390422582626,
+      "step": 2300
+    },
+    {
+      "epoch": 0.6,
+      "eval_logits/chosen": 0.055425431579351425,
+      "eval_logits/rejected": 0.10016035288572311,
+      "eval_logps/chosen": -348.0062561035156,
+      "eval_logps/rejected": -351.98907470703125,
+      "eval_loss": 0.037648506462574005,
+      "eval_rewards/accuracies": 0.6315000057220459,
+      "eval_rewards/chosen": -0.07045942544937134,
+      "eval_rewards/margins": 0.037713680416345596,
+      "eval_rewards/rejected": -0.10817310214042664,
+      "eval_runtime": 545.8267,
+      "eval_samples_per_second": 3.664,
+      "eval_steps_per_second": 0.916,
+      "step": 2300
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0276876167168042e-06,
+      "logits/chosen": 0.1101096048951149,
+      "logits/rejected": 0.1835123598575592,
+      "logps/chosen": -352.8796691894531,
+      "logps/rejected": -364.6676330566406,
+      "loss": 0.0467,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.06968484818935394,
+      "rewards/margins": 0.0384695827960968,
+      "rewards/rejected": -0.10815443843603134,
+      "step": 2310
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.0052744209246682e-06,
+      "logits/chosen": 0.03556061536073685,
+      "logits/rejected": 0.07367236167192459,
+      "logps/chosen": -324.57086181640625,
+      "logps/rejected": -350.7970275878906,
+      "loss": 0.0361,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.06864863634109497,
+      "rewards/margins": 0.03674275428056717,
+      "rewards/rejected": -0.10539139807224274,
+      "step": 2320
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.9829025345760127e-06,
+      "logits/chosen": 0.04471544176340103,
+      "logits/rejected": 0.056022871285676956,
+      "logps/chosen": -386.85791015625,
+      "logps/rejected": -369.82635498046875,
+      "loss": 0.0331,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.07391612976789474,
+      "rewards/margins": 0.03182296082377434,
+      "rewards/rejected": -0.10573909431695938,
+      "step": 2330
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.9605738257169115e-06,
+      "logits/chosen": 0.0936531201004982,
+      "logits/rejected": 0.06764701753854752,
+      "logps/chosen": -318.5469665527344,
+      "logps/rejected": -350.8421325683594,
+      "loss": 0.0384,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.07255914807319641,
+      "rewards/margins": 0.0357024148106575,
+      "rewards/rejected": -0.10826156288385391,
+      "step": 2340
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.9382901587881275e-06,
+      "logits/chosen": 0.008178415708243847,
+      "logits/rejected": 0.066588394343853,
+      "logps/chosen": -330.1451721191406,
+      "logps/rejected": -346.47430419921875,
+      "loss": 0.0335,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.07381810247898102,
+      "rewards/margins": 0.034645192325115204,
+      "rewards/rejected": -0.10846330970525742,
+      "step": 2350
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.916053394469437e-06,
+      "logits/chosen": 0.03935049846768379,
+      "logits/rejected": 0.06327758729457855,
+      "logps/chosen": -295.48382568359375,
+      "logps/rejected": -335.5404357910156,
+      "loss": 0.0459,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.07076247036457062,
+      "rewards/margins": 0.04089931771159172,
+      "rewards/rejected": -0.11166179180145264,
+      "step": 2360
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.8938653895242604e-06,
+      "logits/chosen": 0.0441306047141552,
+      "logits/rejected": 0.05181695893406868,
+      "logps/chosen": -310.1441955566406,
+      "logps/rejected": -329.85321044921875,
+      "loss": 0.0417,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.0688004419207573,
+      "rewards/margins": 0.04298040270805359,
+      "rewards/rejected": -0.11178084462881088,
+      "step": 2370
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.8717279966446267e-06,
+      "logits/chosen": 0.0024087964557111263,
+      "logits/rejected": 0.06057130545377731,
+      "logps/chosen": -343.87603759765625,
+      "logps/rejected": -374.98980712890625,
+      "loss": 0.0397,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.0747489333152771,
+      "rewards/margins": 0.052171558141708374,
+      "rewards/rejected": -0.12692050635814667,
+      "step": 2380
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8496430642964698e-06,
+      "logits/chosen": -0.018443766981363297,
+      "logits/rejected": 0.03082500398159027,
+      "logps/chosen": -327.8390808105469,
+      "logps/rejected": -341.80145263671875,
+      "loss": 0.0309,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.07468292862176895,
+      "rewards/margins": 0.0353861041367054,
+      "rewards/rejected": -0.11006903648376465,
+      "step": 2390
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.827612436565286e-06,
+      "logits/chosen": -0.018162129446864128,
+      "logits/rejected": -0.0051040188409388065,
+      "logps/chosen": -328.34332275390625,
+      "logps/rejected": -339.1662292480469,
+      "loss": 0.0393,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.07663413137197495,
+      "rewards/margins": 0.03774600848555565,
+      "rewards/rejected": -0.1143801361322403,
+      "step": 2400
+    },
+    {
+      "epoch": 0.63,
+      "eval_logits/chosen": -0.02714073285460472,
+      "eval_logits/rejected": 0.01235498208552599,
+      "eval_logps/chosen": -361.4958190917969,
+      "eval_logps/rejected": -368.2056884765625,
+      "eval_loss": 0.037392448633909225,
+      "eval_rewards/accuracies": 0.6330000162124634,
+      "eval_rewards/chosen": -0.0839490070939064,
+      "eval_rewards/margins": 0.04044075310230255,
+      "eval_rewards/rejected": -0.12438975274562836,
+      "eval_runtime": 545.9401,
+      "eval_samples_per_second": 3.663,
+      "eval_steps_per_second": 0.916,
+      "step": 2400
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8056379530021492e-06,
+      "logits/chosen": -0.04801153391599655,
+      "logits/rejected": -0.007933981716632843,
+      "logps/chosen": -338.10321044921875,
+      "logps/rejected": -344.618408203125,
+      "loss": 0.0426,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.08458362519741058,
+      "rewards/margins": 0.0438532829284668,
+      "rewards/rejected": -0.1284368932247162,
+      "step": 2410
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.7837214484701154e-06,
+      "logits/chosen": -0.009363202378153801,
+      "logits/rejected": -0.0007576922071166337,
+      "logps/chosen": -360.673828125,
+      "logps/rejected": -400.9598693847656,
+      "loss": 0.0485,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.09015808999538422,
+      "rewards/margins": 0.05674852058291435,
+      "rewards/rejected": -0.14690658450126648,
+      "step": 2420
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7618647529910043e-06,
+      "logits/chosen": -0.04702354967594147,
+      "logits/rejected": 0.04674084484577179,
+      "logps/chosen": -352.09051513671875,
+      "logps/rejected": -376.13238525390625,
+      "loss": 0.0417,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.09370934963226318,
+      "rewards/margins": 0.05088300257921219,
+      "rewards/rejected": -0.14459234476089478,
+      "step": 2430
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7400696915925996e-06,
+      "logits/chosen": -0.012523381039500237,
+      "logits/rejected": 0.03584936261177063,
+      "logps/chosen": -374.0272216796875,
+      "logps/rejected": -420.229736328125,
+      "loss": 0.0249,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.09751933068037033,
+      "rewards/margins": 0.061813026666641235,
+      "rewards/rejected": -0.15933236479759216,
+      "step": 2440
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.718338084156254e-06,
+      "logits/chosen": -0.001403535483404994,
+      "logits/rejected": 0.017204532399773598,
+      "logps/chosen": -368.22369384765625,
+      "logps/rejected": -374.7392578125,
+      "loss": 0.034,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.09341777861118317,
+      "rewards/margins": 0.04063834622502327,
+      "rewards/rejected": -0.13405612111091614,
+      "step": 2450
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.6966717452649372e-06,
+      "logits/chosen": -0.04094991832971573,
+      "logits/rejected": -0.020757248625159264,
+      "logps/chosen": -364.2580871582031,
+      "logps/rejected": -398.494384765625,
+      "loss": 0.0296,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.08606752008199692,
+      "rewards/margins": 0.04812884330749512,
+      "rewards/rejected": -0.13419637084007263,
+      "step": 2460
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6750724840517103e-06,
+      "logits/chosen": 0.002780568553134799,
+      "logits/rejected": 0.02848714590072632,
+      "logps/chosen": -376.7506408691406,
+      "logps/rejected": -383.3822326660156,
+      "loss": 0.0466,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.08360211551189423,
+      "rewards/margins": 0.0551789291203022,
+      "rewards/rejected": -0.13878104090690613,
+      "step": 2470
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6535421040486686e-06,
+      "logits/chosen": -0.004565836396068335,
+      "logits/rejected": -0.002664464060217142,
+      "logps/chosen": -336.34228515625,
+      "logps/rejected": -350.0891418457031,
+      "loss": 0.0457,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.08158624917268753,
+      "rewards/margins": 0.04326051473617554,
+      "rewards/rejected": -0.12484675645828247,
+      "step": 2480
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6320824030363458e-06,
+      "logits/chosen": -0.048592709004879,
+      "logits/rejected": 0.05497083067893982,
+      "logps/chosen": -351.0393371582031,
+      "logps/rejected": -367.6898498535156,
+      "loss": 0.0431,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.08544863760471344,
+      "rewards/margins": 0.0455031618475914,
+      "rewards/rejected": -0.13095179200172424,
+      "step": 2490
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6106951728936028e-06,
+      "logits/chosen": -0.007420344743877649,
+      "logits/rejected": 0.027076199650764465,
+      "logps/chosen": -322.1276550292969,
+      "logps/rejected": -354.7091369628906,
+      "loss": 0.0501,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.09365580976009369,
+      "rewards/margins": 0.048761606216430664,
+      "rewards/rejected": -0.14241740107536316,
+      "step": 2500
+    },
+    {
+      "epoch": 0.65,
+      "eval_logits/chosen": -0.030681947246193886,
+      "eval_logits/rejected": 0.005308403167873621,
+      "eval_logps/chosen": -374.56878662109375,
+      "eval_logps/rejected": -385.8455810546875,
+      "eval_loss": 0.03733815625309944,
+      "eval_rewards/accuracies": 0.6265000104904175,
+      "eval_rewards/chosen": -0.09702196717262268,
+      "eval_rewards/margins": 0.04500770941376686,
+      "eval_rewards/rejected": -0.14202965795993805,
+      "eval_runtime": 546.009,
+      "eval_samples_per_second": 3.663,
+      "eval_steps_per_second": 0.916,
+      "step": 2500
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5893821994479996e-06,
+      "logits/chosen": 0.01902809366583824,
+      "logits/rejected": 0.04253349453210831,
+      "logps/chosen": -372.2449645996094,
+      "logps/rejected": -404.98687744140625,
+      "loss": 0.0346,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.09531093388795853,
+      "rewards/margins": 0.058450400829315186,
+      "rewards/rejected": -0.15376132726669312,
+      "step": 2510
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5681452623266868e-06,
+      "logits/chosen": 0.004775013774633408,
+      "logits/rejected": 0.018243897706270218,
+      "logps/chosen": -391.09014892578125,
+      "logps/rejected": -392.67510986328125,
+      "loss": 0.0339,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.09394645690917969,
+      "rewards/margins": 0.04599941149353981,
+      "rewards/rejected": -0.139945849776268,
+      "step": 2520
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5469861348078014e-06,
+      "logits/chosen": -0.01885377988219261,
+      "logits/rejected": -0.01242881640791893,
+      "logps/chosen": -364.51019287109375,
+      "logps/rejected": -374.30877685546875,
+      "loss": 0.0484,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0994337946176529,
+      "rewards/margins": 0.041206371039152145,
+      "rewards/rejected": -0.14064016938209534,
+      "step": 2530
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5259065836724035e-06,
+      "logits/chosen": -0.019729355350136757,
+      "logits/rejected": 0.008078807964920998,
+      "logps/chosen": -359.4223937988281,
+      "logps/rejected": -383.68096923828125,
+      "loss": 0.0318,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.09255600720643997,
+      "rewards/margins": 0.04096725583076477,
+      "rewards/rejected": -0.13352327048778534,
+      "step": 2540
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.5049083690569456e-06,
+      "logits/chosen": -0.06140296906232834,
+      "logits/rejected": -0.017861146479845047,
+      "logps/chosen": -376.9162292480469,
+      "logps/rejected": -377.25262451171875,
+      "loss": 0.0441,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.10426129400730133,
+      "rewards/margins": 0.04735453054308891,
+      "rewards/rejected": -0.15161582827568054,
+      "step": 2550
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4839932443063057e-06,
+      "logits/chosen": 0.030533451586961746,
+      "logits/rejected": 0.007474319543689489,
+      "logps/chosen": -335.75897216796875,
+      "logps/rejected": -344.4579772949219,
+      "loss": 0.0474,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.0987381637096405,
+      "rewards/margins": 0.04033283516764641,
+      "rewards/rejected": -0.1390710026025772,
+      "step": 2560
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4631629558273803e-06,
+      "logits/chosen": 0.023572232574224472,
+      "logits/rejected": 0.009190462529659271,
+      "logps/chosen": -380.98858642578125,
+      "logps/rejected": -396.1246337890625,
+      "loss": 0.0419,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.0922577902674675,
+      "rewards/margins": 0.05769185349345207,
+      "rewards/rejected": -0.14994964003562927,
+      "step": 2570
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.4424192429432657e-06,
+      "logits/chosen": 0.07105614989995956,
+      "logits/rejected": 0.044981636106967926,
+      "logps/chosen": -398.01300048828125,
+      "logps/rejected": -352.92376708984375,
+      "loss": 0.0433,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.09294580668210983,
+      "rewards/margins": 0.04410626366734505,
+      "rewards/rejected": -0.13705205917358398,
+      "step": 2580
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.421763837748016e-06,
+      "logits/chosen": 0.010540916584432125,
+      "logits/rejected": 0.0199065450578928,
+      "logps/chosen": -365.95538330078125,
+      "logps/rejected": -411.8755798339844,
+      "loss": 0.0435,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.10294492542743683,
+      "rewards/margins": 0.0462300069630146,
+      "rewards/rejected": -0.14917492866516113,
+      "step": 2590
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.401198464962021e-06,
+      "logits/chosen": 0.03292522951960564,
+      "logits/rejected": -0.010659189894795418,
+      "logps/chosen": -360.2803039550781,
+      "logps/rejected": -370.08087158203125,
+      "loss": 0.03,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.09060030430555344,
+      "rewards/margins": 0.04847833514213562,
+      "rewards/rejected": -0.13907863199710846,
+      "step": 2600
+    },
+    {
+      "epoch": 0.68,
+      "eval_logits/chosen": -0.00636453228071332,
+      "eval_logits/rejected": 0.03250597044825554,
+      "eval_logps/chosen": -372.34637451171875,
+      "eval_logps/rejected": -384.5748291015625,
+      "eval_loss": 0.037237949669361115,
+      "eval_rewards/accuracies": 0.628000020980835,
+      "eval_rewards/chosen": -0.09479956328868866,
+      "eval_rewards/margins": 0.04595935344696045,
+      "eval_rewards/rejected": -0.1407589167356491,
+      "eval_runtime": 545.883,
+      "eval_samples_per_second": 3.664,
+      "eval_steps_per_second": 0.916,
+      "step": 2600
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.3807248417879896e-06,
+      "logits/chosen": 0.02124766632914543,
+      "logits/rejected": 0.005783993750810623,
+      "logps/chosen": -339.2605285644531,
+      "logps/rejected": -335.9830322265625,
+      "loss": 0.0335,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.08307014405727386,
+      "rewards/margins": 0.041536845266819,
+      "rewards/rejected": -0.12460698932409286,
+      "step": 2610
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3603446777675665e-06,
+      "logits/chosen": -0.002301536500453949,
+      "logits/rejected": -0.007860266603529453,
+      "logps/chosen": -349.5294494628906,
+      "logps/rejected": -364.48309326171875,
+      "loss": 0.0343,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.0928717777132988,
+      "rewards/margins": 0.03952028229832649,
+      "rewards/rejected": -0.13239207863807678,
+      "step": 2620
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3400596746385817e-06,
+      "logits/chosen": 0.00916803628206253,
+      "logits/rejected": 0.01391880214214325,
+      "logps/chosen": -381.51617431640625,
+      "logps/rejected": -387.173095703125,
+      "loss": 0.0376,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.0942435935139656,
+      "rewards/margins": 0.047798626124858856,
+      "rewards/rejected": -0.14204223453998566,
+      "step": 2630
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3198715261929587e-06,
+      "logits/chosen": 0.034454770386219025,
+      "logits/rejected": 0.06986280530691147,
+      "logps/chosen": -409.3059387207031,
+      "logps/rejected": -363.70660400390625,
+      "loss": 0.0396,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.0899733379483223,
+      "rewards/margins": 0.035371553152799606,
+      "rewards/rejected": -0.1253449022769928,
+      "step": 2640
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.2997819181352823e-06,
+      "logits/chosen": 0.03116777166724205,
+      "logits/rejected": 0.01502093393355608,
+      "logps/chosen": -385.01220703125,
+      "logps/rejected": -391.8352966308594,
+      "loss": 0.0337,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.08890195190906525,
+      "rewards/margins": 0.044724203646183014,
+      "rewards/rejected": -0.13362616300582886,
+      "step": 2650
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2797925279420454e-06,
+      "logits/chosen": 0.06440167129039764,
+      "logits/rejected": 0.09733875095844269,
+      "logps/chosen": -374.38348388671875,
+      "logps/rejected": -375.3147277832031,
+      "loss": 0.0386,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.08725802600383759,
+      "rewards/margins": 0.03805771470069885,
+      "rewards/rejected": -0.12531574070453644,
+      "step": 2660
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2599050247215764e-06,
+      "logits/chosen": 0.06206239387392998,
+      "logits/rejected": 0.060874927788972855,
+      "logps/chosen": -366.9176330566406,
+      "logps/rejected": -395.0933532714844,
+      "loss": 0.0292,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.08329717069864273,
+      "rewards/margins": 0.0593089684844017,
+      "rewards/rejected": -0.14260615408420563,
+      "step": 2670
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2401210690746705e-06,
+      "logits/chosen": 0.049030765891075134,
+      "logits/rejected": 0.13232190907001495,
+      "logps/chosen": -356.2248840332031,
+      "logps/rejected": -359.5081481933594,
+      "loss": 0.0297,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.0947878286242485,
+      "rewards/margins": 0.0313425287604332,
+      "rewards/rejected": -0.1261303573846817,
+      "step": 2680
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2204423129559306e-06,
+      "logits/chosen": -0.04865225777029991,
+      "logits/rejected": 0.04526076465845108,
+      "logps/chosen": -389.2451477050781,
+      "logps/rejected": -390.4857177734375,
+      "loss": 0.0401,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.09563232213258743,
+      "rewards/margins": 0.0318005234003067,
+      "rewards/rejected": -0.12743285298347473,
+      "step": 2690
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.20087039953583e-06,
+      "logits/chosen": 0.03936707600951195,
+      "logits/rejected": 0.040941834449768066,
+      "logps/chosen": -360.7561950683594,
+      "logps/rejected": -379.5013427734375,
+      "loss": 0.0445,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.08158311247825623,
+      "rewards/margins": 0.0466335192322731,
+      "rewards/rejected": -0.12821663916110992,
+      "step": 2700
+    },
+    {
+      "epoch": 0.71,
+      "eval_logits/chosen": -0.0008235538261942565,
+      "eval_logits/rejected": 0.03942526876926422,
+      "eval_logps/chosen": -370.2886962890625,
+      "eval_logps/rejected": -381.6030578613281,
+      "eval_loss": 0.03717409819364548,
+      "eval_rewards/accuracies": 0.6255000233650208,
+      "eval_rewards/chosen": -0.09274187684059143,
+      "eval_rewards/margins": 0.04504526033997536,
+      "eval_rewards/rejected": -0.1377871334552765,
+      "eval_runtime": 545.9505,
+      "eval_samples_per_second": 3.663,
+      "eval_steps_per_second": 0.916,
+      "step": 2700
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.181406963063507e-06,
+      "logits/chosen": 0.022398801520466805,
+      "logits/rejected": 0.00628666440024972,
+      "logps/chosen": -368.5730285644531,
+      "logps/rejected": -354.22003173828125,
+      "loss": 0.04,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.09365259855985641,
+      "rewards/margins": 0.035916369408369064,
+      "rewards/rejected": -0.12956897914409637,
+      "step": 2710
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1620536287303052e-06,
+      "logits/chosen": 0.04186456650495529,
+      "logits/rejected": 0.07249974459409714,
+      "logps/chosen": -348.0757751464844,
+      "logps/rejected": -374.15216064453125,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.09035263955593109,
+      "rewards/margins": 0.04991690441966057,
+      "rewards/rejected": -0.14026953279972076,
+      "step": 2720
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1428120125340717e-06,
+      "logits/chosen": 0.07720698416233063,
+      "logits/rejected": -0.000991599983535707,
+      "logps/chosen": -353.360107421875,
+      "logps/rejected": -362.484130859375,
+      "loss": 0.044,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.08344024419784546,
+      "rewards/margins": 0.04450567439198494,
+      "rewards/rejected": -0.1279458999633789,
+      "step": 2730
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.123683721144223e-06,
+      "logits/chosen": 0.02114693447947502,
+      "logits/rejected": 0.011313997209072113,
+      "logps/chosen": -359.75347900390625,
+      "logps/rejected": -375.54345703125,
+      "loss": 0.0396,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.09027194231748581,
+      "rewards/margins": 0.054341159760951996,
+      "rewards/rejected": -0.1446131020784378,
+      "step": 2740
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.1046703517675848e-06,
+      "logits/chosen": 0.04119975492358208,
+      "logits/rejected": 0.03296607732772827,
+      "logps/chosen": -331.5726013183594,
+      "logps/rejected": -364.89666748046875,
+      "loss": 0.0444,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.08693388849496841,
+      "rewards/margins": 0.035929180681705475,
+      "rewards/rejected": -0.12286306917667389,
+      "step": 2750
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.085773492015028e-06,
+      "logits/chosen": 0.017023924738168716,
+      "logits/rejected": 0.04285174608230591,
+      "logps/chosen": -372.84906005859375,
+      "logps/rejected": -359.8387451171875,
+      "loss": 0.0388,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.0875464603304863,
+      "rewards/margins": 0.04696623235940933,
+      "rewards/rejected": -0.13451267778873444,
+      "step": 2760
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.0669947197689034e-06,
+      "logits/chosen": 0.02556072548031807,
+      "logits/rejected": 0.04901180416345596,
+      "logps/chosen": -308.6146545410156,
+      "logps/rejected": -353.16522216796875,
+      "loss": 0.0435,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.08332064002752304,
+      "rewards/margins": 0.03549160063266754,
+      "rewards/rejected": -0.11881224066019058,
+      "step": 2770
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.048335603051291e-06,
+      "logits/chosen": 0.003486833069473505,
+      "logits/rejected": 0.054467201232910156,
+      "logps/chosen": -356.0873718261719,
+      "logps/rejected": -378.89324951171875,
+      "loss": 0.0423,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.08631832897663116,
+      "rewards/margins": 0.047230057418346405,
+      "rewards/rejected": -0.13354839384555817,
+      "step": 2780
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0297976998930665e-06,
+      "logits/chosen": 0.03703586384654045,
+      "logits/rejected": 0.034189797937870026,
+      "logps/chosen": -355.9411926269531,
+      "logps/rejected": -358.441162109375,
+      "loss": 0.0366,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.07967467606067657,
+      "rewards/margins": 0.043767232447862625,
+      "rewards/rejected": -0.1234418973326683,
+      "step": 2790
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0113825582038078e-06,
+      "logits/chosen": 0.06561113893985748,
+      "logits/rejected": 0.082789346575737,
+      "logps/chosen": -358.72772216796875,
+      "logps/rejected": -340.65576171875,
+      "loss": 0.0359,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.091328464448452,
+      "rewards/margins": 0.03305225819349289,
+      "rewards/rejected": -0.12438073009252548,
+      "step": 2800
+    },
+    {
+      "epoch": 0.73,
+      "eval_logits/chosen": 0.04762628674507141,
+      "eval_logits/rejected": 0.09258405864238739,
+      "eval_logps/chosen": -359.7132568359375,
+      "eval_logps/rejected": -368.16766357421875,
+      "eval_loss": 0.03694864735007286,
+      "eval_rewards/accuracies": 0.637499988079071,
+      "eval_rewards/chosen": -0.08216645568609238,
+      "eval_rewards/margins": 0.04218530282378197,
+      "eval_rewards/rejected": -0.12435176223516464,
+      "eval_runtime": 545.8876,
+      "eval_samples_per_second": 3.664,
+      "eval_steps_per_second": 0.916,
+      "step": 2800
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.930917156425477e-07,
+      "logits/chosen": 0.06098620221018791,
+      "logits/rejected": 0.13049380481243134,
+      "logps/chosen": -314.9092102050781,
+      "logps/rejected": -343.68585205078125,
+      "loss": 0.0417,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.06992046535015106,
+      "rewards/margins": 0.057133208960294724,
+      "rewards/rejected": -0.12705367803573608,
+      "step": 2810
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.749266994893756e-07,
+      "logits/chosen": 0.05095028877258301,
+      "logits/rejected": 0.10279621183872223,
+      "logps/chosen": -321.6026306152344,
+      "logps/rejected": -364.67626953125,
+      "loss": 0.0384,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.08201439678668976,
+      "rewards/margins": 0.03452789783477783,
+      "rewards/rejected": -0.11654229462146759,
+      "step": 2820
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.56889026517913e-07,
+      "logits/chosen": 0.08180872350931168,
+      "logits/rejected": 0.13031435012817383,
+      "logps/chosen": -350.0518493652344,
+      "logps/rejected": -357.79571533203125,
+      "loss": 0.0461,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.0801549106836319,
+      "rewards/margins": 0.045111220329999924,
+      "rewards/rejected": -0.12526611983776093,
+      "step": 2830
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.389802028686617e-07,
+      "logits/chosen": 0.08433196693658829,
+      "logits/rejected": 0.09517958015203476,
+      "logps/chosen": -291.19061279296875,
+      "logps/rejected": -303.54461669921875,
+      "loss": 0.0374,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.07507316023111343,
+      "rewards/margins": 0.04450450837612152,
+      "rewards/rejected": -0.11957766860723495,
+      "step": 2840
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 9.212017239232427e-07,
+      "logits/chosen": 0.09773010015487671,
+      "logits/rejected": 0.15997204184532166,
+      "logps/chosen": -342.41326904296875,
+      "logps/rejected": -395.35614013671875,
+      "loss": 0.0397,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.07223471254110336,
+      "rewards/margins": 0.052558619529008865,
+      "rewards/rejected": -0.12479333579540253,
+      "step": 2850
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 9.03555074179533e-07,
+      "logits/chosen": 0.07126441597938538,
+      "logits/rejected": 0.14168313145637512,
+      "logps/chosen": -344.53961181640625,
+      "logps/rejected": -368.14300537109375,
+      "loss": 0.0324,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.07919279485940933,
+      "rewards/margins": 0.051605600863695145,
+      "rewards/rejected": -0.13079838454723358,
+      "step": 2860
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.860417271277067e-07,
+      "logits/chosen": 0.07190985977649689,
+      "logits/rejected": 0.11755266040563583,
+      "logps/chosen": -317.7795715332031,
+      "logps/rejected": -351.59735107421875,
+      "loss": 0.0496,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.07767555862665176,
+      "rewards/margins": 0.04091283679008484,
+      "rewards/rejected": -0.118588387966156,
+      "step": 2870
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.686631451272029e-07,
+      "logits/chosen": 0.059776317328214645,
+      "logits/rejected": 0.053002677857875824,
+      "logps/chosen": -345.6569519042969,
+      "logps/rejected": -334.6269836425781,
+      "loss": 0.0437,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.07967682182788849,
+      "rewards/margins": 0.041141774505376816,
+      "rewards/rejected": -0.120818592607975,
+      "step": 2880
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.514207792846168e-07,
+      "logits/chosen": 0.09157435595989227,
+      "logits/rejected": 0.08564511686563492,
+      "logps/chosen": -330.6698303222656,
+      "logps/rejected": -374.27740478515625,
+      "loss": 0.0497,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.08086385577917099,
+      "rewards/margins": 0.04518595337867737,
+      "rewards/rejected": -0.12604978680610657,
+      "step": 2890
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.343160693325356e-07,
+      "logits/chosen": 0.0515270009636879,
+      "logits/rejected": 0.06402511894702911,
+      "logps/chosen": -312.79669189453125,
+      "logps/rejected": -340.7076110839844,
+      "loss": 0.0454,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.08327269554138184,
+      "rewards/margins": 0.038690682500600815,
+      "rewards/rejected": -0.12196336686611176,
+      "step": 2900
+    },
+    {
+      "epoch": 0.76,
+      "eval_logits/chosen": 0.03624504432082176,
+      "eval_logits/rejected": 0.07880530506372452,
+      "eval_logps/chosen": -363.6590576171875,
+      "eval_logps/rejected": -374.6195068359375,
+      "eval_loss": 0.03678791970014572,
+      "eval_rewards/accuracies": 0.6340000033378601,
+      "eval_rewards/chosen": -0.08611226081848145,
+      "eval_rewards/margins": 0.04469134658575058,
+      "eval_rewards/rejected": -0.13080359995365143,
+      "eval_runtime": 545.731,
+      "eval_samples_per_second": 3.665,
+      "eval_steps_per_second": 0.916,
+      "step": 2900
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.173504435093174e-07,
+      "logits/chosen": 0.04690922051668167,
+      "logits/rejected": 0.06846790015697479,
+      "logps/chosen": -373.71429443359375,
+      "logps/rejected": -387.99420166015625,
+      "loss": 0.0397,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.09103576838970184,
+      "rewards/margins": 0.036001771688461304,
+      "rewards/rejected": -0.12703752517700195,
+      "step": 2910
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.00525318439836e-07,
+      "logits/chosen": 0.06871423125267029,
+      "logits/rejected": 0.0787956491112709,
+      "logps/chosen": -354.68341064453125,
+      "logps/rejected": -353.810791015625,
+      "loss": 0.0335,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.08626209199428558,
+      "rewards/margins": 0.04417368024587631,
+      "rewards/rejected": -0.1304357945919037,
+      "step": 2920
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.838420990171927e-07,
+      "logits/chosen": 0.045043814927339554,
+      "logits/rejected": 0.09084521234035492,
+      "logps/chosen": -390.70123291015625,
+      "logps/rejected": -389.6815185546875,
+      "loss": 0.0322,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.0797785222530365,
+      "rewards/margins": 0.050302654504776,
+      "rewards/rejected": -0.1300811767578125,
+      "step": 2930
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.673021782854084e-07,
+      "logits/chosen": 0.06473254412412643,
+      "logits/rejected": 0.06569622457027435,
+      "logps/chosen": -334.15618896484375,
+      "logps/rejected": -390.80804443359375,
+      "loss": 0.049,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.08651457726955414,
+      "rewards/margins": 0.04847872629761696,
+      "rewards/rejected": -0.1349932849407196,
+      "step": 2940
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.509069373231039e-07,
+      "logits/chosen": 0.07900504767894745,
+      "logits/rejected": 0.05937303975224495,
+      "logps/chosen": -382.0351867675781,
+      "logps/rejected": -403.01422119140625,
+      "loss": 0.033,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.08951593935489655,
+      "rewards/margins": 0.051025889813899994,
+      "rewards/rejected": -0.14054183661937714,
+      "step": 2950
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.346577451281822e-07,
+      "logits/chosen": 0.02836814895272255,
+      "logits/rejected": 0.06619714200496674,
+      "logps/chosen": -337.5442810058594,
+      "logps/rejected": -352.7892761230469,
+      "loss": 0.0499,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.09030725061893463,
+      "rewards/margins": 0.03954026475548744,
+      "rewards/rejected": -0.12984752655029297,
+      "step": 2960
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 7.185559585035138e-07,
+      "logits/chosen": 0.03714161738753319,
+      "logits/rejected": 0.06548570096492767,
+      "logps/chosen": -369.113037109375,
+      "logps/rejected": -373.710205078125,
+      "loss": 0.0539,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.08472782373428345,
+      "rewards/margins": 0.036620158702135086,
+      "rewards/rejected": -0.12134796380996704,
+      "step": 2970
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 7.026029219436504e-07,
+      "logits/chosen": 0.06240475922822952,
+      "logits/rejected": 0.07280924171209335,
+      "logps/chosen": -389.9732360839844,
+      "logps/rejected": -399.385009765625,
+      "loss": 0.0347,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.08604921400547028,
+      "rewards/margins": 0.05031247064471245,
+      "rewards/rejected": -0.13636168837547302,
+      "step": 2980
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.867999675225523e-07,
+      "logits/chosen": 0.06554336845874786,
+      "logits/rejected": 0.05114714428782463,
+      "logps/chosen": -378.0634765625,
+      "logps/rejected": -363.99188232421875,
+      "loss": 0.0318,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.08256720006465912,
+      "rewards/margins": 0.03878363221883774,
+      "rewards/rejected": -0.12135082483291626,
+      "step": 2990
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.711484147823663e-07,
+      "logits/chosen": 0.03155375272035599,
+      "logits/rejected": 0.03810650482773781,
+      "logps/chosen": -349.73095703125,
+      "logps/rejected": -372.1285400390625,
+      "loss": 0.0422,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.08169834315776825,
+      "rewards/margins": 0.057754795998334885,
+      "rewards/rejected": -0.13945314288139343,
+      "step": 3000
+    },
+    {
+      "epoch": 0.79,
+      "eval_logits/chosen": 0.035446614027023315,
+      "eval_logits/rejected": 0.0778362900018692,
+      "eval_logps/chosen": -364.7430114746094,
+      "eval_logps/rejected": -375.5085754394531,
+      "eval_loss": 0.036815349012613297,
+      "eval_rewards/accuracies": 0.6349999904632568,
+      "eval_rewards/chosen": -0.08719620108604431,
+      "eval_rewards/margins": 0.04449642822146416,
+      "eval_rewards/rejected": -0.13169263303279877,
+      "eval_runtime": 545.987,
+      "eval_samples_per_second": 3.663,
+      "eval_steps_per_second": 0.916,
+      "step": 3000
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.556495706232413e-07,
+      "logits/chosen": 0.060652829706668854,
+      "logits/rejected": 0.08359045535326004,
+      "logps/chosen": -356.06365966796875,
+      "logps/rejected": -361.7148132324219,
+      "loss": 0.0417,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.08335570245981216,
+      "rewards/margins": 0.04776981100440025,
+      "rewards/rejected": -0.1311255246400833,
+      "step": 3010
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.403047291942057e-07,
+      "logits/chosen": 0.04646974802017212,
+      "logits/rejected": 0.05604598671197891,
+      "logps/chosen": -359.0965881347656,
+      "logps/rejected": -381.06622314453125,
+      "loss": 0.0309,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.08178387582302094,
+      "rewards/margins": 0.0419863797724247,
+      "rewards/rejected": -0.12377025187015533,
+      "step": 3020
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.251151717851023e-07,
+      "logits/chosen": 0.015839481726288795,
+      "logits/rejected": 0.026612812653183937,
+      "logps/chosen": -384.18182373046875,
+      "logps/rejected": -420.2266540527344,
+      "loss": 0.0505,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.09612445533275604,
+      "rewards/margins": 0.059503644704818726,
+      "rewards/rejected": -0.15562811493873596,
+      "step": 3030
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 6.100821667196041e-07,
+      "logits/chosen": 0.07615561038255692,
+      "logits/rejected": 0.10104755312204361,
+      "logps/chosen": -382.831787109375,
+      "logps/rejected": -364.81298828125,
+      "loss": 0.0368,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.07995451986789703,
+      "rewards/margins": 0.04422109201550484,
+      "rewards/rejected": -0.12417560815811157,
+      "step": 3040
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.952069692493062e-07,
+      "logits/chosen": 0.043039046227931976,
+      "logits/rejected": 0.08566012233495712,
+      "logps/chosen": -378.10992431640625,
+      "logps/rejected": -387.7176208496094,
+      "loss": 0.0332,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.07815327495336533,
+      "rewards/margins": 0.05145417898893356,
+      "rewards/rejected": -0.12960746884346008,
+      "step": 3050
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.80490821448918e-07,
+      "logits/chosen": 0.07400702685117722,
+      "logits/rejected": 0.06525278091430664,
+      "logps/chosen": -302.47467041015625,
+      "logps/rejected": -326.3743591308594,
+      "loss": 0.0319,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.08332636952400208,
+      "rewards/margins": 0.045311104506254196,
+      "rewards/rejected": -0.12863746285438538,
+      "step": 3060
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.659349521125459e-07,
+      "logits/chosen": 0.022895220667123795,
+      "logits/rejected": 0.059297479689121246,
+      "logps/chosen": -319.58843994140625,
+      "logps/rejected": -370.5553283691406,
+      "loss": 0.0441,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.08423876017332077,
+      "rewards/margins": 0.05277423933148384,
+      "rewards/rejected": -0.1370130032300949,
+      "step": 3070
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.5154057665109e-07,
+      "logits/chosen": 0.062281131744384766,
+      "logits/rejected": 0.0732177197933197,
+      "logps/chosen": -364.0726623535156,
+      "logps/rejected": -367.91485595703125,
+      "loss": 0.033,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.08627767115831375,
+      "rewards/margins": 0.04671277850866318,
+      "rewards/rejected": -0.13299044966697693,
+      "step": 3080
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.373088969907586e-07,
+      "logits/chosen": 0.010433671995997429,
+      "logits/rejected": 0.06592810153961182,
+      "logps/chosen": -336.7867126464844,
+      "logps/rejected": -380.44720458984375,
+      "loss": 0.0302,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.07851160317659378,
+      "rewards/margins": 0.05124678090214729,
+      "rewards/rejected": -0.12975837290287018,
+      "step": 3090
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.23241101472709e-07,
+      "logits/chosen": 0.07944878190755844,
+      "logits/rejected": 0.05929890275001526,
+      "logps/chosen": -330.8269958496094,
+      "logps/rejected": -356.66485595703125,
+      "loss": 0.0401,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.07750087231397629,
+      "rewards/margins": 0.04321221262216568,
+      "rewards/rejected": -0.12071307003498077,
+      "step": 3100
+    },
+    {
+      "epoch": 0.81,
+      "eval_logits/chosen": 0.034525465220212936,
+      "eval_logits/rejected": 0.07782163470983505,
+      "eval_logps/chosen": -361.9237976074219,
+      "eval_logps/rejected": -372.1985168457031,
+      "eval_loss": 0.03679418936371803,
+      "eval_rewards/accuracies": 0.6349999904632568,
+      "eval_rewards/chosen": -0.08437702059745789,
+      "eval_rewards/margins": 0.04400559142231941,
+      "eval_rewards/rejected": -0.1283826231956482,
+      "eval_runtime": 545.9386,
+      "eval_samples_per_second": 3.663,
+      "eval_steps_per_second": 0.916,
+      "step": 3100
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.09338364753818e-07,
+      "logits/chosen": 0.0181669220328331,
+      "logits/rejected": 0.08202117681503296,
+      "logps/chosen": -373.920654296875,
+      "logps/rejected": -394.57464599609375,
+      "loss": 0.0481,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.08286649733781815,
+      "rewards/margins": 0.05774938315153122,
+      "rewards/rejected": -0.14061586558818817,
+      "step": 3110
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.956018477086005e-07,
+      "logits/chosen": 0.09085467457771301,
+      "logits/rejected": 0.06141798943281174,
+      "logps/chosen": -363.1712951660156,
+      "logps/rejected": -348.1229248046875,
+      "loss": 0.0363,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.08368535339832306,
+      "rewards/margins": 0.04151429235935211,
+      "rewards/rejected": -0.12519964575767517,
+      "step": 3120
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.820326973322764e-07,
+      "logits/chosen": 0.08711811155080795,
+      "logits/rejected": 0.08708525449037552,
+      "logps/chosen": -365.7955017089844,
+      "logps/rejected": -349.32879638671875,
+      "loss": 0.0338,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.07129119336605072,
+      "rewards/margins": 0.06465110182762146,
+      "rewards/rejected": -0.13594229519367218,
+      "step": 3130
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.686320466449981e-07,
+      "logits/chosen": 0.04728562757372856,
+      "logits/rejected": 0.054091863334178925,
+      "logps/chosen": -310.783447265625,
+      "logps/rejected": -346.09674072265625,
+      "loss": 0.0378,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.07566200941801071,
+      "rewards/margins": 0.05796832591295242,
+      "rewards/rejected": -0.13363032042980194,
+      "step": 3140
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.554010145972418e-07,
+      "logits/chosen": 0.054544102400541306,
+      "logits/rejected": 0.08318766951560974,
+      "logps/chosen": -365.8963623046875,
+      "logps/rejected": -345.4806213378906,
+      "loss": 0.0359,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.08155672252178192,
+      "rewards/margins": 0.04104948788881302,
+      "rewards/rejected": -0.12260621786117554,
+      "step": 3150
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.4234070597637455e-07,
+      "logits/chosen": 0.03188851475715637,
+      "logits/rejected": 0.10954336076974869,
+      "logps/chosen": -362.37628173828125,
+      "logps/rejected": -372.03070068359375,
+      "loss": 0.0324,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.09456036984920502,
+      "rewards/margins": 0.030316606163978577,
+      "rewards/rejected": -0.12487699091434479,
+      "step": 3160
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.2945221131440783e-07,
+      "logits/chosen": 0.04685058444738388,
+      "logits/rejected": 0.06858415901660919,
+      "logps/chosen": -331.08538818359375,
+      "logps/rejected": -342.9245300292969,
+      "loss": 0.0486,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -0.08771758526563644,
+      "rewards/margins": 0.04199302941560745,
+      "rewards/rejected": -0.1297105997800827,
+      "step": 3170
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.167366067969381e-07,
+      "logits/chosen": 0.02194828912615776,
+      "logits/rejected": 0.039914991706609726,
+      "logps/chosen": -351.51318359375,
+      "logps/rejected": -380.74798583984375,
+      "loss": 0.0383,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.08071055263280869,
+      "rewards/margins": 0.04869350045919418,
+      "rewards/rejected": -0.12940405309200287,
+      "step": 3180
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.041949541732826e-07,
+      "logits/chosen": 0.092781662940979,
+      "logits/rejected": 0.12146890163421631,
+      "logps/chosen": -380.84161376953125,
+      "logps/rejected": -362.23211669921875,
+      "loss": 0.0461,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.08747414499521255,
+      "rewards/margins": 0.03685353696346283,
+      "rewards/rejected": -0.12432768195867538,
+      "step": 3190
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.9182830066782614e-07,
+      "logits/chosen": 0.05555509403347969,
+      "logits/rejected": 0.050342459231615067,
+      "logps/chosen": -368.70513916015625,
+      "logps/rejected": -359.16888427734375,
+      "loss": 0.0455,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.08999396860599518,
+      "rewards/margins": 0.03136914223432541,
+      "rewards/rejected": -0.12136310338973999,
+      "step": 3200
+    },
+    {
+      "epoch": 0.84,
+      "eval_logits/chosen": 0.043583810329437256,
+      "eval_logits/rejected": 0.08712277561426163,
+      "eval_logps/chosen": -361.7043151855469,
+      "eval_logps/rejected": -371.32403564453125,
+      "eval_loss": 0.03676051273941994,
+      "eval_rewards/accuracies": 0.6334999799728394,
+      "eval_rewards/chosen": -0.0841575413942337,
+      "eval_rewards/margins": 0.04335058107972145,
+      "eval_rewards/rejected": -0.12750811874866486,
+      "eval_runtime": 545.9555,
+      "eval_samples_per_second": 3.663,
+      "eval_steps_per_second": 0.916,
+      "step": 3200
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.796376788925771e-07,
+      "logits/chosen": 0.018586691468954086,
+      "logits/rejected": 0.09413950145244598,
+      "logps/chosen": -328.3235778808594,
+      "logps/rejected": -340.29248046875,
+      "loss": 0.0548,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.0838296040892601,
+      "rewards/margins": 0.040683310478925705,
+      "rewards/rejected": -0.12451291084289551,
+      "step": 3210
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.676241067609465e-07,
+      "logits/chosen": 0.04617813602089882,
+      "logits/rejected": 0.033351633697748184,
+      "logps/chosen": -321.99627685546875,
+      "logps/rejected": -355.32525634765625,
+      "loss": 0.0414,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.0823250561952591,
+      "rewards/margins": 0.044673480093479156,
+      "rewards/rejected": -0.12699854373931885,
+      "step": 3220
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.5578858740274976e-07,
+      "logits/chosen": 0.04456084221601486,
+      "logits/rejected": 0.08620361983776093,
+      "logps/chosen": -382.176025390625,
+      "logps/rejected": -372.05157470703125,
+      "loss": 0.0395,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.083371601998806,
+      "rewards/margins": 0.03640345484018326,
+      "rewards/rejected": -0.11977505683898926,
+      "step": 3230
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.44132109080447e-07,
+      "logits/chosen": 0.03808216005563736,
+      "logits/rejected": 0.11458434909582138,
+      "logps/chosen": -343.13671875,
+      "logps/rejected": -343.605224609375,
+      "loss": 0.0384,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -0.08665122836828232,
+      "rewards/margins": 0.02820972166955471,
+      "rewards/rejected": -0.11486093699932098,
+      "step": 3240
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.3265564510662344e-07,
+      "logits/chosen": 0.07402416318655014,
+      "logits/rejected": 0.11608324199914932,
+      "logps/chosen": -329.2789306640625,
+      "logps/rejected": -336.4209899902344,
+      "loss": 0.0359,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.07753623276948929,
+      "rewards/margins": 0.047694385051727295,
+      "rewards/rejected": -0.125230610370636,
+      "step": 3250
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.213601537627195e-07,
+      "logits/chosen": 0.09460578113794327,
+      "logits/rejected": 0.08506201207637787,
+      "logps/chosen": -342.90802001953125,
+      "logps/rejected": -366.2782897949219,
+      "loss": 0.0305,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.08474042266607285,
+      "rewards/margins": 0.037823766469955444,
+      "rewards/rejected": -0.12256418168544769,
+      "step": 3260
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 3.1024657821901063e-07,
+      "logits/chosen": 0.0904582291841507,
+      "logits/rejected": 0.07600073516368866,
+      "logps/chosen": -327.2945861816406,
+      "logps/rejected": -349.3861389160156,
+      "loss": 0.0269,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.07415864616632462,
+      "rewards/margins": 0.04656793922185898,
+      "rewards/rejected": -0.1207265853881836,
+      "step": 3270
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.9931584645585654e-07,
+      "logits/chosen": 0.11054618656635284,
+      "logits/rejected": 0.10943951457738876,
+      "logps/chosen": -369.9562683105469,
+      "logps/rejected": -407.5022888183594,
+      "loss": 0.0431,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.08715574443340302,
+      "rewards/margins": 0.048971183598041534,
+      "rewards/rejected": -0.13612693548202515,
+      "step": 3280
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.885688711862136e-07,
+      "logits/chosen": 0.07641705870628357,
+      "logits/rejected": 0.07921125739812851,
+      "logps/chosen": -332.45367431640625,
+      "logps/rejected": -327.5888671875,
+      "loss": 0.0393,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.07755440473556519,
+      "rewards/margins": 0.034201376140117645,
+      "rewards/rejected": -0.11175577342510223,
+      "step": 3290
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.7800654977942486e-07,
+      "logits/chosen": 0.05387617275118828,
+      "logits/rejected": 0.09707017242908478,
+      "logps/chosen": -376.83563232421875,
+      "logps/rejected": -338.41668701171875,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -0.08481625467538834,
+      "rewards/margins": 0.036756731569767,
+      "rewards/rejected": -0.12157298624515533,
+      "step": 3300
+    },
+    {
+      "epoch": 0.86,
+      "eval_logits/chosen": 0.04915827140212059,
+      "eval_logits/rejected": 0.09358298033475876,
+      "eval_logps/chosen": -359.51458740234375,
+      "eval_logps/rejected": -368.5754699707031,
+      "eval_loss": 0.036767907440662384,
+      "eval_rewards/accuracies": 0.6349999904632568,
+      "eval_rewards/chosen": -0.08196782320737839,
+      "eval_rewards/margins": 0.042791757732629776,
+      "eval_rewards/rejected": -0.12475959211587906,
+      "eval_runtime": 545.8185,
+      "eval_samples_per_second": 3.664,
+      "eval_steps_per_second": 0.916,
+      "step": 3300
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.6762976418628797e-07,
+      "logits/chosen": 0.07799197733402252,
+      "logits/rejected": 0.07325412333011627,
+      "logps/chosen": -330.1886901855469,
+      "logps/rejected": -340.3829345703125,
+      "loss": 0.0329,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.0796724408864975,
+      "rewards/margins": 0.036878086626529694,
+      "rewards/rejected": -0.1165505200624466,
+      "step": 3310
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.5743938086541354e-07,
+      "logits/chosen": 0.079283706843853,
+      "logits/rejected": 0.07669277489185333,
+      "logps/chosen": -372.9908752441406,
+      "logps/rejected": -385.1440124511719,
+      "loss": 0.0292,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.0772135779261589,
+      "rewards/margins": 0.0662803053855896,
+      "rewards/rejected": -0.1434938907623291,
+      "step": 3320
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.4743625071087574e-07,
+      "logits/chosen": 0.02787538431584835,
+      "logits/rejected": 0.0959104374051094,
+      "logps/chosen": -356.8220520019531,
+      "logps/rejected": -377.5616149902344,
+      "loss": 0.0427,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.09016017615795135,
+      "rewards/margins": 0.0506926067173481,
+      "rewards/rejected": -0.14085277915000916,
+      "step": 3330
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.3762120898116498e-07,
+      "logits/chosen": 0.07221674174070358,
+      "logits/rejected": 0.08215569704771042,
+      "logps/chosen": -345.12359619140625,
+      "logps/rejected": -347.78363037109375,
+      "loss": 0.0391,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.07958263903856277,
+      "rewards/margins": 0.029747938737273216,
+      "rewards/rejected": -0.10933057963848114,
+      "step": 3340
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.2799507522944048e-07,
+      "logits/chosen": 0.12004667520523071,
+      "logits/rejected": 0.11475028842687607,
+      "logps/chosen": -373.525390625,
+      "logps/rejected": -343.428466796875,
+      "loss": 0.0314,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.07973743975162506,
+      "rewards/margins": 0.029728572815656662,
+      "rewards/rejected": -0.10946600139141083,
+      "step": 3350
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.1855865323510056e-07,
+      "logits/chosen": 0.10749117285013199,
+      "logits/rejected": 0.12759584188461304,
+      "logps/chosen": -366.70648193359375,
+      "logps/rejected": -383.4859619140625,
+      "loss": 0.0339,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.0783984437584877,
+      "rewards/margins": 0.04649888351559639,
+      "rewards/rejected": -0.12489732354879379,
+      "step": 3360
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.0931273093666575e-07,
+      "logits/chosen": 0.11750154197216034,
+      "logits/rejected": 0.12564271688461304,
+      "logps/chosen": -342.0103454589844,
+      "logps/rejected": -381.0246887207031,
+      "loss": 0.0433,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.07662364095449448,
+      "rewards/margins": 0.04676859825849533,
+      "rewards/rejected": -0.12339223921298981,
+      "step": 3370
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.002580803659873e-07,
+      "logits/chosen": 0.08527069538831711,
+      "logits/rejected": 0.059754401445388794,
+      "logps/chosen": -373.43756103515625,
+      "logps/rejected": -378.9043884277344,
+      "loss": 0.0349,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.08346894383430481,
+      "rewards/margins": 0.04092109948396683,
+      "rewards/rejected": -0.12439002841711044,
+      "step": 3380
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.913954575837826e-07,
+      "logits/chosen": 0.12140188366174698,
+      "logits/rejected": 0.14931640028953552,
+      "logps/chosen": -376.1172180175781,
+      "logps/rejected": -387.53985595703125,
+      "loss": 0.0544,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.08679510653018951,
+      "rewards/margins": 0.04618311673402786,
+      "rewards/rejected": -0.13297823071479797,
+      "step": 3390
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.827256026165028e-07,
+      "logits/chosen": 0.08391343057155609,
+      "logits/rejected": 0.050997696816921234,
+      "logps/chosen": -350.81414794921875,
+      "logps/rejected": -374.4341735839844,
+      "loss": 0.0415,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.08157356083393097,
+      "rewards/margins": 0.06317374110221863,
+      "rewards/rejected": -0.1447473168373108,
+      "step": 3400
+    },
+    {
+      "epoch": 0.89,
+      "eval_logits/chosen": 0.049248889088630676,
+      "eval_logits/rejected": 0.0925382599234581,
+      "eval_logps/chosen": -362.0814514160156,
+      "eval_logps/rejected": -371.9386901855469,
+      "eval_loss": 0.03667452186346054,
+      "eval_rewards/accuracies": 0.6365000009536743,
+      "eval_rewards/chosen": -0.08453461527824402,
+      "eval_rewards/margins": 0.04358814284205437,
+      "eval_rewards/rejected": -0.12812276184558868,
+      "eval_runtime": 545.8064,
+      "eval_samples_per_second": 3.664,
+      "eval_steps_per_second": 0.916,
+      "step": 3400
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.7424923939454274e-07,
+      "logits/chosen": 0.11083859205245972,
+      "logits/rejected": 0.09370967000722885,
+      "logps/chosen": -367.8453369140625,
+      "logps/rejected": -375.06512451171875,
+      "loss": 0.0437,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.08840908110141754,
+      "rewards/margins": 0.03927050530910492,
+      "rewards/rejected": -0.12767958641052246,
+      "step": 3410
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.6596707569179304e-07,
+      "logits/chosen": 0.04985417053103447,
+      "logits/rejected": 0.06979259103536606,
+      "logps/chosen": -342.79449462890625,
+      "logps/rejected": -352.67376708984375,
+      "loss": 0.0342,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -0.07754086703062057,
+      "rewards/margins": 0.053954072296619415,
+      "rewards/rejected": -0.13149493932724,
+      "step": 3420
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.578798030665385e-07,
+      "logits/chosen": 0.07979889214038849,
+      "logits/rejected": 0.07929590344429016,
+      "logps/chosen": -368.12603759765625,
+      "logps/rejected": -341.88787841796875,
+      "loss": 0.0372,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.08373133838176727,
+      "rewards/margins": 0.03338909149169922,
+      "rewards/rejected": -0.1171204224228859,
+      "step": 3430
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.499880968037165e-07,
+      "logits/chosen": 0.042693786323070526,
+      "logits/rejected": 0.11750195920467377,
+      "logps/chosen": -369.6103820800781,
+      "logps/rejected": -383.6126403808594,
+      "loss": 0.0457,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.08840308338403702,
+      "rewards/margins": 0.05164941027760506,
+      "rewards/rejected": -0.14005249738693237,
+      "step": 3440
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.4229261585852805e-07,
+      "logits/chosen": 0.029912296682596207,
+      "logits/rejected": 0.10099168866872787,
+      "logps/chosen": -348.51287841796875,
+      "logps/rejected": -372.84783935546875,
+      "loss": 0.0474,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.09279443323612213,
+      "rewards/margins": 0.04674132168292999,
+      "rewards/rejected": -0.13953575491905212,
+      "step": 3450
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.3479400280141886e-07,
+      "logits/chosen": 0.06193657964468002,
+      "logits/rejected": 0.05271482467651367,
+      "logps/chosen": -314.01336669921875,
+      "logps/rejected": -321.3383483886719,
+      "loss": 0.0373,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.07553164660930634,
+      "rewards/margins": 0.03188655152916908,
+      "rewards/rejected": -0.10741820186376572,
+      "step": 3460
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.2749288376442044e-07,
+      "logits/chosen": 0.09809266030788422,
+      "logits/rejected": 0.09477825462818146,
+      "logps/chosen": -329.39422607421875,
+      "logps/rejected": -358.54327392578125,
+      "loss": 0.0377,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.08319079875946045,
+      "rewards/margins": 0.040943823754787445,
+      "rewards/rejected": -0.1241346225142479,
+      "step": 3470
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.203898683888713e-07,
+      "logits/chosen": 0.09441249072551727,
+      "logits/rejected": 0.10563359409570694,
+      "logps/chosen": -340.467529296875,
+      "logps/rejected": -365.0688171386719,
+      "loss": 0.0413,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.08817145973443985,
+      "rewards/margins": 0.04508964717388153,
+      "rewards/rejected": -0.13326111435890198,
+      "step": 3480
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.1348554977451132e-07,
+      "logits/chosen": 0.08656655251979828,
+      "logits/rejected": 0.07914017140865326,
+      "logps/chosen": -328.9432373046875,
+      "logps/rejected": -381.4420166015625,
+      "loss": 0.0367,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.08895920217037201,
+      "rewards/margins": 0.04924055188894272,
+      "rewards/rejected": -0.13819976150989532,
+      "step": 3490
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 1.0678050442995802e-07,
+      "logits/chosen": 0.04111575707793236,
+      "logits/rejected": 0.041034113615751266,
+      "logps/chosen": -340.8121032714844,
+      "logps/rejected": -344.55120849609375,
+      "loss": 0.0399,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.08154693990945816,
+      "rewards/margins": 0.0527295246720314,
+      "rewards/rejected": -0.13427644968032837,
+      "step": 3500
+    },
+    {
+      "epoch": 0.92,
+      "eval_logits/chosen": 0.0507049635052681,
+      "eval_logits/rejected": 0.09372657537460327,
+      "eval_logps/chosen": -362.8265075683594,
+      "eval_logps/rejected": -372.8226623535156,
+      "eval_loss": 0.0366637259721756,
+      "eval_rewards/accuracies": 0.6324999928474426,
+      "eval_rewards/chosen": -0.08527969568967819,
+      "eval_rewards/margins": 0.04372705891728401,
+      "eval_rewards/rejected": -0.1290067583322525,
+      "eval_runtime": 546.0606,
+      "eval_samples_per_second": 3.663,
+      "eval_steps_per_second": 0.916,
+      "step": 3500
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 1.0027529222456755e-07,
+      "logits/chosen": 0.08684168756008148,
+      "logits/rejected": 0.06390564143657684,
+      "logps/chosen": -348.2730712890625,
+      "logps/rejected": -373.5179443359375,
+      "loss": 0.0296,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.08695141971111298,
+      "rewards/margins": 0.04123169928789139,
+      "rewards/rejected": -0.12818311154842377,
+      "step": 3510
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 9.397045634168766e-08,
+      "logits/chosen": 0.038009434938430786,
+      "logits/rejected": 0.121595099568367,
+      "logps/chosen": -351.69281005859375,
+      "logps/rejected": -353.6463928222656,
+      "loss": 0.0416,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0778459906578064,
+      "rewards/margins": 0.04862997680902481,
+      "rewards/rejected": -0.1264759600162506,
+      "step": 3520
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.78665232332998e-08,
+      "logits/chosen": 0.06455771625041962,
+      "logits/rejected": 0.1094939261674881,
+      "logps/chosen": -343.566650390625,
+      "logps/rejected": -379.09088134765625,
+      "loss": 0.0351,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.08551275730133057,
+      "rewards/margins": 0.04307129234075546,
+      "rewards/rejected": -0.12858405709266663,
+      "step": 3530
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 8.196400257606208e-08,
+      "logits/chosen": 0.03702528402209282,
+      "logits/rejected": 0.08741430938243866,
+      "logps/chosen": -365.49432373046875,
+      "logps/rejected": -367.8874816894531,
+      "loss": 0.0359,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.0883045345544815,
+      "rewards/margins": 0.0368974395096302,
+      "rewards/rejected": -0.1252019703388214,
+      "step": 3540
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.626338722875076e-08,
+      "logits/chosen": 0.0906023234128952,
+      "logits/rejected": 0.05330279469490051,
+      "logps/chosen": -367.0556640625,
+      "logps/rejected": -372.4805908203125,
+      "loss": 0.038,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.09004491567611694,
+      "rewards/margins": 0.036292098462581635,
+      "rewards/rejected": -0.12633702158927917,
+      "step": 3550
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.076515319110688e-08,
+      "logits/chosen": 0.11597539484500885,
+      "logits/rejected": 0.0733027309179306,
+      "logps/chosen": -377.59466552734375,
+      "logps/rejected": -374.91522216796875,
+      "loss": 0.0432,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.08780401200056076,
+      "rewards/margins": 0.0429513193666935,
+      "rewards/rejected": -0.13075533509254456,
+      "step": 3560
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.54697595640899e-08,
+      "logits/chosen": 0.14302846789360046,
+      "logits/rejected": 0.08493879437446594,
+      "logps/chosen": -361.47222900390625,
+      "logps/rejected": -380.40802001953125,
+      "loss": 0.0414,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.08389062434434891,
+      "rewards/margins": 0.04273436591029167,
+      "rewards/rejected": -0.12662498652935028,
+      "step": 3570
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.037764851154426e-08,
+      "logits/chosen": 0.05772379785776138,
+      "logits/rejected": 0.1054513081908226,
+      "logps/chosen": -363.78582763671875,
+      "logps/rejected": -362.5559387207031,
+      "loss": 0.0294,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.08675915002822876,
+      "rewards/margins": 0.0439150407910347,
+      "rewards/rejected": -0.13067419826984406,
+      "step": 3580
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.548924522327748e-08,
+      "logits/chosen": 0.029304370284080505,
+      "logits/rejected": 0.11757569015026093,
+      "logps/chosen": -359.40325927734375,
+      "logps/rejected": -383.1105651855469,
+      "loss": 0.032,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.08611874282360077,
+      "rewards/margins": 0.047209080308675766,
+      "rewards/rejected": -0.13332784175872803,
+      "step": 3590
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.0804957879556915e-08,
+      "logits/chosen": 0.053399138152599335,
+      "logits/rejected": 0.10144010931253433,
+      "logps/chosen": -406.5163269042969,
+      "logps/rejected": -388.9293212890625,
+      "loss": 0.0386,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.09166134893894196,
+      "rewards/margins": 0.04550682380795479,
+      "rewards/rejected": -0.13716816902160645,
+      "step": 3600
+    },
+    {
+      "epoch": 0.94,
+      "eval_logits/chosen": 0.04790234938263893,
+      "eval_logits/rejected": 0.09088514745235443,
+      "eval_logps/chosen": -363.0745544433594,
+      "eval_logps/rejected": -373.1802673339844,
+      "eval_loss": 0.03666089475154877,
+      "eval_rewards/accuracies": 0.6330000162124634,
+      "eval_rewards/chosen": -0.08552772551774979,
+      "eval_rewards/margins": 0.04383661970496178,
+      "eval_rewards/rejected": -0.12936435639858246,
+      "eval_runtime": 545.9387,
+      "eval_samples_per_second": 3.663,
+      "eval_steps_per_second": 0.916,
+      "step": 3600
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 4.632517761702815e-08,
+      "logits/chosen": 0.08091746270656586,
+      "logits/rejected": 0.0940876379609108,
+      "logps/chosen": -363.04736328125,
+      "logps/rejected": -372.74114990234375,
+      "loss": 0.0451,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.07975031435489655,
+      "rewards/margins": 0.05104423686861992,
+      "rewards/rejected": -0.13079455494880676,
+      "step": 3610
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.205027849605359e-08,
+      "logits/chosen": 0.0881701111793518,
+      "logits/rejected": 0.06205165386199951,
+      "logps/chosen": -303.1463928222656,
+      "logps/rejected": -342.3080749511719,
+      "loss": 0.0412,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.07364924252033234,
+      "rewards/margins": 0.04020664840936661,
+      "rewards/rejected": -0.11385589838027954,
+      "step": 3620
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.798061746947995e-08,
+      "logits/chosen": 0.09008945524692535,
+      "logits/rejected": 0.06884221732616425,
+      "logps/chosen": -425.0069274902344,
+      "logps/rejected": -379.8099060058594,
+      "loss": 0.0216,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.08133997023105621,
+      "rewards/margins": 0.0427660197019577,
+      "rewards/rejected": -0.12410600483417511,
+      "step": 3630
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.411653435283158e-08,
+      "logits/chosen": 0.06556277722120285,
+      "logits/rejected": 0.09353873878717422,
+      "logps/chosen": -386.49285888671875,
+      "logps/rejected": -384.0513610839844,
+      "loss": 0.0411,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.09560206532478333,
+      "rewards/margins": 0.039342962205410004,
+      "rewards/rejected": -0.13494503498077393,
+      "step": 3640
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.04583517959367e-08,
+      "logits/chosen": 0.052277348935604095,
+      "logits/rejected": 0.046373482793569565,
+      "logps/chosen": -375.5270080566406,
+      "logps/rejected": -373.5906066894531,
+      "loss": 0.0364,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.07689857482910156,
+      "rewards/margins": 0.055169038474559784,
+      "rewards/rejected": -0.13206762075424194,
+      "step": 3650
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.7006375255985984e-08,
+      "logits/chosen": 0.06302618980407715,
+      "logits/rejected": 0.06741125881671906,
+      "logps/chosen": -318.1965026855469,
+      "logps/rejected": -331.21453857421875,
+      "loss": 0.034,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.07973267138004303,
+      "rewards/margins": 0.04354851320385933,
+      "rewards/rejected": -0.12328118085861206,
+      "step": 3660
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.3760892972027328e-08,
+      "logits/chosen": 0.06619513034820557,
+      "logits/rejected": 0.11819871515035629,
+      "logps/chosen": -373.35748291015625,
+      "logps/rejected": -358.6504821777344,
+      "loss": 0.0412,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.08528436720371246,
+      "rewards/margins": 0.044048283249139786,
+      "rewards/rejected": -0.12933266162872314,
+      "step": 3670
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.072217594089765e-08,
+      "logits/chosen": 0.13347527384757996,
+      "logits/rejected": 0.09470942616462708,
+      "logps/chosen": -320.5458068847656,
+      "logps/rejected": -335.824951171875,
+      "loss": 0.0397,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.07533252984285355,
+      "rewards/margins": 0.03474471718072891,
+      "rewards/rejected": -0.11007724702358246,
+      "step": 3680
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.789047789459375e-08,
+      "logits/chosen": 0.05744536966085434,
+      "logits/rejected": 0.08061154186725616,
+      "logps/chosen": -362.21209716796875,
+      "logps/rejected": -378.00335693359375,
+      "loss": 0.0439,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.08542350679636002,
+      "rewards/margins": 0.043891094624996185,
+      "rewards/rejected": -0.1293146014213562,
+      "step": 3690
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.5266035279088708e-08,
+      "logits/chosen": 0.06346717476844788,
+      "logits/rejected": 0.07365471124649048,
+      "logps/chosen": -402.5925598144531,
+      "logps/rejected": -365.1163024902344,
+      "loss": 0.0372,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.08496888726949692,
+      "rewards/margins": 0.04230805113911629,
+      "rewards/rejected": -0.1272769272327423,
+      "step": 3700
+    },
+    {
+      "epoch": 0.97,
+      "eval_logits/chosen": 0.04801753908395767,
+      "eval_logits/rejected": 0.09104061126708984,
+      "eval_logps/chosen": -363.4133605957031,
+      "eval_logps/rejected": -373.5262451171875,
+      "eval_loss": 0.03667059913277626,
+      "eval_rewards/accuracies": 0.637499988079071,
+      "eval_rewards/chosen": -0.08586656302213669,
+      "eval_rewards/margins": 0.04384375363588333,
+      "eval_rewards/rejected": -0.1297103315591812,
+      "eval_runtime": 545.7678,
+      "eval_samples_per_second": 3.665,
+      "eval_steps_per_second": 0.916,
+      "step": 3700
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.2849067234584623e-08,
+      "logits/chosen": 0.05220942944288254,
+      "logits/rejected": 0.0869947299361229,
+      "logps/chosen": -338.590576171875,
+      "logps/rejected": -357.0976257324219,
+      "loss": 0.0345,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.08234889060258865,
+      "rewards/margins": 0.042240582406520844,
+      "rewards/rejected": -0.1245894581079483,
+      "step": 3710
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.0639775577218625e-08,
+      "logits/chosen": 0.0788496658205986,
+      "logits/rejected": 0.09466052800416946,
+      "logps/chosen": -363.7762756347656,
+      "logps/rejected": -385.69989013671875,
+      "loss": 0.0306,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.07673437148332596,
+      "rewards/margins": 0.051070310175418854,
+      "rewards/rejected": -0.12780466675758362,
+      "step": 3720
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.638344782207486e-09,
+      "logits/chosen": 0.03938276693224907,
+      "logits/rejected": 0.08784598857164383,
+      "logps/chosen": -393.9554748535156,
+      "logps/rejected": -406.38482666015625,
+      "loss": 0.0356,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.09428262710571289,
+      "rewards/margins": 0.03793628141283989,
+      "rewards/rejected": -0.13221892714500427,
+      "step": 3730
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 6.84494196844715e-09,
+      "logits/chosen": 0.08683189004659653,
+      "logits/rejected": 0.12081396579742432,
+      "logps/chosen": -344.96807861328125,
+      "logps/rejected": -362.85516357421875,
+      "loss": 0.0368,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.08126471191644669,
+      "rewards/margins": 0.044216252863407135,
+      "rewards/rejected": -0.12548096477985382,
+      "step": 3740
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 5.259716884556121e-09,
+      "logits/chosen": 0.0422549732029438,
+      "logits/rejected": 0.07403327524662018,
+      "logps/chosen": -394.56243896484375,
+      "logps/rejected": -378.70654296875,
+      "loss": 0.0275,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.08536888659000397,
+      "rewards/margins": 0.045694462954998016,
+      "rewards/rejected": -0.13106335699558258,
+      "step": 3750
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 3.882801896372967e-09,
+      "logits/chosen": 0.01860896311700344,
+      "logits/rejected": 0.08341463655233383,
+      "logps/chosen": -366.6634216308594,
+      "logps/rejected": -373.90216064453125,
+      "loss": 0.0336,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.08629665523767471,
+      "rewards/margins": 0.03330114483833313,
+      "rewards/rejected": -0.11959780752658844,
+      "step": 3760
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 2.7143119759026614e-09,
+      "logits/chosen": 0.055999159812927246,
+      "logits/rejected": 0.120489202439785,
+      "logps/chosen": -347.31268310546875,
+      "logps/rejected": -386.26239013671875,
+      "loss": 0.0314,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.07796981930732727,
+      "rewards/margins": 0.04153292626142502,
+      "rewards/rejected": -0.11950274556875229,
+      "step": 3770
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.754344691717591e-09,
+      "logits/chosen": 0.036989547312259674,
+      "logits/rejected": 0.04935828596353531,
+      "logps/chosen": -339.2876892089844,
+      "logps/rejected": -330.2525329589844,
+      "loss": 0.0384,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.07570748031139374,
+      "rewards/margins": 0.043455857783555984,
+      "rewards/rejected": -0.11916334927082062,
+      "step": 3780
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.0029802008096335e-09,
+      "logits/chosen": 0.10811793804168701,
+      "logits/rejected": 0.06557553261518478,
+      "logps/chosen": -347.88677978515625,
+      "logps/rejected": -349.9596252441406,
+      "loss": 0.0382,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.08263111114501953,
+      "rewards/margins": 0.04796644672751427,
+      "rewards/rejected": -0.1305975615978241,
+      "step": 3790
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.602812418974534e-10,
+      "logits/chosen": 0.03417276591062546,
+      "logits/rejected": 0.07545647025108337,
+      "logps/chosen": -344.6578674316406,
+      "logps/rejected": -355.7608947753906,
+      "loss": 0.033,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -0.09350161999464035,
+      "rewards/margins": 0.030212825164198875,
+      "rewards/rejected": -0.12371446192264557,
+      "step": 3800
+    },
+    {
+      "epoch": 0.99,
+      "eval_logits/chosen": 0.04812031611800194,
+      "eval_logits/rejected": 0.09107756614685059,
+      "eval_logps/chosen": -363.3738098144531,
+      "eval_logps/rejected": -373.5426025390625,
+      "eval_loss": 0.03665079548954964,
+      "eval_rewards/accuracies": 0.6324999928474426,
+      "eval_rewards/chosen": -0.08582703769207001,
+      "eval_rewards/margins": 0.04389968514442444,
+      "eval_rewards/rejected": -0.12972670793533325,
+      "eval_runtime": 545.8267,
+      "eval_samples_per_second": 3.664,
+      "eval_steps_per_second": 0.916,
+      "step": 3800
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.2629313018819312e-10,
+      "logits/chosen": 0.05122808739542961,
+      "logits/rejected": 0.11236433684825897,
+      "logps/chosen": -381.97100830078125,
+      "logps/rejected": -399.4225158691406,
+      "loss": 0.0314,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -0.08122538030147552,
+      "rewards/margins": 0.03989487886428833,
+      "rewards/rejected": -0.12112025916576385,
+      "step": 3810
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.0437535929996855e-12,
+      "logits/chosen": 0.0683218464255333,
+      "logits/rejected": 0.040451761335134506,
+      "logps/chosen": -367.6907043457031,
+      "logps/rejected": -393.71368408203125,
+      "loss": 0.031,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.0849466547369957,
+      "rewards/margins": 0.05080549046397209,
+      "rewards/rejected": -0.1357521414756775,
+      "step": 3820
+    },
+    {
+      "epoch": 1.0,
+      "step": 3821,
+      "total_flos": 0.0,
+      "train_loss": 0.04185271042490998,
+      "train_runtime": 56342.1754,
+      "train_samples_per_second": 1.085,
+      "train_steps_per_second": 0.068
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 3821,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}